mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-13 16:46:55 +00:00
Compare commits
72 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 195f19217f | |||
| 59d8d93832 | |||
| 59494d5089 | |||
| 019e80a218 | |||
| 9546baf1ab | |||
| 60d8e8a20b | |||
| 24ca61eb6e | |||
| d92c563b9e | |||
| e087044658 | |||
| 16d381fc0e | |||
| 1021d7228a | |||
| 0a246e3736 | |||
| 380ed40b47 | |||
| 679ea238de | |||
| baadaccc30 | |||
| 87d47a6e5d | |||
| fba0b34f19 | |||
| c9eeb2fa8a | |||
| 2f83d6789b | |||
| 7a4a3d27c6 | |||
| 4c5e73b2f2 | |||
| 4c44bc649a | |||
| 3b49842df0 | |||
| 848b330825 | |||
| 270a003c55 | |||
| 8b57076194 | |||
| d31bd3cd10 | |||
| 698ebdfb3f | |||
| c7233d6624 | |||
| 493a2cc1ba | |||
| 3ebb426abe | |||
| 5aac224a97 | |||
| 1b6ae33ce0 | |||
| 537d34b8cd | |||
| 3fdf2964c8 | |||
| 2e5874f839 | |||
| 5d05897ae0 | |||
| 6850482247 | |||
| b00b7ab8f1 | |||
| 924958bab5 | |||
| 968ec4a8be | |||
| 8d34b4d101 | |||
| 882ad4c113 | |||
| e9728192e2 | |||
| a206a0779e | |||
| 6cce3d60bb | |||
| 42433584ab | |||
| ba6a0f25d9 | |||
| 5e3010c6b5 | |||
| bc888931fd | |||
| 4ac7c56c89 | |||
| ddacce6e75 | |||
| 31cb720471 | |||
| 603bdea516 | |||
| a076ae4045 | |||
| 8a8be12f0b | |||
| 2ecf6b4575 | |||
| 2aa0148454 | |||
| 3289d40ce9 | |||
| 849837e262 | |||
| 727a10e111 | |||
| 3747d19ce5 | |||
| 1148e76279 | |||
| 320b788a50 | |||
| 3c31eaf06f | |||
| fe2516ee86 | |||
| 7ca69eb39c | |||
| 95627cb601 | |||
| d97e059c3c | |||
| d900e11a09 | |||
| f1ff9a36bc | |||
| 276eea1fba |
@@ -34,7 +34,8 @@ e2e-vault = { max-threads = 1 }
|
||||
|
||||
# Reliability / fault-injection e2e tests each spawn a single-node 4-disk RustFS
|
||||
# server and manipulate its disk directories at runtime (crates/e2e_test:
|
||||
# reliability_disk_fault_test, degraded_read_eof_regression_test / dist-13). They
|
||||
# reliability_disk_fault_test, degraded_read_eof_regression_test / dist-13, and
|
||||
# replacement_privileged_e2e_test when explicitly run as root on Linux). They
|
||||
# are correct in isolation but resource-heavy; serialize them under nextest's
|
||||
# process boundary (serial_test's #[serial] does not cross it) so several 4-disk
|
||||
# servers never run at once. ci-7's nightly picks these up via the e2e suite;
|
||||
@@ -90,7 +91,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
# e2e-reliability test-group note above). The matching ci-profile override is at
|
||||
# the end of the file, after [profile.ci] is declared.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
|
||||
test-group = 'e2e-reliability'
|
||||
|
||||
[[profile.default.overrides]]
|
||||
@@ -155,7 +156,7 @@ retries = 2
|
||||
# quarantine: no retries, just single-threaded so several 4-disk servers never
|
||||
# run concurrently when ci-7's nightly runs the full e2e suite.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
|
||||
test-group = 'e2e-reliability'
|
||||
|
||||
# Serialize the multipart crash-consistency scenarios under the ci profile too
|
||||
@@ -218,7 +219,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
# the nightly profile derives its set as "the replication module MINUS this
|
||||
# allowlist", so any new replication test lands in nightly by default (never
|
||||
# silently unrun) until it is explicitly blessed as fast here. Keep the two
|
||||
# regexes byte-identical. Count invariant: 20 here + 36 nightly = 56 total
|
||||
# regexes byte-identical. Count invariant: 20 here + 49 nightly = 69 total
|
||||
# (authority: `cargo nextest list`; docs/testing/e2e-suite-inventory.md).
|
||||
# HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane
|
||||
# (#4724) because they set a loopback (127.0.0.1) replication target that the
|
||||
@@ -344,7 +345,7 @@ path = "junit.xml"
|
||||
# object_lambda) — too heavy for the merge budget; they run in ci-7's
|
||||
# nightly 4-node lane.
|
||||
# * replication_extension_test — repl-1 already splits it into the PR
|
||||
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (27 slow) lanes and reserves
|
||||
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (49 slow) lanes and reserves
|
||||
# it for those, so e2e-full does not double-run it.
|
||||
# * #[ignore]d tests — nextest skips them by default (no --run-ignored); the
|
||||
# manual-localhost:9000 reliant/policy tests are ci-13's migration.
|
||||
@@ -383,7 +384,7 @@ path = "junit.xml"
|
||||
# quarantine: no retries, just single-threaded so several 4-disk servers never
|
||||
# run concurrently.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
|
||||
test-group = 'e2e-reliability'
|
||||
|
||||
[[profile.e2e-full.overrides]]
|
||||
|
||||
@@ -17,9 +17,11 @@
|
||||
# =============================================================================
|
||||
#
|
||||
# Metric source: the KMS operation-policy choke point in
|
||||
# crates/kms/src/policy.rs. All label values are bounded static strings
|
||||
# (operation, op_class, outcome, error_class, backend, scope); key identifiers,
|
||||
# key material, and tokens never appear in labels.
|
||||
# crates/kms/src/policy.rs, except KmsKeyRotationOverdue, which reads the
|
||||
# label-less key-lifecycle gauge published by the deletion worker's sweep
|
||||
# (crates/kms/src/deletion_worker.rs). All label values are bounded static
|
||||
# strings (operation, op_class, outcome, error_class, backend, scope); key
|
||||
# identifiers, key material, and tokens never appear in labels.
|
||||
#
|
||||
# Response procedures: docs/operations/kms-observability-runbook.md
|
||||
#
|
||||
@@ -212,3 +214,38 @@ groups:
|
||||
circuit_open until the half-open probe succeeds or returns
|
||||
a non-retryable failure.
|
||||
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendcircuitopen"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 7. KmsKeyRotationOverdue
|
||||
# The least recently rotated usable key has gone more than 400
|
||||
# days without a rotation (measured from creation for keys with
|
||||
# no recorded rotation). Direct gauge state published by the
|
||||
# deletion worker's sweep, so no traffic guard applies; the
|
||||
# one-hour hold only bridges scrape gaps. The worker runs only
|
||||
# on backends with the schedule_deletion capability, so on the
|
||||
# Static backend the series never exists and this alert cannot
|
||||
# fire — that backend cannot rotate either; see the rotation
|
||||
# driver matrix in docs/operations/kms-backend-security.md.
|
||||
# Threshold: 400 days — conservative default sitting above a
|
||||
# one-year rotation policy. Align it with the rotation period
|
||||
# your compliance policy requires, and with
|
||||
# RUSTFS_KMS_ROTATION_MAX_AGE_SECS so the per-key rotation_due
|
||||
# verdict and this aggregate alert agree.
|
||||
# ------------------------------------------------------------------
|
||||
- alert: KmsKeyRotationOverdue
|
||||
expr: |
|
||||
rustfs_kms_oldest_key_rotation_age_seconds > (400 * 86400)
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
component: kms
|
||||
annotations:
|
||||
summary: "Oldest KMS key unrotated for more than 400 days"
|
||||
description: >-
|
||||
The least recently rotated usable KMS key was last rotated
|
||||
{{ $value | humanizeDuration }} ago (measured from creation
|
||||
for keys with no recorded rotation). List keys through the
|
||||
admin API and read rotation_due / rotation_due_reason for
|
||||
the per-key verdict; an "unsupported" reason means the
|
||||
backend cannot rotate at all.
|
||||
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmskeyrotationoverdue"
|
||||
|
||||
@@ -55,3 +55,142 @@ jobs:
|
||||
|
||||
- name: Build RustFS
|
||||
run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins
|
||||
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
# Vault Transit backends to every for_each_backend spec in
|
||||
# crates/kms/tests/behavior_*.rs (see crates/kms/AGENTS.md). rotate and
|
||||
# versioning are advertised only by the Vault backends, so without this lane
|
||||
# no CI run ever asserts the working half of behavior_rotation.rs — a
|
||||
# rotation that silently dropped historical key versions would stay green.
|
||||
# The same lane runs the dev-Vault #[ignore] tests and the two self-hosting
|
||||
# live scripts (AppRole login, three-node Raft leader failover).
|
||||
#
|
||||
# GitHub-hosted ubuntu-latest, deliberately not the self-hosted sm-standard
|
||||
# fleet: the HA failover script needs a working Docker daemon, and the
|
||||
# self-hosted fleet is heterogeneous — a docker-dependent workflow has been
|
||||
# burned by it before (see the banner in e2e-s3tests.yml, rustfs/backlog#1149).
|
||||
kms-vault-lane:
|
||||
name: KMS live Vault lane
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
# Root token of the ephemeral loopback dev server. Not a secret: the
|
||||
# server lives only for this job, listens on 127.0.0.1, and holds only
|
||||
# keys the tests create. The literal value matters — the dev-Vault
|
||||
# #[ignore] fixtures in crates/kms/src/backends/vault.rs hardcode it.
|
||||
VAULT_LANE_TOKEN: dev-only-token
|
||||
VAULT_LANE_ADDR: http://127.0.0.1:8200
|
||||
# Keeps a runner-level proxy from swallowing the loopback dev-server
|
||||
# traffic (see crates/kms/AGENTS.md). Actions env keys are
|
||||
# case-insensitive, so only the uppercase form is set; reqwest reads
|
||||
# either casing.
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
steps:
|
||||
- name: Checkout main branch
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: main
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
# Dedicated key: rust-cache cannot tell runner images apart, so
|
||||
# sharing a key with an sm-standard lane would let two different
|
||||
# system images overwrite each other's artifacts (same reasoning as
|
||||
# ci.yml's ci-uring lane). Saved from this nightly job itself so the
|
||||
# next night starts warm.
|
||||
cache-shared-key: kms-vault-lane
|
||||
cache-save-if: 'true'
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Install Vault CLI
|
||||
run: |
|
||||
set -euo pipefail
|
||||
wget -qO- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list >/dev/null
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq vault
|
||||
vault version
|
||||
|
||||
- name: Start Vault dev server with KV2 and Transit engines
|
||||
run: |
|
||||
set -euo pipefail
|
||||
nohup vault server -dev \
|
||||
-dev-root-token-id="${VAULT_LANE_TOKEN}" \
|
||||
-dev-listen-address=127.0.0.1:8200 >/tmp/vault-dev.log 2>&1 &
|
||||
for _ in $(seq 1 60); do
|
||||
if curl -fsS "${VAULT_LANE_ADDR}/v1/sys/health" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
curl -fsS "${VAULT_LANE_ADDR}/v1/sys/health"
|
||||
export VAULT_ADDR="${VAULT_LANE_ADDR}" VAULT_TOKEN="${VAULT_LANE_TOKEN}"
|
||||
# Dev mode mounts KV v2 at secret/ by default; Transit is explicit.
|
||||
# Prove both engines actually work rather than assuming the defaults.
|
||||
vault secrets enable transit
|
||||
vault kv put secret/rustfs-ci-lane-probe value=ok >/dev/null
|
||||
vault kv get secret/rustfs-ci-lane-probe >/dev/null
|
||||
vault write -f transit/keys/rustfs-ci-lane-probe >/dev/null
|
||||
|
||||
- name: Run rustfs-kms suite with the Vault lane on
|
||||
env:
|
||||
RUSTFS_KMS_VAULT_TOKEN: ${{ env.VAULT_LANE_TOKEN }}
|
||||
RUSTFS_KMS_VAULT_ADDR: ${{ env.VAULT_LANE_ADDR }}
|
||||
run: cargo test -p rustfs-kms --locked
|
||||
|
||||
- name: Run dev-Vault ignored tests
|
||||
env:
|
||||
RUSTFS_KMS_VAULT_TOKEN: ${{ env.VAULT_LANE_TOKEN }}
|
||||
RUSTFS_KMS_VAULT_ADDR: ${{ env.VAULT_LANE_ADDR }}
|
||||
# Filters select the dev-Vault-only #[ignore] tests. The AWS #[ignore]
|
||||
# tests (backends::aws, service_manager) stay excluded — they need real
|
||||
# AWS credentials and create billable keys. The AppRole and HA #[ignore]
|
||||
# tests are excluded here because their own scripts below provision the
|
||||
# Vault topology they need.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cargo test -p rustfs-kms --locked --lib backends::contract_tests -- --ignored
|
||||
cargo test -p rustfs-kms --locked --lib backends::vault -- --ignored
|
||||
cargo test -p rustfs-kms --locked --test vault_fault_injection -- --ignored
|
||||
|
||||
- name: Run AppRole live checks (self-hosting ephemeral Vault)
|
||||
run: bash scripts/test/vault_approle_kms_live.sh
|
||||
|
||||
- name: Show Vault dev server log on failure
|
||||
if: failure()
|
||||
run: tail -n 200 /tmp/vault-dev.log || true
|
||||
|
||||
# Three-node Raft leader failover (crates/kms/tests/vault_ha_failover_live.rs,
|
||||
# first validated by rustfs/rustfs#5653). Its own job so an election-timing
|
||||
# flake cannot mask the main lane's verdict, and vice versa. The script
|
||||
# provisions and tears down its own Docker cluster.
|
||||
kms-vault-ha-failover:
|
||||
name: KMS Vault HA failover lane
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
steps:
|
||||
- name: Checkout main branch
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: main
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
cache-shared-key: kms-vault-lane
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Run HA leader failover live checks (three-node Raft cluster in Docker)
|
||||
run: bash scripts/test/vault_ha_kms_live.sh
|
||||
|
||||
@@ -322,6 +322,17 @@ jobs:
|
||||
sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential
|
||||
sudo gem install fpm
|
||||
|
||||
# Create config file for fpm (DEB build creates it in its package dir structure,
|
||||
# but fpm needs the file to exist before packaging)
|
||||
mkdir -p ./tmp-pkg/etc/default
|
||||
cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://rustfs.com/docs/ for more information
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ROOT_USER=""
|
||||
# RUSTFS_ROOT_PASSWORD=""
|
||||
ENVEOF
|
||||
|
||||
fpm -s dir -t rpm \
|
||||
--name rustfs \
|
||||
--version "$VERSION" \
|
||||
@@ -362,6 +373,7 @@ jobs:
|
||||
) \
|
||||
--config-files /etc/default/rustfs \
|
||||
./bin/rustfs=/usr/bin/rustfs \
|
||||
./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
|
||||
deploy/build/rustfs.service=/lib/systemd/system/rustfs.service \
|
||||
LICENSE=/usr/share/doc/rustfs/LICENSE \
|
||||
README.md=/usr/share/doc/rustfs/README.md
|
||||
|
||||
+64
-20
@@ -1,6 +1,6 @@
|
||||
# ARCHITECTURE.md
|
||||
|
||||
> Last updated: 2026-07-02 · Revision: 2
|
||||
> Last updated: 2026-08-12 · Revision: 3
|
||||
>
|
||||
> This document describes the high-level architecture of RustFS.
|
||||
> If you want to familiarize yourself with the code base, you are in the right place!
|
||||
@@ -119,19 +119,44 @@ module split is tracked under `docs/architecture/`.
|
||||
|
||||
3. **Each type has exactly one definition.** Types shared across crates must be defined
|
||||
in one crate and re-exported or imported by others.
|
||||
- ⚠️ VIOLATED: `ReplicationStats` (4 copies), `LastMinuteLatency` (3 copies),
|
||||
`BackpressureConfig` (3 copies), `DataUsageInfo` (2 copies).
|
||||
- ⚠️ VIOLATED: `ReplicationStats` names three unrelated types
|
||||
(`crates/data-usage/src/data_usage.rs`,
|
||||
`crates/obs/src/metrics/collectors/replication.rs`,
|
||||
`crates/ecstore/src/bucket/replication/replication_state.rs`) — a naming
|
||||
collision, not copies; renaming is tracked in rustfs/backlog#1847.
|
||||
- `LastMinuteLatency` has two deliberately different implementations: the
|
||||
per-second bucketed accumulator in `crates/common/src/last_minute.rs` and
|
||||
the in-memory endpoint-health sample tracker in
|
||||
`crates/ecstore/src/bucket/bucket_target_sys.rs` (its doc comment explains
|
||||
why it stays local).
|
||||
- ✅ RESOLVED: `BackpressureConfig` and `DataUsageInfo` each have exactly one
|
||||
definition (`crates/io-core/src/backpressure.rs`,
|
||||
`crates/data-usage/src/data_usage.rs`). A zero-consumer
|
||||
`BackpressureSettings` copy lingers in `crates/io-metrics/src/config.rs`;
|
||||
its removal is tracked in rustfs/backlog#1833.
|
||||
|
||||
4. **ecstore does not know about HTTP or S3 protocol details.** It operates on
|
||||
storage-level abstractions (objects, buckets, disks, pools).
|
||||
- ⚠️ VIOLATED: 58 files under `crates/ecstore/src` reference `s3s`
|
||||
(`rg -l 's3s' crates/ecstore/src | wc -l`), `crates/ecstore/src/client/`
|
||||
is a ~9.4K-line embedded S3 HTTP client, and `crates/ecstore/Cargo.toml`
|
||||
depends on `s3s`, `http`, `hyper`/`hyper-util`/`hyper-rustls`, and
|
||||
`reqwest`. Target state: the engine's need to act as an S3 client
|
||||
(tiering, replication targets) is served by an extracted client crate,
|
||||
and ecstore holds no wire or DTO types.
|
||||
|
||||
5. **The `rustfs` binary crate is the only place that wires everything together.**
|
||||
Individual crates should be testable in isolation.
|
||||
|
||||
6. **Error types use `thiserror` with descriptive names** (e.g., `StorageError`,
|
||||
not bare `Error`).
|
||||
- ⚠️ VIOLATED: 6 crates use `pub enum Error`; 2 crates use `snafu`;
|
||||
`heal` use `anyhow` in library code.
|
||||
- ✅ RESOLVED (strategy): `snafu` is gone from source
|
||||
(`rg -l snafu crates/ rustfs/` is empty) and library code no longer uses
|
||||
`anyhow` (remaining hits are test code and the `e2e_test` crate; `heal`
|
||||
uses `thiserror`).
|
||||
- ⚠️ VIOLATED (naming): 6 crates still export a bare `pub enum Error`:
|
||||
`crypto`, `filemeta`, `heal`, `iam`, `policy`, and `replication`
|
||||
(`src/resync.rs`) — all `thiserror`-derived.
|
||||
|
||||
## Known Structural Issues
|
||||
|
||||
@@ -140,13 +165,25 @@ module split is tracked under `docs/architecture/`.
|
||||
|
||||
### Critical
|
||||
|
||||
- **common/scanner code duplication (~3K lines).** `scanner` depends on `common`
|
||||
but maintains its own copies of `DataUsageInfo`, `LastMinuteLatency`, and related
|
||||
types instead of importing them.
|
||||
- **scanner/data-usage duplicate `.usage-cache.bin` serialization types.** The
|
||||
original finding ("common/scanner code duplication, ~3K lines") is resolved:
|
||||
`scanner` imports the shared data-usage types from `rustfs-data-usage` (see
|
||||
the `pub use rustfs_data_usage::…` re-exports at the top of
|
||||
`crates/scanner/src/data_usage_define.rs`). What remains: `scanner` and
|
||||
`data-usage` each hold their own serialization types for the scanner cache
|
||||
file (`DataUsageCacheInfo`/`DataUsageEntryInfo` in
|
||||
`crates/scanner/src/data_usage_define.rs` vs
|
||||
`DataUsageCacheInfo`/`DataUsageEntry` in
|
||||
`crates/data-usage/src/data_usage.rs`); convergence is tracked in
|
||||
rustfs/backlog#1828.
|
||||
|
||||
- **ecstore is a monolith (87K lines, 163 files).** It contains disk management,
|
||||
bucket management, erasure coding, replication, lifecycle, RPC, and configuration
|
||||
— all in one crate. It should be decomposed along its existing subdirectories.
|
||||
- **ecstore is a monolith (265 files, ~288K lines — roughly half is inline
|
||||
`#[cfg(test)]` code).** Measured with
|
||||
`find crates/ecstore/src -name '*.rs' | xargs wc -l`. It contains disk
|
||||
management, bucket management, erasure coding, replication, lifecycle, RPC,
|
||||
and configuration — all in one crate. It should be decomposed along its
|
||||
existing subdirectories; the split plan lives in
|
||||
[docs/architecture/ecstore-module-split-plan.md](docs/architecture/ecstore-module-split-plan.md).
|
||||
|
||||
### High
|
||||
|
||||
@@ -154,19 +191,26 @@ module split is tracked under `docs/architecture/`.
|
||||
`common → filemeta/madmin` edges must stay removed so leaf/helper crates do
|
||||
not regain upward dependencies.
|
||||
|
||||
- **Three-layer BackpressureConfig/DeadlockConfig duplication** across io-core,
|
||||
concurrency, and `rustfs/src/storage`. Storage policies now expose and consume
|
||||
explicit projections into the concurrency/io-core policy shapes, and workload
|
||||
- **Three-layer backpressure/deadlock policy bridging** across io-core,
|
||||
concurrency, and `rustfs/src/storage`. The config types are no longer
|
||||
duplicated (`BackpressureConfig` and `DeadlockDetectorConfig` are each
|
||||
defined once, in io-core). Storage policies expose and consume explicit
|
||||
projections into the concurrency/io-core policy shapes, and workload
|
||||
admission snapshots are composed through provider registries; later work
|
||||
should use those bridges before deleting compatibility wrappers.
|
||||
|
||||
### Medium
|
||||
|
||||
- **Inconsistent error handling.** Three strategies (thiserror/snafu/anyhow) and
|
||||
mixed naming (bare `Error` vs descriptive names).
|
||||
- **Bare `Error` naming.** Error-handling strategy has converged on `thiserror`
|
||||
(no `snafu`, no `anyhow` in library code); the remaining inconsistency is the
|
||||
bare `pub enum Error` naming in the 6 crates listed under Invariant 6.
|
||||
|
||||
- **Ambiguous common vs utils boundary.** Both described as "utilities and data
|
||||
structures." Need clear ownership rules.
|
||||
- **`common` is mostly parked domain code, not shared utilities.** Of its
|
||||
6,724 lines, ~83% is scanner/heal domain code stranded there to break
|
||||
dependency cycles (`metrics.rs`, ~4,810 lines of scanner-domain metrics;
|
||||
`heal_channel.rs`, ~776 lines of heal-domain channel types). The
|
||||
"common vs utils" naming ambiguity is secondary to moving that code to its
|
||||
domain owners.
|
||||
|
||||
## Cross-Cutting Concerns
|
||||
|
||||
@@ -232,7 +276,7 @@ The binary (`main.rs`) boots in this order:
|
||||
|
||||
```
|
||||
┌─────────┐
|
||||
│ rustfs │ (binary + lib, 75K lines)
|
||||
│ rustfs │ (binary + lib)
|
||||
│ main │
|
||||
└────┬────┘
|
||||
│
|
||||
@@ -255,7 +299,7 @@ The binary (`main.rs`) boots in this order:
|
||||
│ │ │
|
||||
┌─────▼──────┐ ┌──────▼──────┐ ┌──────▼──────┐
|
||||
│ ecstore │ │ rio │ │ io-core │
|
||||
│ (87K,core) │ │ (readers) │ │ (zero-copy) │
|
||||
│ (core) │ │ (readers) │ │ (zero-copy) │
|
||||
└─────┬──────┘ └─────────────┘ └─────────────┘
|
||||
│
|
||||
┌─────┬──┼──┬─────┬──────┐
|
||||
|
||||
Generated
+154
-101
@@ -104,6 +104,12 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aliasable"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd"
|
||||
|
||||
[[package]]
|
||||
name = "aligned-vec"
|
||||
version = "0.6.4"
|
||||
@@ -266,24 +272,24 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||
|
||||
[[package]]
|
||||
name = "apache-avro"
|
||||
version = "0.21.0"
|
||||
version = "0.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf"
|
||||
checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c"
|
||||
dependencies = [
|
||||
"bigdecimal",
|
||||
"bon",
|
||||
"digest 0.10.7",
|
||||
"digest 0.11.3",
|
||||
"log",
|
||||
"miniz_oxide",
|
||||
"miniz_oxide 0.9.1",
|
||||
"num-bigint 0.4.8",
|
||||
"ouroboros",
|
||||
"quad-rand",
|
||||
"rand 0.9.5",
|
||||
"rand 0.10.2",
|
||||
"regex-lite",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"serde_json",
|
||||
"strum 0.27.2",
|
||||
"strum_macros 0.27.2",
|
||||
"strum",
|
||||
"thiserror 2.0.20",
|
||||
"uuid",
|
||||
]
|
||||
@@ -1458,7 +1464,7 @@ dependencies = [
|
||||
"addr2line",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"miniz_oxide",
|
||||
"miniz_oxide 0.8.9",
|
||||
"object 0.37.3",
|
||||
"rustc-demangle",
|
||||
"windows-link",
|
||||
@@ -1801,6 +1807,15 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
|
||||
|
||||
[[package]]
|
||||
name = "castaway"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a"
|
||||
dependencies = [
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cbc"
|
||||
version = "0.1.2"
|
||||
@@ -1994,7 +2009,7 @@ version = "4.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"heck 0.5.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
@@ -2063,6 +2078,19 @@ dependencies = [
|
||||
"unicode-width 0.2.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "compact_str"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416"
|
||||
dependencies = [
|
||||
"castaway",
|
||||
"cfg-if",
|
||||
"itoa",
|
||||
"static_assertions",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "compression-codecs"
|
||||
version = "0.4.38"
|
||||
@@ -4162,7 +4190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
"miniz_oxide 0.8.9",
|
||||
"zlib-rs",
|
||||
]
|
||||
|
||||
@@ -4226,9 +4254,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218"
|
||||
checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
@@ -4241,9 +4269,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae"
|
||||
checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
@@ -4251,15 +4279,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7"
|
||||
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458"
|
||||
checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
@@ -4268,9 +4296,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a"
|
||||
checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed"
|
||||
|
||||
[[package]]
|
||||
name = "futures-lite"
|
||||
@@ -4287,13 +4315,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b"
|
||||
checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
"syn 3.0.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4309,21 +4337,21 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307"
|
||||
checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109"
|
||||
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.33"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa"
|
||||
checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
@@ -4832,6 +4860,12 @@ dependencies = [
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
@@ -4991,9 +5025,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath"
|
||||
version = "0.23.1"
|
||||
version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be80823867e0c9820c9237c38b21f9f4aa1ebb0db1f98ff25ac0b1d2c088a470"
|
||||
checksum = "62e810bedda5a467ef5c9b5c8a20763fefebc89b63ef36f7ee44a143085204a2"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -5025,9 +5059,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros"
|
||||
version = "0.23.1"
|
||||
version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61d1fb3ee80ae7b4743d29487665766ce5a1442e959521790e86317f89dcd5a3"
|
||||
checksum = "01bdc59bfc1a9984bee2ff5da63b2f6fccbaa57cd9a4119d709524632bddf341"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5036,15 +5070,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros-meta"
|
||||
version = "0.23.1"
|
||||
version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "feede71fa226b0b5d523e58e7b0a1462935c0b8a00584a6669f45d564086209d"
|
||||
checksum = "d9216e8a01abe1e1671c376dc8736fb1bf772d7a889538d25f9e1200120ced38"
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-meta"
|
||||
version = "0.23.1"
|
||||
version = "0.23.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "424fe0a13105d3731f65237785f5b95c3e4b8bfae4a039d932f56192cd74afc0"
|
||||
checksum = "f22a9d20435fb79511b19dae37b3607224cd98f342a410702d84657cc38fc72f"
|
||||
dependencies = [
|
||||
"hotpath-macros-meta",
|
||||
]
|
||||
@@ -5420,9 +5454,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "io-uring"
|
||||
version = "0.7.13"
|
||||
version = "0.7.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0"
|
||||
checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"cfg-if",
|
||||
@@ -5896,18 +5930,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "liblzma"
|
||||
version = "0.4.7"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "45aec2360b3933207e27908049d8e4df4e476b58180afb1e56b2a4fb72efe4ba"
|
||||
checksum = "2fe0a34ca854fd4f20c07f696fc8675aec78f87d88d29f5e10257a7490a1b2e1"
|
||||
dependencies = [
|
||||
"liblzma-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "liblzma-sys"
|
||||
version = "0.4.7"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a046c7f353ba30f810545151e04f63545833803f5b86ee3ddf1517247fe560a5"
|
||||
checksum = "a0dad045e4b1b7b170be4b60b54b780cafb4490165461bac7d1cf7b703f61d5f"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
@@ -6369,6 +6403,15 @@ dependencies = [
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "minlz"
|
||||
version = "1.2.3"
|
||||
@@ -6416,9 +6459,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "moka"
|
||||
version = "0.12.15"
|
||||
version = "0.12.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046"
|
||||
checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9"
|
||||
dependencies = [
|
||||
"async-lock",
|
||||
"crossbeam-channel",
|
||||
@@ -6463,7 +6506,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4db8a44120571277accfaa3f3d91e7d3989d601d817c2fc01a9391b86135666"
|
||||
dependencies = [
|
||||
"darling 0.23.0",
|
||||
"heck",
|
||||
"heck 0.5.0",
|
||||
"manyhow",
|
||||
"num-bigint 0.4.8",
|
||||
"proc-macro-crate",
|
||||
@@ -6747,9 +6790,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.46"
|
||||
version = "0.1.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
|
||||
checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
@@ -7168,6 +7211,30 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ouroboros"
|
||||
version = "0.18.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59"
|
||||
dependencies = [
|
||||
"aliasable",
|
||||
"ouroboros_macro",
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ouroboros_macro"
|
||||
version = "0.18.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0"
|
||||
dependencies = [
|
||||
"heck 0.4.1",
|
||||
"proc-macro2",
|
||||
"proc-macro2-diagnostics",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "outref"
|
||||
version = "0.5.2"
|
||||
@@ -7720,9 +7787,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.14.0"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3"
|
||||
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic-util"
|
||||
@@ -7903,6 +7970,19 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2-diagnostics"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
"version_check",
|
||||
"yansi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prometheus"
|
||||
version = "0.14.0"
|
||||
@@ -7962,7 +8042,7 @@ version = "0.13.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"heck 0.5.0",
|
||||
"itertools 0.14.0",
|
||||
"log",
|
||||
"multimap",
|
||||
@@ -7982,7 +8062,7 @@ version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"heck 0.5.0",
|
||||
"itertools 0.14.0",
|
||||
"log",
|
||||
"multimap",
|
||||
@@ -8074,9 +8154,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pulldown-cmark-to-cmark"
|
||||
version = "22.0.0"
|
||||
version = "22.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90"
|
||||
checksum = "ab1ad36992cead65f02aa399a373a42730922f1525d988172634fdefdecb8a60"
|
||||
dependencies = [
|
||||
"pulldown-cmark",
|
||||
]
|
||||
@@ -8430,9 +8510,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rcgen"
|
||||
version = "0.14.8"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055"
|
||||
checksum = "091e7a8e7d86e6feb87a27ce8e2cba29d49eff9507afeebefab7eeb2ca667fb4"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"pem",
|
||||
@@ -8837,9 +8917,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "russh"
|
||||
version = "0.62.5"
|
||||
version = "0.62.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da7c230e0ed9cbeb92fbad6c8848985d6df2a1464c0dc247a021abd666e9005e"
|
||||
checksum = "b41043523e0edcbd4e31d00903e26f12994f63b21bae9904f7405c1ed92752a5"
|
||||
dependencies = [
|
||||
"aes 0.9.2",
|
||||
"aws-lc-rs",
|
||||
@@ -9265,7 +9345,6 @@ dependencies = [
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.0-rc.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
"rmp-serde",
|
||||
"rustfs-filemeta",
|
||||
@@ -9457,7 +9536,6 @@ dependencies = [
|
||||
"futures",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"libc",
|
||||
"metrics",
|
||||
"rustfs-common",
|
||||
"rustfs-concurrency",
|
||||
@@ -9494,6 +9572,7 @@ dependencies = [
|
||||
"moka",
|
||||
"openidconnect",
|
||||
"pollster",
|
||||
"rcgen",
|
||||
"reqwest",
|
||||
"rustfs-config",
|
||||
"rustfs-credentials",
|
||||
@@ -9505,6 +9584,8 @@ dependencies = [
|
||||
"rustfs-storage-api",
|
||||
"rustfs-test-utils",
|
||||
"rustfs-utils",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
@@ -9700,6 +9781,7 @@ name = "rustfs-lock"
|
||||
version = "1.0.0-rc.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"compact_str",
|
||||
"crossbeam-queue",
|
||||
"futures",
|
||||
"hotpath",
|
||||
@@ -9710,7 +9792,6 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
"smartstring",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tonic",
|
||||
@@ -9900,7 +9981,7 @@ dependencies = [
|
||||
"rustfs-crypto",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum 0.28.0",
|
||||
"strum",
|
||||
"temp-env",
|
||||
"test-case",
|
||||
"thiserror 2.0.20",
|
||||
@@ -10544,9 +10625,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.13"
|
||||
version = "0.103.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
|
||||
checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
@@ -10926,9 +11007,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_with"
|
||||
version = "3.21.0"
|
||||
version = "3.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
|
||||
checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bs58",
|
||||
@@ -10936,6 +11017,7 @@ dependencies = [
|
||||
"hex",
|
||||
"indexmap 1.9.3",
|
||||
"indexmap 2.14.0",
|
||||
"jiff",
|
||||
"schemars 0.9.0",
|
||||
"schemars 1.2.2",
|
||||
"serde_core",
|
||||
@@ -10946,9 +11028,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_with_macros"
|
||||
version = "3.21.0"
|
||||
version = "3.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
|
||||
checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46"
|
||||
dependencies = [
|
||||
"darling 0.23.0",
|
||||
"proc-macro2",
|
||||
@@ -11242,17 +11324,6 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smartstring"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"static_assertions",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "snafu"
|
||||
version = "0.6.10"
|
||||
@@ -11499,31 +11570,13 @@ version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "strum"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
|
||||
|
||||
[[package]]
|
||||
name = "strum"
|
||||
version = "0.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd"
|
||||
dependencies = [
|
||||
"strum_macros 0.28.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strum_macros"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
"strum_macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11532,7 +11585,7 @@ version = "0.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"heck 0.5.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -12805,9 +12858,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "whoami"
|
||||
version = "2.1.2"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "998767ef88740d1f5b0682a9c53c24431453923962269c2db68ee43788c5a40d"
|
||||
checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"libredox",
|
||||
|
||||
+9
-9
@@ -142,10 +142,10 @@ async-recursion = "1.1.1"
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.50.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.33"
|
||||
futures-core = "0.3.33"
|
||||
futures = "0.3.34"
|
||||
futures-core = "0.3.34"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.33"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "6.8.0" }
|
||||
lapin = { default-features = false, version = "4.10.0" }
|
||||
@@ -171,7 +171,7 @@ tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.0" }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = "0.21.0"
|
||||
apache-avro = "0.22.0"
|
||||
bytes = { version = "1.12.1" }
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
@@ -268,7 +268,7 @@ lz4 = "1.28.1"
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
mime_guess = "2.0.5"
|
||||
moka = { version = "0.12.15" }
|
||||
moka = { version = "0.12.16" }
|
||||
netif = "0.1.6"
|
||||
num_cpus = { version = "1.17.0" }
|
||||
nvml-wrapper = "0.12.1"
|
||||
@@ -294,7 +294,7 @@ serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
smallvec = { version = "1.15.2" }
|
||||
smartstring = "1.0.1"
|
||||
compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.2.2" }
|
||||
strum = { version = "0.28.0" }
|
||||
@@ -339,8 +339,8 @@ pyroscope = { version = "2.1.1" }
|
||||
libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.1" }
|
||||
rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.5" }
|
||||
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.6" }
|
||||
russh-sftp = "2.4.0"
|
||||
|
||||
# WebDAV
|
||||
@@ -349,7 +349,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7", features = ["extended"] }
|
||||
hotpath = { version = "0.23.1", default-features = false }
|
||||
hotpath = { version = "0.23.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ pub mod heal_channel;
|
||||
pub mod last_minute;
|
||||
pub mod metrics;
|
||||
mod readiness;
|
||||
pub mod table_catalog;
|
||||
|
||||
pub use globals::*;
|
||||
pub use readiness::{GlobalReadiness, SystemStage};
|
||||
|
||||
@@ -915,11 +915,13 @@ const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1;
|
||||
const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
|
||||
const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3;
|
||||
const SCAN_CYCLE_RESULT_SUPERSEDED: u8 = 4;
|
||||
const SCAN_CYCLE_RESULT_DEFERRED: u8 = 5;
|
||||
const SCAN_CYCLE_RESULT_UNKNOWN_LABEL: &str = "unknown";
|
||||
const SCAN_CYCLE_RESULT_SUCCESS_LABEL: &str = "success";
|
||||
const SCAN_CYCLE_RESULT_ERROR_LABEL: &str = "error";
|
||||
const SCAN_CYCLE_RESULT_PARTIAL_LABEL: &str = "partial";
|
||||
const SCAN_CYCLE_RESULT_SUPERSEDED_LABEL: &str = "superseded";
|
||||
const SCAN_CYCLE_RESULT_DEFERRED_LABEL: &str = "deferred";
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum ScanCyclePartialReason {
|
||||
@@ -1424,6 +1426,7 @@ fn scan_cycle_result_label(result: u8) -> &'static str {
|
||||
SCAN_CYCLE_RESULT_ERROR => SCAN_CYCLE_RESULT_ERROR_LABEL,
|
||||
SCAN_CYCLE_RESULT_PARTIAL => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
|
||||
SCAN_CYCLE_RESULT_SUPERSEDED => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL,
|
||||
SCAN_CYCLE_RESULT_DEFERRED => SCAN_CYCLE_RESULT_DEFERRED_LABEL,
|
||||
_ => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
|
||||
}
|
||||
}
|
||||
@@ -1752,6 +1755,11 @@ pub fn emit_scan_cycle_superseded(duration: Duration) {
|
||||
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL).increment(1);
|
||||
}
|
||||
|
||||
pub fn emit_scan_cycle_deferred(duration: Duration) {
|
||||
global_metrics().record_scan_cycle_deferred(duration);
|
||||
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_DEFERRED_LABEL).increment(1);
|
||||
}
|
||||
|
||||
pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
|
||||
let result = if success { "success" } else { "error" };
|
||||
global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
|
||||
@@ -2549,6 +2557,17 @@ impl Metrics {
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_deferred(&self, duration: Duration) {
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.last_scan_cycle_result
|
||||
.store(SCAN_CYCLE_RESULT_DEFERRED, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason
|
||||
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) {
|
||||
self.record_scan_cycle_partial_with_source(duration, reason, None);
|
||||
}
|
||||
@@ -4264,6 +4283,21 @@ mod tests {
|
||||
assert_eq!(report.partial_cycles, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_tracks_deferred_cycle_without_failed_increment() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scan_cycle_deferred(Duration::from_millis(250));
|
||||
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_DEFERRED_LABEL);
|
||||
assert_eq!(report.last_cycle_result_code, u64::from(SCAN_CYCLE_RESULT_DEFERRED));
|
||||
assert_eq!(report.last_cycle_duration_seconds, 0.25);
|
||||
assert_eq!(report.failed_cycles, 0);
|
||||
assert_eq!(report.superseded_cycles, 0);
|
||||
assert_eq!(report.partial_cycles, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_tracks_successful_scan_cycle_without_failed_increment() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
/// Cross-crate lock identity used to fence table-bucket publication against
|
||||
/// object mutations that bypass the S3 request authorization layer.
|
||||
pub const TABLE_BUCKET_PUBLICATION_LOCK_PATH: &str = ".rustfs-table/warehouses/default/publication.lock";
|
||||
@@ -97,6 +97,14 @@ Current guidance:
|
||||
- enables minimal payload mode for GET health responses (`status`, `ready` only).
|
||||
- `RUSTFS_HEALTH_READINESS_CACHE_TTL_MS`
|
||||
- TTL for readiness cache evaluation.
|
||||
- `RUSTFS_HEALTH_OBJECT_PROGRESS_ENABLE`
|
||||
- withdraws readiness when bounded object read/write stages stop completing while requests remain active.
|
||||
- default is `true`.
|
||||
- `RUSTFS_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS`
|
||||
- maximum time without completion in a bounded object stage before readiness is withdrawn.
|
||||
- default is `30000`; `0` uses the default.
|
||||
- the effective value is at least 5 seconds longer than `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT`.
|
||||
- this readiness SLO is independent of disk read/write failure deadlines and may withdraw traffic before those deadlines expire.
|
||||
- `RUSTFS_HEALTH_COMPAT_BUSY_CHECK_ENABLE`
|
||||
- enables busy protection behavior for health probes.
|
||||
- default is `false`.
|
||||
|
||||
@@ -22,6 +22,19 @@ pub const DEFAULT_HEALTH_ENDPOINT_ENABLE: bool = true;
|
||||
pub const ENV_HEALTH_READINESS_CACHE_TTL_MS: &str = "RUSTFS_HEALTH_READINESS_CACHE_TTL_MS";
|
||||
pub const DEFAULT_HEALTH_READINESS_CACHE_TTL_MS: u64 = 1000;
|
||||
|
||||
/// Enable readiness withdrawal when bounded object read/write stages stop
|
||||
/// completing while requests remain active.
|
||||
pub const ENV_HEALTH_OBJECT_PROGRESS_ENABLE: &str = "RUSTFS_HEALTH_OBJECT_PROGRESS_ENABLE";
|
||||
pub const DEFAULT_HEALTH_OBJECT_PROGRESS_ENABLE: bool = true;
|
||||
|
||||
/// Requested time without completion in a bounded object stage before local
|
||||
/// readiness is withdrawn (milliseconds). A value of `0` uses the default;
|
||||
/// runtime adds a safety floor based on the object-lock acquisition timeout.
|
||||
pub const ENV_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS: &str = "RUSTFS_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS";
|
||||
pub const DEFAULT_HEALTH_OBJECT_PROGRESS_TIMEOUT_MS: u64 = 30_000;
|
||||
/// Additional time beyond the configured object-lock acquisition deadline.
|
||||
pub const HEALTH_OBJECT_PROGRESS_LOCK_MARGIN_MS: u64 = 5_000;
|
||||
|
||||
/// Timeout for cluster health readiness collectors (milliseconds).
|
||||
/// This bounds expensive storage and lock quorum checks used by cluster probes.
|
||||
pub const ENV_HEALTH_CLUSTER_TIMEOUT_MS: &str = "RUSTFS_HEALTH_CLUSTER_TIMEOUT_MS";
|
||||
|
||||
@@ -81,6 +81,8 @@ pub const ENV_TEST_IAM_FAIL_INIT_ATTEMPTS: &str = "RUSTFS_TEST_IAM_FAIL_INIT_ATT
|
||||
pub const ENV_TEST_IAM_RETRY_INTERVAL_MS: &str = "RUSTFS_TEST_IAM_RETRY_INTERVAL_MS";
|
||||
/// Runtime env var controlling the transition worker count.
|
||||
pub const ENV_TRANSITION_WORKERS: &str = "RUSTFS_MAX_TRANSITION_WORKERS";
|
||||
/// Runtime env var controlling the expiry worker count.
|
||||
pub const ENV_MAX_EXPIRY_WORKERS: &str = "RUSTFS_MAX_EXPIRY_WORKERS";
|
||||
/// Runtime env var controlling the absolute maximum transition workers.
|
||||
pub const ENV_TRANSITION_WORKERS_ABSOLUTE_MAX: &str = "RUSTFS_ABSOLUTE_MAX_WORKERS";
|
||||
/// Runtime env var controlling the transition queue capacity.
|
||||
|
||||
@@ -36,6 +36,11 @@ pub const ENV_TRUST_SYSTEM_CA: &str = "RUSTFS_TRUST_SYSTEM_CA";
|
||||
/// To change this behavior, set the environment variable RUSTFS_TRUST_SYSTEM_CA=1
|
||||
pub const DEFAULT_TRUST_SYSTEM_CA: bool = false;
|
||||
|
||||
/// Environment variable for an extra outbound root CA certificate bundle.
|
||||
/// Use this to trust an internal CA for outbound HTTPS clients without replacing
|
||||
/// the default operating-system/web PKI roots via SSL_CERT_FILE.
|
||||
pub const ENV_RUSTFS_EXTRA_CA_CERT: &str = "RUSTFS_EXTRA_CA_CERT";
|
||||
|
||||
/// Environment variable to trust leaf certificates as CA
|
||||
/// When set to "1", RustFS will treat leaf certificates as CA certificates for trust validation.
|
||||
/// By default, this is disabled.
|
||||
|
||||
@@ -37,7 +37,6 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-filemeta/hotpath-cpu"]
|
||||
hotpath.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
rmp-serde = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
|
||||
[lib]
|
||||
|
||||
@@ -846,8 +846,15 @@ impl DataUsageEntry {
|
||||
}
|
||||
}
|
||||
|
||||
/// Data usage cache info
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
/// Read-only projection of the scanner's `.usage-cache.bin` info block.
|
||||
///
|
||||
/// The canonical wire format is written by the hand-written map-encoded
|
||||
/// `Serialize` on the scanner-side `DataUsageCacheInfo`
|
||||
/// (`crates/scanner/src/data_usage_define.rs`), which carries 16 fields.
|
||||
/// This type decodes only the shared subset and is deliberately not
|
||||
/// `Serialize`: a derived (array) encoding of this 6-field subset would
|
||||
/// corrupt the cache for scanner readers, so no write path may exist here.
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
pub struct DataUsageCacheInfo {
|
||||
pub name: String,
|
||||
pub next_cycle: u64,
|
||||
@@ -863,8 +870,12 @@ pub struct DataUsageCacheInfo {
|
||||
pub snapshot_complete: bool,
|
||||
}
|
||||
|
||||
/// Data usage cache
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
/// Read-only projection of a scanner-written `.usage-cache.bin` file.
|
||||
///
|
||||
/// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`)
|
||||
/// owns the persisted format; this type only decodes it (see
|
||||
/// [`DataUsageCacheInfo`]) and must never grow a serialization path.
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
pub struct DataUsageCache {
|
||||
pub info: DataUsageCacheInfo,
|
||||
pub cache: HashMap<String, DataUsageEntry>,
|
||||
@@ -1186,31 +1197,10 @@ impl DataUsageCache {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn marshal_msg(&self) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut buf = Vec::new();
|
||||
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let t: Self = rmp_serde::from_slice(buf)?;
|
||||
Ok(t)
|
||||
}
|
||||
|
||||
// Note: load and save methods are storage-specific and should be implemented
|
||||
// in the ecstore crate where storage access is available
|
||||
}
|
||||
|
||||
/// Trait for storage-specific operations on DataUsageCache
|
||||
#[async_trait::async_trait]
|
||||
pub trait DataUsageCacheStorage {
|
||||
/// Load data usage cache from backend storage
|
||||
async fn load(store: &dyn std::any::Any, name: &str) -> Result<Self, Box<dyn std::error::Error + Send + Sync>>
|
||||
where
|
||||
Self: Sized;
|
||||
|
||||
/// Save data usage cache to backend storage
|
||||
async fn save(&self, name: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
}
|
||||
|
||||
// Helper structs and functions for cache operations
|
||||
@@ -1832,6 +1822,82 @@ mod tests {
|
||||
assert!(decoded.all_tier_stats.is_none());
|
||||
}
|
||||
|
||||
/// Scanner-written `.usage-cache.bin` bytes: a 2-element array of the
|
||||
/// canonical 16-field map-encoded info block and one map-encoded entry.
|
||||
/// Captured from the canonical writer's `marshal_msg` — see
|
||||
/// `usage_cache_wire_format_is_pinned` in
|
||||
/// `crates/scanner/src/data_usage_define.rs`, which pins these exact
|
||||
/// bytes and documents regeneration. Hardcoded here because a
|
||||
/// dev-dependency on rustfs-scanner would pull the whole ecstore tree
|
||||
/// into this crate's test build, and a fixture generated at test runtime
|
||||
/// could not detect writer drift anyway.
|
||||
const SCANNER_USAGE_CACHE_WIRE_FIXTURE: &[u8] = &[
|
||||
0x92, 0xde, 0x00, 0x10, 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
||||
0x74, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x07, 0xac, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72,
|
||||
0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x09, 0xab, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x92,
|
||||
0xce, 0x65, 0x53, 0xf1, 0x00, 0x00, 0xac, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0xc3,
|
||||
0xa9, 0x6c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0xc0, 0xab, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74,
|
||||
0x69, 0x6f, 0x6e, 0xc0, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x81,
|
||||
0xb0, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2f, 0x6c, 0x6f, 0x73, 0x74, 0x0b, 0xb1, 0x73,
|
||||
0x63, 0x61, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72, 0xb2, 0x77, 0x69, 0x72,
|
||||
0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0xaf, 0x73, 0x63, 0x61, 0x6e,
|
||||
0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0xc0, 0xad, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67,
|
||||
0x5f, 0x68, 0x65, 0x61, 0x6c, 0x73, 0x91, 0x9a, 0xa6, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0xab, 0x77, 0x69, 0x72, 0x65,
|
||||
0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0xa6, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x6e, 0xc0, 0x01, 0x64, 0xcc, 0xc8, 0x03,
|
||||
0xa8, 0x64, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0xa6, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0xab, 0x6f, 0x62, 0x6a,
|
||||
0x65, 0x63, 0x74, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0xc0, 0xa6, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x92, 0x01, 0x02, 0xb1,
|
||||
0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0xc3, 0xb0, 0x73,
|
||||
0x63, 0x61, 0x6e, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0xdc, 0x00, 0x20, 0x03, 0x03,
|
||||
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
|
||||
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0xb0, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6b, 0x65, 0x79,
|
||||
0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x01, 0x81, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
||||
0x74, 0x8b, 0xa8, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x90, 0xa4, 0x73, 0x69, 0x7a, 0x65, 0xcd, 0x10, 0x00,
|
||||
0xa7, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x03, 0xa8, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x05, 0xae,
|
||||
0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x01, 0xa9, 0x6f, 0x62, 0x6a, 0x5f,
|
||||
0x73, 0x69, 0x7a, 0x65, 0x73, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x6f, 0x62,
|
||||
0x6a, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x72,
|
||||
0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0xc0, 0xa9, 0x63, 0x6f,
|
||||
0x6d, 0x70, 0x61, 0x63, 0x74, 0x65, 0x64, 0xc3, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65,
|
||||
0x63, 0x74, 0x73, 0x02, 0xae, 0x61, 0x6c, 0x6c, 0x5f, 0x74, 0x69, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x91,
|
||||
0x81, 0xa4, 0x57, 0x41, 0x52, 0x4d, 0x93, 0xcd, 0x08, 0x00, 0x02, 0x01,
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn thin_usage_cache_decodes_scanner_wire_fixture() {
|
||||
let decoded =
|
||||
DataUsageCache::unmarshal(SCANNER_USAGE_CACHE_WIRE_FIXTURE).expect("thin projection decodes a scanner-written cache");
|
||||
|
||||
// The six fields shared with the scanner's 16-field info block; the
|
||||
// remaining ten (lifecycle, replication, checkpoint, heals, ...) must
|
||||
// be skipped, not error.
|
||||
assert_eq!(decoded.info.name, "wire-bucket");
|
||||
assert_eq!(decoded.info.next_cycle, 7);
|
||||
assert_eq!(
|
||||
decoded.info.last_update,
|
||||
Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000))
|
||||
);
|
||||
assert!(decoded.info.skip_healing);
|
||||
assert_eq!(decoded.info.failed_objects.get("wire-bucket/lost"), Some(&11));
|
||||
assert!(decoded.info.snapshot_complete);
|
||||
|
||||
// Entries use the shared canonical map-encoded type end to end.
|
||||
let entry = decoded.cache.get("wire-bucket").expect("fixture entry decodes");
|
||||
assert_eq!(entry.size, 4096);
|
||||
assert_eq!(entry.objects, 3);
|
||||
assert_eq!(entry.versions, 5);
|
||||
assert_eq!(entry.delete_markers, 1);
|
||||
assert!(entry.compacted);
|
||||
assert_eq!(entry.failed_objects, 2);
|
||||
assert_eq!(
|
||||
entry.all_tier_stats.as_ref().and_then(|tiers| tiers.tiers.get("WARM")),
|
||||
Some(&TierStats {
|
||||
total_size: 2048,
|
||||
num_versions: 2,
|
||||
num_objects: 1,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_path_uses_portable_slash_semantics() {
|
||||
for (input, expected) in [
|
||||
|
||||
+102
-18
@@ -40,7 +40,8 @@ use http::header::{CONTENT_TYPE, HOST};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::collections::BTreeSet;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::error::Error;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::info;
|
||||
@@ -59,13 +60,26 @@ pub(crate) struct VersionShardCensus {
|
||||
pub version_id: Option<String>,
|
||||
pub has_xl_meta: bool,
|
||||
pub data_dir: Option<String>,
|
||||
pub erasure_index: Option<usize>,
|
||||
pub expected_part_numbers: BTreeSet<usize>,
|
||||
pub present_part_numbers: BTreeSet<usize>,
|
||||
pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
|
||||
pub inline_data_fingerprint: Option<PartShardFingerprint>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct PartShardFingerprint {
|
||||
pub size: u64,
|
||||
pub sha256: String,
|
||||
}
|
||||
|
||||
impl VersionShardCensus {
|
||||
pub(crate) fn is_complete(&self) -> bool {
|
||||
self.has_xl_meta && self.expected_part_numbers == self.present_part_numbers
|
||||
self.has_xl_meta
|
||||
&& self.expected_part_numbers.len() == self.present_part_fingerprints.len()
|
||||
&& self
|
||||
.expected_part_numbers
|
||||
.iter()
|
||||
.all(|part_number| self.present_part_fingerprints.contains_key(part_number))
|
||||
}
|
||||
|
||||
pub(crate) fn matches_manifest(&self, manifest: &Self) -> bool {
|
||||
@@ -73,10 +87,25 @@ impl VersionShardCensus {
|
||||
&& self.is_complete()
|
||||
&& manifest.is_complete()
|
||||
&& self.data_dir == manifest.data_dir
|
||||
&& self.erasure_index == manifest.erasure_index
|
||||
&& self.expected_part_numbers == manifest.expected_part_numbers
|
||||
&& self.present_part_fingerprints == manifest.present_part_fingerprints
|
||||
&& self.inline_data_fingerprint == manifest.inline_data_fingerprint
|
||||
}
|
||||
}
|
||||
|
||||
fn sha256_hex(data: &[u8]) -> String {
|
||||
let digest = Sha256::digest(data);
|
||||
digest.iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
|
||||
fn shard_fingerprint(data: &[u8]) -> ChaosResult<PartShardFingerprint> {
|
||||
Ok(PartShardFingerprint {
|
||||
size: u64::try_from(data.len())?,
|
||||
sha256: sha256_hex(data),
|
||||
})
|
||||
}
|
||||
|
||||
/// Single-node RustFS server with `disk_count` local volume directories that
|
||||
/// can be faulted individually while the server is running.
|
||||
pub struct DiskFaultHarness {
|
||||
@@ -283,8 +312,10 @@ pub(crate) fn census_object_version_on_disk(
|
||||
version_id,
|
||||
has_xl_meta: false,
|
||||
data_dir: None,
|
||||
erasure_index: None,
|
||||
expected_part_numbers: BTreeSet::new(),
|
||||
present_part_numbers: BTreeSet::new(),
|
||||
present_part_fingerprints: BTreeMap::new(),
|
||||
inline_data_fingerprint: None,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -296,20 +327,31 @@ pub(crate) fn census_object_version_on_disk(
|
||||
file_info.parts.iter().map(|part| part.number).collect()
|
||||
};
|
||||
let data_dir = file_info.data_dir.map(|id| id.to_string());
|
||||
let erasure_index = Some(file_info.erasure.index);
|
||||
let inline_data_fingerprint = file_info.data.as_deref().map(shard_fingerprint).transpose()?;
|
||||
let part_dir = data_dir.as_ref().map_or_else(|| object_dir.clone(), |id| object_dir.join(id));
|
||||
let present_part_numbers = match std::fs::read_dir(&part_dir) {
|
||||
Ok(entries) => entries
|
||||
.filter_map(Result::ok)
|
||||
.filter_map(|entry| {
|
||||
entry
|
||||
.file_type()
|
||||
.ok()
|
||||
.filter(|kind| kind.is_file())
|
||||
.and_then(|_| entry.file_name().to_str().map(str::to_owned))
|
||||
})
|
||||
.filter_map(|name| name.strip_prefix("part.").and_then(|number| number.parse::<usize>().ok()))
|
||||
.collect(),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => BTreeSet::new(),
|
||||
let present_part_fingerprints = match std::fs::read_dir(&part_dir) {
|
||||
Ok(entries) => {
|
||||
let mut fingerprints = BTreeMap::new();
|
||||
for entry in entries {
|
||||
let entry = entry?;
|
||||
if !entry.file_type()?.is_file() {
|
||||
continue;
|
||||
}
|
||||
let file_name = entry.file_name();
|
||||
let Some(part_number) = file_name
|
||||
.to_str()
|
||||
.and_then(|name| name.strip_prefix("part."))
|
||||
.and_then(|number| number.parse::<usize>().ok())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let data = std::fs::read(entry.path())?;
|
||||
fingerprints.insert(part_number, shard_fingerprint(&data)?);
|
||||
}
|
||||
fingerprints
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => BTreeMap::new(),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
|
||||
@@ -317,8 +359,10 @@ pub(crate) fn census_object_version_on_disk(
|
||||
version_id,
|
||||
has_xl_meta: true,
|
||||
data_dir,
|
||||
erasure_index,
|
||||
expected_part_numbers,
|
||||
present_part_numbers,
|
||||
present_part_fingerprints,
|
||||
inline_data_fingerprint,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -358,3 +402,43 @@ pub async fn signed_admin_post(url: &str, body: Option<&str>, access_key: &str,
|
||||
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn complete_census() -> VersionShardCensus {
|
||||
VersionShardCensus {
|
||||
version_id: Some("version".to_string()),
|
||||
has_xl_meta: true,
|
||||
data_dir: Some("data-dir".to_string()),
|
||||
erasure_index: Some(3),
|
||||
expected_part_numbers: BTreeSet::from([1]),
|
||||
present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
|
||||
inline_data_fingerprint: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shard_fingerprint_uses_physical_length_and_sha256() {
|
||||
assert_eq!(
|
||||
shard_fingerprint(b"abc").unwrap(),
|
||||
PartShardFingerprint {
|
||||
size: 3,
|
||||
sha256: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad".to_string(),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_requires_matching_inline_payload() {
|
||||
let mut expected = complete_census();
|
||||
expected.expected_part_numbers.clear();
|
||||
expected.present_part_fingerprints.clear();
|
||||
expected.inline_data_fingerprint = Some(shard_fingerprint(b"expected").unwrap());
|
||||
let mut changed = expected.clone();
|
||||
changed.inline_data_fingerprint = Some(shard_fingerprint(b"changed").unwrap());
|
||||
assert!(expected.matches_manifest(&expected));
|
||||
assert!(!changed.matches_manifest(&expected));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +67,16 @@ fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
fn capture_command_logs(command: &mut Command, log_path: Option<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let Some(log_path) = log_path else {
|
||||
return Ok(());
|
||||
};
|
||||
let file = stdfs::OpenOptions::new().create(true).append(true).open(log_path)?;
|
||||
let stderr_file = file.try_clone()?;
|
||||
command.stdout(Stdio::from(file)).stderr(Stdio::from(stderr_file));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn build_test_s3_config(
|
||||
endpoint_url: &str,
|
||||
access_key: &str,
|
||||
@@ -557,13 +567,7 @@ impl RustFSTestEnvironment {
|
||||
for (key, value) in extra_env {
|
||||
command.env(key, value);
|
||||
}
|
||||
// Optionally capture the child's stdout+stderr to a file so the test can
|
||||
// grep server logs (e.g. to confirm which GET reader path was taken).
|
||||
if let Some(log_path) = &self.capture_log_path {
|
||||
let file = stdfs::OpenOptions::new().create(true).append(true).open(log_path)?;
|
||||
let stderr_file = file.try_clone()?;
|
||||
command.stdout(Stdio::from(file)).stderr(Stdio::from(stderr_file));
|
||||
}
|
||||
capture_command_logs(&mut command, self.capture_log_path.as_deref())?;
|
||||
let process = command.args(&args).spawn()?;
|
||||
|
||||
self.process = Some(process);
|
||||
@@ -1051,6 +1055,7 @@ pub struct RustFSTestClusterEnvironment {
|
||||
pub secret_key: String,
|
||||
pub extra_env: Vec<(String, String)>,
|
||||
pub node_extra_env: Vec<Vec<(String, String)>>,
|
||||
pub node_capture_log_paths: Vec<Option<String>>,
|
||||
pub topology: ClusterTopology,
|
||||
}
|
||||
|
||||
@@ -1150,6 +1155,7 @@ impl RustFSTestClusterEnvironment {
|
||||
secret_key: "rustfs-cluster-test-secret".to_string(),
|
||||
extra_env,
|
||||
node_extra_env: vec![Vec::new(); topology.node_count],
|
||||
node_capture_log_paths: vec![None; topology.node_count],
|
||||
topology,
|
||||
})
|
||||
}
|
||||
@@ -1179,6 +1185,20 @@ impl RustFSTestClusterEnvironment {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Capture stdout+stderr for a single cluster node process.
|
||||
pub fn set_node_capture_log_path<P>(
|
||||
&mut self,
|
||||
node_idx: usize,
|
||||
path: P,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
where
|
||||
P: Into<String>,
|
||||
{
|
||||
self.ensure_node_index(node_idx)?;
|
||||
self.node_capture_log_paths[node_idx] = Some(path.into());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_node_index(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
if node_idx >= self.nodes.len() {
|
||||
return Err(format!("node_idx {node_idx} is invalid").into());
|
||||
@@ -1268,6 +1288,7 @@ impl RustFSTestClusterEnvironment {
|
||||
for (key, value) in &self.node_extra_env[i] {
|
||||
command.env(key, value);
|
||||
}
|
||||
capture_command_logs(&mut command, self.node_capture_log_paths[i].as_deref())?;
|
||||
|
||||
let process = command.current_dir(&node.data_dir).spawn()?;
|
||||
|
||||
@@ -1294,6 +1315,7 @@ impl RustFSTestClusterEnvironment {
|
||||
|
||||
let binary_path = rustfs_binary_path();
|
||||
let volumes_arg = self.build_volumes_arg();
|
||||
let log_path = self.node_capture_log_paths[node_idx].clone();
|
||||
let node = &mut self.nodes[node_idx];
|
||||
info!("Starting cluster node {} on {}", node_idx, node.address);
|
||||
|
||||
@@ -1312,6 +1334,7 @@ impl RustFSTestClusterEnvironment {
|
||||
for (key, value) in &self.node_extra_env[node_idx] {
|
||||
command.env(key, value);
|
||||
}
|
||||
capture_command_logs(&mut command, log_path.as_deref())?;
|
||||
|
||||
let process = command.current_dir(&node.data_dir).spawn()?;
|
||||
node.process = Some(process);
|
||||
@@ -1563,6 +1586,7 @@ mod tests {
|
||||
secret_key: DEFAULT_SECRET_KEY.to_string(),
|
||||
extra_env: Vec::new(),
|
||||
node_extra_env: vec![Vec::new(); topology.node_count],
|
||||
node_capture_log_paths: vec![None; topology.node_count],
|
||||
topology,
|
||||
}
|
||||
}
|
||||
@@ -1658,6 +1682,16 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_node_log_capture_supports_per_node_paths() {
|
||||
let mut env = fake_cluster(ClusterTopology::single_pool(3));
|
||||
env.set_node_capture_log_path(1, "/tmp/node1.log").unwrap();
|
||||
assert_eq!(env.node_capture_log_paths[0], None);
|
||||
assert_eq!(env.node_capture_log_paths[1], Some("/tmp/node1.log".to_string()));
|
||||
assert_eq!(env.node_capture_log_paths[2], None);
|
||||
assert!(env.set_node_capture_log_path(3, "/tmp/invalid.log").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_node_env_rejects_invalid_index() {
|
||||
let mut env = fake_cluster(ClusterTopology::single_pool(4));
|
||||
|
||||
@@ -30,10 +30,10 @@ use s3s::access::{S3Access, S3AccessContext};
|
||||
use s3s::auth::SimpleAuth;
|
||||
use s3s::dto::{
|
||||
AbortMultipartUploadInput, AbortMultipartUploadOutput, CompleteMultipartUploadInput, CompleteMultipartUploadOutput,
|
||||
CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteObjectInput, DeleteObjectOutput, ETag,
|
||||
CreateMultipartUploadInput, CreateMultipartUploadOutput, DeleteMarkerEntry, DeleteObjectInput, DeleteObjectOutput, ETag,
|
||||
GetBucketVersioningInput, GetBucketVersioningOutput, GetObjectInput, GetObjectOutput, HeadBucketInput, HeadBucketOutput,
|
||||
HeadObjectInput, HeadObjectOutput, PutObjectInput, PutObjectOutput, StreamingBlob, Timestamp, TimestampFormat,
|
||||
UploadPartInput, UploadPartOutput,
|
||||
HeadObjectInput, HeadObjectOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ObjectVersionId, PutObjectInput,
|
||||
PutObjectOutput, StreamingBlob, Timestamp, TimestampFormat, UploadPartInput, UploadPartOutput,
|
||||
};
|
||||
use s3s::service::{S3Service, S3ServiceBuilder};
|
||||
use s3s::validation::{AwsNameValidation, NameValidation};
|
||||
@@ -91,6 +91,7 @@ pub enum Operation {
|
||||
GetObject,
|
||||
HeadObject,
|
||||
DeleteObject,
|
||||
ListObjectVersions,
|
||||
CreateMultipartUpload,
|
||||
UploadPart,
|
||||
CompleteMultipartUpload,
|
||||
@@ -109,6 +110,8 @@ pub enum FaultAction {
|
||||
/// already have buffered the rest of the current frame; the journal reports
|
||||
/// the threshold, and the backend never receives or stores the request.
|
||||
DisconnectAfterBytes(usize),
|
||||
/// Apply the request, then close the connection before returning its response.
|
||||
DisconnectAfterResponse,
|
||||
/// Drain a request body in fixed-size slices, sleeping after every slice.
|
||||
SlowDrain { chunk_bytes: usize, delay: Duration },
|
||||
/// Store the request normally but replace the response ETag.
|
||||
@@ -141,6 +144,8 @@ struct ControlState {
|
||||
|
||||
#[derive(Default)]
|
||||
struct StoreState {
|
||||
assign_own_version_ids: bool,
|
||||
assign_own_multipart_version_ids: bool,
|
||||
buckets: HashMap<String, BucketState>,
|
||||
uploads: HashMap<String, MultipartState>,
|
||||
total_bytes: usize,
|
||||
@@ -383,6 +388,22 @@ impl FakeS3Target {
|
||||
.is_some_and(|version| !version.delete_marker)
|
||||
}
|
||||
|
||||
/// Make the target mint its own version ids instead of mirroring the
|
||||
/// forwarded source version id — models a generic S3 service.
|
||||
pub fn assign_own_version_ids(&self, enabled: bool) {
|
||||
lock(&self.backend.store).assign_own_version_ids = enabled;
|
||||
}
|
||||
|
||||
/// Mint own version ids for the multipart path only — models a target
|
||||
/// that adopts PutObject version ids but not CreateMultipartUpload ones.
|
||||
pub fn assign_own_multipart_version_ids(&self, enabled: bool) {
|
||||
lock(&self.backend.store).assign_own_multipart_version_ids = enabled;
|
||||
}
|
||||
|
||||
pub fn active_multipart_upload_count(&self) -> usize {
|
||||
lock(&self.backend.store).uploads.len()
|
||||
}
|
||||
|
||||
/// Queue `times` copies of a fault for one operation.
|
||||
pub fn inject(&self, operation: Operation, action: FaultAction, times: usize) {
|
||||
if times == 0 {
|
||||
@@ -433,6 +454,25 @@ impl FakeS3Target {
|
||||
lock(&self.control).requests.drain(..).collect()
|
||||
}
|
||||
|
||||
/// Stored versions for one key as `(version_id, is_delete_marker)`, oldest
|
||||
/// first. Empty when the bucket or key does not exist. Lets purge tests
|
||||
/// assert on the target's actual state instead of inferring it from the
|
||||
/// request journal (a versioned DELETE is a silent no-op for missing ids).
|
||||
pub fn stored_versions(&self, bucket: &str, key: &str) -> Vec<(String, bool)> {
|
||||
let state = lock(&self.backend.store);
|
||||
state
|
||||
.buckets
|
||||
.get(bucket)
|
||||
.and_then(|bucket_state| bucket_state.objects.get(key))
|
||||
.map(|versions| {
|
||||
versions
|
||||
.iter()
|
||||
.map(|version| (version.version_id.clone(), version.delete_marker))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub async fn shutdown(mut self) {
|
||||
let _ = self.shutdown.send(true);
|
||||
if let Some(task) = self.task.take() {
|
||||
@@ -633,6 +673,7 @@ fn parse_request(method: &Method, uri: &Uri) -> ParsedRequest {
|
||||
let operation = match (method, key.is_some()) {
|
||||
(&Method::HEAD, false) => Operation::HeadBucket,
|
||||
(&Method::GET, false) if query.contains_key("versioning") => Operation::GetBucketVersioning,
|
||||
(&Method::GET, false) if query.contains_key("versions") => Operation::ListObjectVersions,
|
||||
(&Method::PUT, true) if upload_id.is_some() && part_number.is_some() => Operation::UploadPart,
|
||||
(&Method::PUT, true) if upload_id.is_some() || query.contains_key("partNumber") => Operation::Unknown,
|
||||
(&Method::POST, true) if query.contains_key("uploads") => Operation::CreateMultipartUpload,
|
||||
@@ -689,10 +730,17 @@ fn validate_retained_identifier(value: String, field: &str) -> S3Result<String>
|
||||
}
|
||||
}
|
||||
|
||||
fn new_version_id(headers: &HeaderMap) -> S3Result<String> {
|
||||
/// `assign_own` models a target that mints its own version ids (a generic S3
|
||||
/// service): the forwarded source-version-id header is validated but NOT
|
||||
/// mirrored into the stored version.
|
||||
fn new_version_id(headers: &HeaderMap, assign_own: bool) -> S3Result<String> {
|
||||
let Some(value) = header_value(headers, &SOURCE_VERSION_ID_HEADERS) else {
|
||||
return Ok(Uuid::new_v4().to_string());
|
||||
};
|
||||
if assign_own {
|
||||
validate_retained_identifier(value.trim().to_owned(), "source version ID")?;
|
||||
return Ok(Uuid::new_v4().to_string());
|
||||
}
|
||||
let value = validate_retained_identifier(value.trim().to_owned(), "source version ID")?;
|
||||
let version_id = Uuid::parse_str(&value).map_err(|_| s3s::s3_error!(InvalidArgument, "source version ID must be a UUID"))?;
|
||||
Ok(version_id.to_string())
|
||||
@@ -777,7 +825,10 @@ async fn apply_non_body_fault(fault: Option<&RequestFault>, control: &Mutex<Cont
|
||||
update_consumed(control, fault.expect("matched fault").sequence, 0);
|
||||
Err(scripted_disconnect_error())
|
||||
}
|
||||
Some(FaultAction::SlowDrain { .. }) | Some(FaultAction::WrongEtag) | None => Ok(()),
|
||||
Some(FaultAction::SlowDrain { .. })
|
||||
| Some(FaultAction::WrongEtag)
|
||||
| Some(FaultAction::DisconnectAfterResponse)
|
||||
| None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -818,7 +869,7 @@ async fn collect_stream(
|
||||
Some(FaultAction::SlowDrain { chunk_bytes, delay }) => {
|
||||
return collect_stream_slow(body, capacity, *chunk_bytes, *delay).await;
|
||||
}
|
||||
Some(FaultAction::WrongEtag) | None => {}
|
||||
Some(FaultAction::WrongEtag) | Some(FaultAction::DisconnectAfterResponse) | None => {}
|
||||
}
|
||||
|
||||
let mut output = BytesMut::with_capacity(capacity);
|
||||
@@ -880,6 +931,9 @@ fn apply_response_fault<T>(mut response: S3Response<T>, fault: Option<&RequestFa
|
||||
if fault.is_some_and(|fault| fault.action == FaultAction::WrongEtag) {
|
||||
response.headers.insert(ETAG, HeaderValue::from_static(WRONG_ETAG));
|
||||
}
|
||||
if fault.is_some_and(|fault| fault.action == FaultAction::DisconnectAfterResponse) {
|
||||
response.headers.insert(DISCONNECT_HEADER, HeaderValue::from_static("true"));
|
||||
}
|
||||
response
|
||||
}
|
||||
|
||||
@@ -1068,6 +1122,63 @@ impl S3 for FakeBackend {
|
||||
))
|
||||
}
|
||||
|
||||
/// Prefix + max-keys subset only — enough for the replication-check probe
|
||||
/// key allocation. No pagination markers or delimiter folding.
|
||||
async fn list_object_versions(
|
||||
&self,
|
||||
req: S3Request<ListObjectVersionsInput>,
|
||||
) -> S3Result<S3Response<ListObjectVersionsOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
apply_non_body_fault(fault.as_ref(), &self.control).await?;
|
||||
let state = lock(&self.store);
|
||||
let Some(bucket_state) = state.buckets.get(&req.input.bucket) else {
|
||||
return Err(s3s::s3_error!(NoSuchBucket, "bucket does not exist"));
|
||||
};
|
||||
let prefix = req.input.prefix.as_deref().unwrap_or_default();
|
||||
let max_keys = req.input.max_keys.unwrap_or(1000).max(0) as usize;
|
||||
|
||||
let mut keys: Vec<&String> = bucket_state.objects.keys().filter(|key| key.starts_with(prefix)).collect();
|
||||
keys.sort();
|
||||
|
||||
let mut versions = Vec::new();
|
||||
let mut delete_markers = Vec::new();
|
||||
'keys: for key in keys {
|
||||
for version in bucket_state.objects[key].iter().rev() {
|
||||
if versions.len() + delete_markers.len() >= max_keys {
|
||||
break 'keys;
|
||||
}
|
||||
if version.delete_marker {
|
||||
delete_markers.push(DeleteMarkerEntry {
|
||||
key: Some(key.clone()),
|
||||
version_id: Some(ObjectVersionId::from(version.version_id.clone())),
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
..Default::default()
|
||||
});
|
||||
} else {
|
||||
versions.push(s3s::dto::ObjectVersion {
|
||||
key: Some(key.clone()),
|
||||
version_id: Some(ObjectVersionId::from(version.version_id.clone())),
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
e_tag: Some(ETag::Strong(version.e_tag.clone())),
|
||||
size: Some(version.body.len() as i64),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(state);
|
||||
|
||||
Ok(apply_response_fault(
|
||||
S3Response::new(ListObjectVersionsOutput {
|
||||
name: Some(req.input.bucket),
|
||||
versions: Some(versions),
|
||||
delete_markers: Some(delete_markers),
|
||||
..Default::default()
|
||||
}),
|
||||
fault.as_ref(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn put_object(&self, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
let _body_permit = timeout(MAX_FAULT_DURATION, Arc::clone(&self.body_limit).acquire_owned())
|
||||
@@ -1078,7 +1189,8 @@ impl S3 for FakeBackend {
|
||||
let input = req.input;
|
||||
let body = collect_stream(input.body, input.content_length, fault.as_ref(), &self.control).await?;
|
||||
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
||||
let version_id = new_version_id(&headers)?;
|
||||
let assign_own = lock(&self.store).assign_own_version_ids;
|
||||
let version_id = new_version_id(&headers, assign_own)?;
|
||||
let e_tag = match source_etag(&headers)? {
|
||||
Some(value) => value,
|
||||
None => {
|
||||
@@ -1121,7 +1233,7 @@ impl S3 for FakeBackend {
|
||||
content_type: version.content_type,
|
||||
metadata: version.metadata,
|
||||
e_tag: Some(ETag::Strong(version.e_tag)),
|
||||
last_modified: Some(version.last_modified),
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
version_id: Some(version.version_id),
|
||||
..Default::default()
|
||||
}),
|
||||
@@ -1143,7 +1255,7 @@ impl S3 for FakeBackend {
|
||||
content_type: version.content_type,
|
||||
metadata: version.metadata,
|
||||
e_tag: Some(ETag::Strong(version.e_tag)),
|
||||
last_modified: Some(version.last_modified),
|
||||
last_modified: Some(version.last_modified.clone()),
|
||||
version_id: Some(version.version_id),
|
||||
..Default::default()
|
||||
}),
|
||||
@@ -1212,7 +1324,9 @@ impl S3 for FakeBackend {
|
||||
));
|
||||
}
|
||||
|
||||
let version_id = new_version_id(&headers)?;
|
||||
// `state` is the live store guard: read the flag from it. Re-locking
|
||||
// would self-deadlock (the store mutex is not reentrant).
|
||||
let version_id = new_version_id(&headers, state.assign_own_version_ids)?;
|
||||
upsert_version(
|
||||
&mut state,
|
||||
&input.bucket,
|
||||
@@ -1252,12 +1366,16 @@ impl S3 for FakeBackend {
|
||||
ensure_upload_budget(&state)?;
|
||||
validate_stored_metadata(&input.content_type, &input.metadata)?;
|
||||
let upload_id = Uuid::new_v4().to_string();
|
||||
// Read the flag before the mutable borrow of `state.uploads` below
|
||||
// (and never re-lock the store: the mutex is not reentrant).
|
||||
let mint_own = state.assign_own_version_ids || state.assign_own_multipart_version_ids;
|
||||
let version_id = new_version_id(&headers, mint_own)?;
|
||||
state.uploads.insert(
|
||||
upload_id.clone(),
|
||||
MultipartState {
|
||||
bucket: input.bucket.clone(),
|
||||
key: input.key.clone(),
|
||||
version_id: new_version_id(&headers)?,
|
||||
version_id,
|
||||
content_type: input.content_type,
|
||||
metadata: input.metadata,
|
||||
parts: BTreeMap::new(),
|
||||
|
||||
@@ -189,8 +189,6 @@ mod tests {
|
||||
("RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT", "100"),
|
||||
("RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED", "true"),
|
||||
("RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED", "true"),
|
||||
// Lower the min-size floor so every non-inline object below is eligible.
|
||||
("RUSTFS_GET_CODEC_STREAMING_MIN_SIZE", "4096"),
|
||||
// Route multipart objects through per-part codec streaming too.
|
||||
("RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE", "true"),
|
||||
// Lock optimization is on by default, but pin it so the gate's
|
||||
@@ -315,6 +313,13 @@ mod tests {
|
||||
},
|
||||
payload(64 * 1024, 2),
|
||||
),
|
||||
(
|
||||
Shape {
|
||||
key: "small-non-inline-256kib-plus",
|
||||
expect_large: true,
|
||||
},
|
||||
payload(256 * 1024 + 1, 6),
|
||||
),
|
||||
(
|
||||
Shape {
|
||||
key: "mid-1_5mib",
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
//! E2E tests for group management (fixes #2028).
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_get, awscurl_put, init_logging};
|
||||
use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use serial_test::serial;
|
||||
@@ -32,6 +32,56 @@ fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_k
|
||||
Client::from_conf(config)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let invalid_groups = [
|
||||
("test group", "group name contains whitespace"),
|
||||
("test=group", "group name contains reserved characters =,"),
|
||||
("test,group", "group name contains reserved characters =,"),
|
||||
];
|
||||
|
||||
for (group, expected_message) in invalid_groups {
|
||||
let body = serde_json::json!({
|
||||
"group": group,
|
||||
"members": [],
|
||||
"isRemove": false,
|
||||
"groupStatus": "enabled"
|
||||
})
|
||||
.to_string();
|
||||
let (status, response_body) = admin_request(
|
||||
&env.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/update-group-members",
|
||||
Some(body),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"invalid group {group:?} must return HTTP 400, body: {response_body}"
|
||||
);
|
||||
assert!(
|
||||
response_body.contains("<Code>InvalidArgument</Code>"),
|
||||
"invalid group {group:?} must return InvalidArgument, body: {response_body}"
|
||||
);
|
||||
assert!(
|
||||
response_body.contains(&format!("<Message>{expected_message}</Message>")),
|
||||
"invalid group {group:?} returned an unexpected message: {response_body}"
|
||||
);
|
||||
}
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Test that deleting a group with members fails, and deleting an empty group succeeds.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
|
||||
@@ -39,6 +39,10 @@ pub mod fault_proxy;
|
||||
#[cfg(test)]
|
||||
mod reliability_disk_fault_test;
|
||||
|
||||
// Privileged Linux-only 3x4 replacement rebuild proof for rustfs#5869/#1791.
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
mod replacement_privileged_e2e_test;
|
||||
|
||||
// dist-13 (backlog#1150/#1155): e2e regression net proving a large-object
|
||||
// degraded EC read never returns a silently truncated body (rustfs#4594/#4560/#4585).
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -285,6 +285,198 @@ async fn allow_anonymous_put_object(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One rejected POST Object upload driven end-to-end (backlog#1838): starts a
|
||||
/// fresh server, allows anonymous PutObject on `bucket`, posts an anonymous
|
||||
/// POST Object form whose policy carries `policy_conditions` and whose form
|
||||
/// carries `form_fields` on top of the mandatory key+policy fields, then
|
||||
/// asserts the expected status, error code, and lowercase-body mention.
|
||||
/// `case` prefixes every assertion message so a failing table row is
|
||||
/// identifiable at a glance.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn run_post_object_policy_case(
|
||||
bucket: &str,
|
||||
object_key: &str,
|
||||
policy_conditions: Vec<serde_json::Value>,
|
||||
form_fields: &[(&str, &str)],
|
||||
file_body: &[u8],
|
||||
expected_status: reqwest::StatusCode,
|
||||
expected_code: &str,
|
||||
expected_mention: &str,
|
||||
case: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(policy_conditions);
|
||||
|
||||
let mut post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy);
|
||||
for (name, value) in form_fields {
|
||||
post_form = post_form.text(name.to_string(), value.to_string());
|
||||
}
|
||||
let post_form = post_form.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(file_body.to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, expected_status, "[{case}] unexpected status, body: {response_body}");
|
||||
assert!(
|
||||
response_body.contains(expected_code),
|
||||
"[{case}] response should contain {expected_code}, got: {response_body}"
|
||||
);
|
||||
assert!(
|
||||
response_body_lower.contains(expected_mention),
|
||||
"[{case}] response should mention {expected_mention}, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Table-driven fold of the nine `*_missing_from_policy_conditions` POST
|
||||
/// Object tests (backlog#1838 PR1). Every row keeps its original test's exact
|
||||
/// bucket, key, form field, file body, and expected error strings; the shared
|
||||
/// shape is: policy pins bucket + key + content-length-range only, the form
|
||||
/// smuggles one extra field the policy never declared, and the upload must be
|
||||
/// rejected with 403 AccessDenied naming the offending field.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_fields_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
// (case, bucket, object_key, form field, file body, expected code, expected mention)
|
||||
type Case = (
|
||||
&'static str,
|
||||
&'static str,
|
||||
&'static str,
|
||||
(&'static str, &'static str),
|
||||
&'static [u8],
|
||||
&'static str,
|
||||
&'static str,
|
||||
);
|
||||
let cases: &[Case] = &[
|
||||
(
|
||||
"cache-control",
|
||||
"anon-post-policy-cache-control-missing",
|
||||
"uploads/cache-control-missing.txt",
|
||||
("Cache-Control", "max-age=60"),
|
||||
b"post-policy-cache-control-missing",
|
||||
"AccessDenied",
|
||||
"cache-control",
|
||||
),
|
||||
(
|
||||
"content-language",
|
||||
"anon-post-policy-content-language-missing",
|
||||
"uploads/content-language-missing.txt",
|
||||
("Content-Language", "en-US"),
|
||||
b"post-policy-content-language-missing",
|
||||
"AccessDenied",
|
||||
"content-language",
|
||||
),
|
||||
(
|
||||
"content-encoding",
|
||||
"anon-post-policy-content-encoding-missing",
|
||||
"uploads/content-encoding-missing.txt",
|
||||
("Content-Encoding", "gzip"),
|
||||
b"post-policy-content-encoding-missing",
|
||||
"AccessDenied",
|
||||
"content-encoding",
|
||||
),
|
||||
(
|
||||
"website-redirect-location",
|
||||
"anon-post-policy-website-redirect-missing",
|
||||
"uploads/website-redirect-missing.txt",
|
||||
("x-amz-website-redirect-location", "/docs/landing.html"),
|
||||
b"post-policy-website-redirect-missing",
|
||||
"AccessDenied",
|
||||
"x-amz-website-redirect-location",
|
||||
),
|
||||
(
|
||||
"expires",
|
||||
"anon-post-policy-expires-missing",
|
||||
"uploads/expires-missing-object.txt",
|
||||
("Expires", "Wed, 21 Oct 2037 07:28:00 GMT"),
|
||||
b"post-policy-expires-missing",
|
||||
"AccessDenied",
|
||||
"expires",
|
||||
),
|
||||
(
|
||||
"tagging",
|
||||
"anon-post-policy-tagging-missing",
|
||||
"uploads/tagging-missing-object.txt",
|
||||
("x-amz-tagging", "project=alpha&env=test"),
|
||||
b"post-policy-tagging-missing",
|
||||
"AccessDenied",
|
||||
"x-amz-tagging",
|
||||
),
|
||||
(
|
||||
"metadata",
|
||||
"anon-post-policy-meta-reject",
|
||||
"uploads/meta-reject-object.txt",
|
||||
("x-amz-meta-project", "alpha-demo"),
|
||||
b"post-policy-body",
|
||||
"<Code>AccessDenied</Code>",
|
||||
"x-amz-meta-project",
|
||||
),
|
||||
(
|
||||
"metadata-new-key",
|
||||
"anon-post-policy-meta-name-missing",
|
||||
"uploads/meta-name-missing.txt",
|
||||
("x-amz-meta-name", "demo-name"),
|
||||
b"post-policy-meta-name-missing",
|
||||
"<Code>AccessDenied</Code>",
|
||||
"x-amz-meta-name",
|
||||
),
|
||||
(
|
||||
"content-type",
|
||||
"anon-post-policy-content-type-missing",
|
||||
"uploads/content-type-missing.txt",
|
||||
("Content-Type", "text/plain"),
|
||||
b"post-policy-content-type-missing",
|
||||
"AccessDenied",
|
||||
"content-type",
|
||||
),
|
||||
];
|
||||
|
||||
for (case, bucket, object_key, form_field, file_body, expected_code, expected_mention) in cases {
|
||||
run_post_object_policy_case(
|
||||
bucket,
|
||||
object_key,
|
||||
vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
],
|
||||
&[*form_field],
|
||||
file_body,
|
||||
reqwest::StatusCode::FORBIDDEN,
|
||||
expected_code,
|
||||
expected_mention,
|
||||
case,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_multipart_control_apis_require_auth() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
@@ -2869,59 +3061,6 @@ async fn test_anonymous_post_object_rejects_cache_control_policy_mismatch() -> R
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_cache_control_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-cache-control-missing";
|
||||
let object_key = "uploads/cache-control-missing.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("Cache-Control", "max-age=60")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-cache-control-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert!(
|
||||
response_body_lower.contains("cache-control"),
|
||||
"response should mention cache-control, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_content_language_field_exact_policy_match()
|
||||
@@ -3034,59 +3173,6 @@ async fn test_anonymous_post_object_rejects_content_language_policy_mismatch()
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_content_language_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-content-language-missing";
|
||||
let object_key = "uploads/content-language-missing.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("Content-Language", "en-US")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-content-language-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert!(
|
||||
response_body_lower.contains("content-language"),
|
||||
"response should mention content-language, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_content_encoding_field_exact_policy_match()
|
||||
@@ -3199,59 +3285,6 @@ async fn test_anonymous_post_object_rejects_content_encoding_policy_mismatch()
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_content_encoding_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-content-encoding-missing";
|
||||
let object_key = "uploads/content-encoding-missing.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("Content-Encoding", "gzip")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-content-encoding-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert!(
|
||||
response_body_lower.contains("content-encoding"),
|
||||
"response should mention content-encoding, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_website_redirect_location_exact_policy_match()
|
||||
@@ -3310,59 +3343,6 @@ async fn test_anonymous_post_object_accepts_website_redirect_location_exact_poli
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_website_redirect_location_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-website-redirect-missing";
|
||||
let object_key = "uploads/website-redirect-missing.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("x-amz-website-redirect-location", "/docs/landing.html")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-website-redirect-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert!(
|
||||
response_body_lower.contains("x-amz-website-redirect-location"),
|
||||
"response should mention x-amz-website-redirect-location, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_website_redirect_location_policy_mismatch()
|
||||
@@ -3529,59 +3509,6 @@ async fn test_anonymous_post_object_rejects_expires_field_policy_mismatch() -> R
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_expires_field_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-expires-missing";
|
||||
let object_key = "uploads/expires-missing-object.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("Expires", "Wed, 21 Oct 2037 07:28:00 GMT")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-expires-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert!(
|
||||
response_body_lower.contains("expires"),
|
||||
"response should mention Expires, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_object_lock_retention_without_permission()
|
||||
@@ -4110,115 +4037,6 @@ async fn test_anonymous_post_object_rejects_tagging_field_policy_mismatch() -> R
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_tagging_field_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-tagging-missing";
|
||||
let object_key = "uploads/tagging-missing-object.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("x-amz-tagging", "project=alpha&env=test")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-tagging-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert!(
|
||||
response_body_lower.contains("x-amz-tagging"),
|
||||
"response should mention x-amz-tagging, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_metadata_field_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-meta-reject";
|
||||
let object_key = "uploads/meta-reject-object.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("x-amz-meta-project", "alpha-demo")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-body".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(
|
||||
response_body.contains("<Code>AccessDenied</Code>"),
|
||||
"response should contain AccessDenied code, got: {response_body}"
|
||||
);
|
||||
assert!(
|
||||
response_body_lower.contains("x-amz-meta-project"),
|
||||
"response should mention the missing metadata field, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_metadata_field_exact_policy_mismatch()
|
||||
@@ -4388,59 +4206,6 @@ async fn test_anonymous_post_object_allows_x_ignore_fields_outside_policy_condit
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_metadata_field_missing_from_policy_conditions_for_new_key()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-meta-name-missing";
|
||||
let object_key = "uploads/meta-name-missing.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("x-amz-meta-name", "demo-name")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-meta-name-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("<Code>AccessDenied</Code>"));
|
||||
assert!(
|
||||
response_body_lower.contains("x-amz-meta-name"),
|
||||
"response should mention x-amz-meta-name, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_metadata_uuid_exact_policy_mismatch()
|
||||
@@ -4876,59 +4641,6 @@ async fn test_anonymous_post_object_rejects_content_type_policy_mismatch() -> Re
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_content_type_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-content-type-missing";
|
||||
let object_key = "uploads/content-type-missing.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("Content-Type", "text/plain")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-content-type-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert!(
|
||||
response_body_lower.contains("content-type"),
|
||||
"response should mention content-type, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_expands_tar_entries_with_prefix_headers()
|
||||
|
||||
@@ -252,6 +252,7 @@ impl QuotaTestEnv {
|
||||
#[cfg(test)]
|
||||
mod integration_tests {
|
||||
use super::*;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
@@ -963,9 +964,27 @@ mod integration_tests {
|
||||
.send()
|
||||
.await;
|
||||
|
||||
assert!(complete_result.is_err());
|
||||
let complete_error = complete_result.expect_err("multipart completion above quota must be rejected");
|
||||
assert_eq!(complete_error.as_service_error().and_then(|error| error.code()), Some("InvalidRequest"));
|
||||
assert!(!env.object_exists("over_quota.txt").await?);
|
||||
|
||||
let staged_parts = env
|
||||
.client
|
||||
.list_parts()
|
||||
.bucket(&env.bucket_name)
|
||||
.key("over_quota.txt")
|
||||
.upload_id(upload_id2)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(staged_parts.parts().len(), 2, "quota rejection must preserve the multipart upload");
|
||||
env.client
|
||||
.abort_multipart_upload()
|
||||
.bucket(&env.bucket_name)
|
||||
.key("over_quota.txt")
|
||||
.upload_id(upload_id2)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
env.cleanup_bucket().await?;
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -349,11 +349,32 @@ mod tests {
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let first_inline = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("versions/inline.bin")
|
||||
.body(ByteStream::from(payload(8 * 1024, 40)))
|
||||
.send()
|
||||
.await?;
|
||||
let first_inline_version = first_inline
|
||||
.version_id()
|
||||
.ok_or("first inline PUT did not return a version ID")?;
|
||||
let second_inline = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("versions/inline.bin")
|
||||
.body(ByteStream::from(payload(8 * 1024, 41)))
|
||||
.send()
|
||||
.await?;
|
||||
let second_inline_version = second_inline
|
||||
.version_id()
|
||||
.ok_or("second inline PUT did not return a version ID")?;
|
||||
|
||||
let first = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload(256 * 1024, 41)))
|
||||
.body(ByteStream::from(payload(128 * 1024, 41)))
|
||||
.send()
|
||||
.await?;
|
||||
let first_version = first.version_id().ok_or("first PUT did not return a version ID")?;
|
||||
@@ -361,16 +382,36 @@ mod tests {
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload(256 * 1024, 42)))
|
||||
.body(ByteStream::from(payload(3 * 1024 * 1024, 42)))
|
||||
.send()
|
||||
.await?;
|
||||
let second_version = second.version_id().ok_or("second PUT did not return a version ID")?;
|
||||
let delete = client.delete_object().bucket(bucket).key(key).send().await?;
|
||||
let delete_version = delete.version_id().ok_or("delete marker did not return a version ID")?;
|
||||
|
||||
let first_inline_census = harness.census_object_version(0, bucket, "versions/inline.bin", Some(first_inline_version))?;
|
||||
let second_inline_census =
|
||||
harness.census_object_version(0, bucket, "versions/inline.bin", Some(second_inline_version))?;
|
||||
let first_census = harness.census_object_version(0, bucket, key, Some(first_version))?;
|
||||
let first_other_disk_census = harness.census_object_version(1, bucket, key, Some(first_version))?;
|
||||
let second_census = harness.census_object_version(0, bucket, key, Some(second_version))?;
|
||||
let delete_census = harness.census_object_version(0, bucket, key, Some(delete_version))?;
|
||||
assert!(
|
||||
first_inline_census.is_complete() && second_inline_census.is_complete(),
|
||||
"inline version physical census is incomplete: first={first_inline_census:?} second={second_inline_census:?}"
|
||||
);
|
||||
assert!(
|
||||
first_inline_census.present_part_fingerprints.is_empty() && second_inline_census.present_part_fingerprints.is_empty(),
|
||||
"inline versions must not select external shard files: first={first_inline_census:?} second={second_inline_census:?}"
|
||||
);
|
||||
assert!(
|
||||
first_inline_census.inline_data_fingerprint.is_some() && second_inline_census.inline_data_fingerprint.is_some(),
|
||||
"inline versions must fingerprint payload bytes stored in xl.meta"
|
||||
);
|
||||
assert_ne!(
|
||||
first_inline_census.inline_data_fingerprint, second_inline_census.inline_data_fingerprint,
|
||||
"same-size inline versions with different payloads must retain distinct xl.meta fingerprints"
|
||||
);
|
||||
assert!(
|
||||
first_census.is_complete(),
|
||||
"first version physical census is incomplete: {first_census:?}"
|
||||
@@ -379,6 +420,14 @@ mod tests {
|
||||
second_census.is_complete(),
|
||||
"second version physical census is incomplete: {second_census:?}"
|
||||
);
|
||||
assert!(
|
||||
first_other_disk_census.is_complete(),
|
||||
"first version physical census on the second disk is incomplete: {first_other_disk_census:?}"
|
||||
);
|
||||
assert_ne!(
|
||||
first_census.erasure_index, first_other_disk_census.erasure_index,
|
||||
"physical census must preserve each disk's erasure index"
|
||||
);
|
||||
assert_ne!(
|
||||
first_census.data_dir, second_census.data_dir,
|
||||
"distinct object versions must select distinct physical data directories"
|
||||
@@ -387,6 +436,24 @@ mod tests {
|
||||
first_census.expected_part_numbers, second_census.expected_part_numbers,
|
||||
"same single-part shape should expose the same part numbers"
|
||||
);
|
||||
let first_part = first_census
|
||||
.present_part_fingerprints
|
||||
.values()
|
||||
.next()
|
||||
.ok_or("first version did not expose a physical part fingerprint")?;
|
||||
let second_part = second_census
|
||||
.present_part_fingerprints
|
||||
.values()
|
||||
.next()
|
||||
.ok_or("second version did not expose a physical part fingerprint")?;
|
||||
assert_ne!(
|
||||
first_part.size, second_part.size,
|
||||
"different shard lengths must retain their physical sizes"
|
||||
);
|
||||
assert_ne!(
|
||||
first_part.sha256, second_part.sha256,
|
||||
"different shard contents must retain their physical hashes"
|
||||
);
|
||||
assert!(
|
||||
delete_census.is_complete(),
|
||||
"delete marker physical census is incomplete: {delete_census:?}"
|
||||
@@ -396,7 +463,7 @@ mod tests {
|
||||
"delete marker must not declare object shards: {delete_census:?}"
|
||||
);
|
||||
assert!(
|
||||
delete_census.present_part_numbers.is_empty(),
|
||||
delete_census.present_part_fingerprints.is_empty(),
|
||||
"delete marker must not select stale object shards: {delete_census:?}"
|
||||
);
|
||||
Ok(())
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,6 +18,7 @@ use crate::common::{
|
||||
};
|
||||
use crate::fake_s3_target::{
|
||||
FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
|
||||
RequestRecord,
|
||||
};
|
||||
use crate::kms::common::{create_key_with_specific_id, sse_customer_key_md5_base64};
|
||||
use crate::storage_api::replication_extension::BucketTargetSys;
|
||||
@@ -628,6 +629,17 @@ async fn put_bucket_replication_with_delete_statuses(
|
||||
target_arn: &str,
|
||||
delete_marker_status: &str,
|
||||
version_delete_status: Option<&str>,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
put_bucket_replication_with_statuses(env, bucket, target_arn, delete_marker_status, version_delete_status, "Enabled").await
|
||||
}
|
||||
|
||||
async fn put_bucket_replication_with_statuses(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
target_arn: &str,
|
||||
delete_marker_status: &str,
|
||||
version_delete_status: Option<&str>,
|
||||
existing_object_status: &str,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let delete_replication = version_delete_status
|
||||
.map(|status| format!("<DeleteReplication><Status>{status}</Status></DeleteReplication>"))
|
||||
@@ -644,7 +656,7 @@ async fn put_bucket_replication_with_delete_statuses(
|
||||
</DeleteMarkerReplication>
|
||||
{delete_replication}
|
||||
<ExistingObjectReplication>
|
||||
<Status>Enabled</Status>
|
||||
<Status>{existing_object_status}</Status>
|
||||
</ExistingObjectReplication>
|
||||
<Destination>
|
||||
<Bucket>{target_arn}</Bucket>
|
||||
@@ -2594,6 +2606,9 @@ async fn test_replication_check_succeeds_with_remote_target() -> Result<(), Box<
|
||||
assert_eq!(payload["Targets"].as_array().map(Vec::len), Some(1));
|
||||
assert_eq!(payload["Targets"][0]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["Put"]["Status"], "OK");
|
||||
// A RustFS target adopts the source version id, so the P1-19
|
||||
// version-identity probe passes.
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["VersionFidelity"]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["DeleteMarker"]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["VersionDelete"]["Status"], "OK");
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["Cleanup"]["Status"], "OK");
|
||||
@@ -7487,3 +7502,897 @@ async fn test_scanner_never_cascades_inbound_replicas() -> TestResult {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// P1-19 review follow-up: multipart fixes the target version at initiate
|
||||
/// and only reports it on completion, so a target can adopt PutObject
|
||||
/// version ids and still mint its own there — the check must not report OK
|
||||
/// while multipart deletes and heals would silently miss.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_flags_multipart_only_version_minting_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "multipart-fidelity-dst";
|
||||
target.create_bucket(target_bucket);
|
||||
// PutObject mirrors the source version id; CreateMultipartUpload does not.
|
||||
target.assign_own_multipart_version_ids(true);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
|
||||
|
||||
let source_bucket = "multipart-fidelity-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
|
||||
let target_arn = set_replication_target_with_options(
|
||||
&source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let response = run_replication_check(&source_env, source_bucket).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let payload: serde_json::Value = response.json().await?;
|
||||
|
||||
assert_eq!(payload["Status"], "FAILED", "multipart drift must fail the check: {payload}");
|
||||
let target_report = &payload["Targets"][0];
|
||||
let fidelity = &target_report["Phases"]["VersionFidelity"];
|
||||
assert_eq!(fidelity["Status"], "FAILED", "{payload}");
|
||||
assert_eq!(fidelity["Code"], "BucketRemoteTargetVersionMismatch", "{payload}");
|
||||
assert!(
|
||||
fidelity["Error"]
|
||||
.as_str()
|
||||
.is_some_and(|error| error.contains("CreateMultipartUpload")),
|
||||
"the failure must name the multipart path: {payload}"
|
||||
);
|
||||
// The PutObject leg mirrored, so it is the multipart probe that failed.
|
||||
assert_eq!(target_report["Phases"]["Put"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "SKIPPED", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
|
||||
|
||||
let probe_key = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::PutObject)
|
||||
.and_then(|record| record.key)
|
||||
.ok_or("the probe PUT never reached the fake target")?;
|
||||
assert!(
|
||||
target.stored_versions(target_bucket, &probe_key).is_empty(),
|
||||
"both probe versions must be cleaned up on the mismatching target"
|
||||
);
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_aborts_failed_multipart_probes() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "multipart-cleanup-dst";
|
||||
target.create_bucket(target_bucket);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
|
||||
|
||||
let source_bucket = "multipart-cleanup-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
|
||||
let target_arn = set_replication_target_with_options(
|
||||
&source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
for failed_operation in [FakeTargetOperation::UploadPart, FakeTargetOperation::CompleteMultipartUpload] {
|
||||
target.clear_faults();
|
||||
target.take_requests();
|
||||
target.inject(failed_operation, FakeTargetFault::Status(StatusCode::SERVICE_UNAVAILABLE), 16);
|
||||
|
||||
let response = run_replication_check(&source_env, source_bucket).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let payload: serde_json::Value = response.json().await?;
|
||||
assert_eq!(
|
||||
payload["Status"], "FAILED",
|
||||
"the injected multipart failure must fail the check: {payload}"
|
||||
);
|
||||
|
||||
let requests = target.requests();
|
||||
assert!(
|
||||
requests.iter().any(|request| {
|
||||
request.operation == failed_operation
|
||||
&& request.fault == Some(FakeTargetFault::Status(StatusCode::SERVICE_UNAVAILABLE))
|
||||
}),
|
||||
"the check must reach the injected {failed_operation:?} failure: {requests:?}"
|
||||
);
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|request| request.operation == FakeTargetOperation::AbortMultipartUpload),
|
||||
"the failed {failed_operation:?} probe must be aborted: {requests:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
target.active_multipart_upload_count(),
|
||||
0,
|
||||
"the failed {failed_operation:?} probe must not leave multipart state"
|
||||
);
|
||||
assert_eq!(payload["Targets"][0]["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
|
||||
}
|
||||
|
||||
target.clear_faults();
|
||||
target.take_requests();
|
||||
target.inject(FakeTargetOperation::CompleteMultipartUpload, FakeTargetFault::DisconnectAfterResponse, 16);
|
||||
|
||||
let response = run_replication_check(&source_env, source_bucket).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let payload: serde_json::Value = response.json().await?;
|
||||
let target_report = &payload["Targets"][0];
|
||||
assert_eq!(target_report["Status"], "FAILED", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["VersionFidelity"]["Status"], "FAILED", "{payload}");
|
||||
assert_eq!(
|
||||
target_report["Phases"]["Cleanup"]["Status"], "OK",
|
||||
"NoSuchUpload after an ambiguous complete means the multipart artifact is gone: {payload}"
|
||||
);
|
||||
let requests = target.requests();
|
||||
let completed_key = requests
|
||||
.iter()
|
||||
.find(|request| {
|
||||
request.operation == FakeTargetOperation::CompleteMultipartUpload
|
||||
&& request.fault == Some(FakeTargetFault::DisconnectAfterResponse)
|
||||
})
|
||||
.and_then(|request| request.key.as_deref())
|
||||
.expect("the scripted complete response disconnect must be observed");
|
||||
assert!(
|
||||
requests
|
||||
.iter()
|
||||
.any(|request| request.operation == FakeTargetOperation::AbortMultipartUpload),
|
||||
"the ambiguous complete must still attempt abort: {requests:?}"
|
||||
);
|
||||
assert_eq!(target.active_multipart_upload_count(), 0);
|
||||
assert!(
|
||||
target.stored_versions(target_bucket, completed_key).is_empty(),
|
||||
"outer cleanup must remove the object committed before the response disconnect"
|
||||
);
|
||||
|
||||
target.clear_faults();
|
||||
target.take_requests();
|
||||
target.inject(FakeTargetOperation::UploadPart, FakeTargetFault::Status(StatusCode::FORBIDDEN), 16);
|
||||
target.inject(
|
||||
FakeTargetOperation::AbortMultipartUpload,
|
||||
FakeTargetFault::Status(StatusCode::SERVICE_UNAVAILABLE),
|
||||
16,
|
||||
);
|
||||
|
||||
let response = run_replication_check(&source_env, source_bucket).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let payload: serde_json::Value = response.json().await?;
|
||||
let target_report = &payload["Targets"][0];
|
||||
assert_eq!(target_report["Status"], "FAILED", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["VersionFidelity"]["Status"], "FAILED", "{payload}");
|
||||
assert_eq!(
|
||||
target_report["Phases"]["Cleanup"]["Status"], "FAILED",
|
||||
"an unremoved multipart probe must be reported as a cleanup failure: {payload}"
|
||||
);
|
||||
assert_eq!(
|
||||
target_report["Error"], "s3:ReplicateObject permissions missing for replication user",
|
||||
"the primary multipart error must remain the target error: {payload}"
|
||||
);
|
||||
assert_eq!(
|
||||
target_report["Phases"]["VersionFidelity"]["Error"], "s3:ReplicateObject permissions missing for replication user",
|
||||
"{payload}"
|
||||
);
|
||||
assert_eq!(
|
||||
target_report["Phases"]["Cleanup"]["Error"], "failed to abort multipart replication probe",
|
||||
"{payload}"
|
||||
);
|
||||
assert!(
|
||||
target.requests().iter().any(|request| {
|
||||
request.operation == FakeTargetOperation::AbortMultipartUpload
|
||||
&& request.fault == Some(FakeTargetFault::Status(StatusCode::SERVICE_UNAVAILABLE))
|
||||
}),
|
||||
"the abort failure must be observed"
|
||||
);
|
||||
assert_eq!(
|
||||
target.active_multipart_upload_count(),
|
||||
1,
|
||||
"the report must match the retained multipart state"
|
||||
);
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// P1-19 (backlog#1675): the supported replication contract is targets that
|
||||
/// adopt the source version id (RustFS/MinIO semantics). A target that mints
|
||||
/// its own version ids silently breaks every version-addressed operation that
|
||||
/// follows — version deletes and heal re-drives never match, diverging the
|
||||
/// two sides. replication-check must surface this explicitly: a
|
||||
/// VersionFidelity phase that compares the probe PUT's response version id
|
||||
/// against the sent source version id and fails with
|
||||
/// BucketRemoteTargetVersionMismatch — while still cleaning up the probe
|
||||
/// object via the version id the target actually assigned.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_replication_check_flags_version_minting_target() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "version-fidelity-dst";
|
||||
target.create_bucket(target_bucket);
|
||||
target.assign_own_version_ids(true);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &env_vars).await?;
|
||||
|
||||
let source_bucket = "version-fidelity-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
|
||||
let target_arn = set_replication_target_with_options(
|
||||
&source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let response = run_replication_check(&source_env, source_bucket).await?;
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let payload: serde_json::Value = response.json().await?;
|
||||
|
||||
assert_eq!(
|
||||
payload["Status"], "FAILED",
|
||||
"a version-minting target must fail the replication check: {payload}"
|
||||
);
|
||||
let target_report = &payload["Targets"][0];
|
||||
assert_eq!(target_report["Status"], "FAILED", "target must be FAILED: {payload}");
|
||||
let fidelity = &target_report["Phases"]["VersionFidelity"];
|
||||
assert_eq!(fidelity["Status"], "FAILED", "VersionFidelity phase must fail: {payload}");
|
||||
assert_eq!(
|
||||
fidelity["Code"], "BucketRemoteTargetVersionMismatch",
|
||||
"the failure must carry a machine-readable code: {payload}"
|
||||
);
|
||||
// The probe PUT itself succeeded (fidelity is judged from its response);
|
||||
// the later mutation phases are pointless against a drifting target and
|
||||
// must be skipped, but cleanup still runs.
|
||||
assert_eq!(target_report["Phases"]["Put"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "SKIPPED", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
|
||||
|
||||
// The probe PUT must carry the source version as `?versionId=` — the
|
||||
// exact shape live replication uses (P0-5), and the only shape MinIO
|
||||
// consumes. The journal records the query value.
|
||||
let probe_put = target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|record| record.operation == FakeTargetOperation::PutObject)
|
||||
.ok_or("the probe PUT never reached the fake target")?;
|
||||
let probe_query_version = probe_put
|
||||
.version_id
|
||||
.as_deref()
|
||||
.ok_or("the probe PUT must carry a versionId query")?;
|
||||
assert!(
|
||||
uuid::Uuid::parse_str(probe_query_version).is_ok(),
|
||||
"the probe versionId query must be the source uuid, got {probe_query_version}"
|
||||
);
|
||||
|
||||
// No probe residue: cleanup must address the version id the target
|
||||
// actually assigned, not the source id (which never matched anything).
|
||||
let probe_key = probe_put.key.ok_or("probe PUT journal record has no key")?;
|
||||
assert!(
|
||||
target.stored_versions(target_bucket, &probe_key).is_empty(),
|
||||
"the probe object must be cleaned up on the mismatching target"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// --- P1-21 (backlog#1675): delayed delete-marker purge failure handling ---
|
||||
//
|
||||
// The fixtures below wire a versioned source bucket to a FakeS3Target with the
|
||||
// default replication shape: DeleteMarkerReplication=Enabled and
|
||||
// DeleteReplication omitted. With version-delete replication unconfigured,
|
||||
// purging the source marker version emits no replication event, and the data
|
||||
// scanner cannot see a source version that is gone — the delayed purge watcher
|
||||
// spawned by the marker replication is the ONLY channel that can remove the
|
||||
// replicated marker from the target.
|
||||
|
||||
const DELAYED_PURGE_KEY: &str = "doc.txt";
|
||||
|
||||
fn delayed_purge_process_env() -> Vec<(&'static str, &'static str)> {
|
||||
let mut env = replication_fast_env();
|
||||
env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
env
|
||||
}
|
||||
|
||||
async fn start_delayed_purge_fixture(
|
||||
source_bucket: &str,
|
||||
target_bucket: &str,
|
||||
) -> Result<(FakeS3Target, RustFSTestEnvironment, Client), Box<dyn Error + Send + Sync>> {
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket(target_bucket);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
source_env
|
||||
.start_rustfs_server_with_env(vec![], &delayed_purge_process_env())
|
||||
.await?;
|
||||
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
|
||||
let target_arn = set_replication_target_with_options(
|
||||
&source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
Ok((target, source_env, source_client))
|
||||
}
|
||||
|
||||
/// PUT an object, stack a delete marker on it, and wait until the fake target
|
||||
/// stores the marker replica. Returns the source marker version id — the fake
|
||||
/// target mirrors it because delete replication forwards
|
||||
/// `x-*-source-version-id`.
|
||||
///
|
||||
/// Timing budget for callers: the delayed purge watcher only observes the
|
||||
/// source for ~4s after the marker replication completes, so the source-side
|
||||
/// marker-version DELETE must be issued promptly after this returns (the
|
||||
/// 100ms journal poll below keeps the detection latency small).
|
||||
async fn replicate_delete_marker(
|
||||
target: &FakeS3Target,
|
||||
target_bucket: &str,
|
||||
source_client: &Client,
|
||||
source_bucket: &str,
|
||||
) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(DELAYED_PURGE_KEY)
|
||||
.body(ByteStream::from_static(b"delayed purge payload"))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let delete = source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(DELAYED_PURGE_KEY)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(delete.delete_marker(), Some(true), "unversioned DELETE must create a marker");
|
||||
let marker_version = delete
|
||||
.version_id()
|
||||
.ok_or("source DELETE omitted the marker version ID")?
|
||||
.to_string();
|
||||
|
||||
// Wait for ANY delete marker: a target that mints its own version ids
|
||||
// does not mirror the source one.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
let replicated = target
|
||||
.stored_versions(target_bucket, DELAYED_PURGE_KEY)
|
||||
.iter()
|
||||
.any(|(_, delete_marker)| *delete_marker);
|
||||
if replicated {
|
||||
return Ok(marker_version);
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(
|
||||
format!("fake target never stored the replicated delete marker; journal: {:?}", target.requests()).into(),
|
||||
);
|
||||
}
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Journal records of purge attempts: target DELETE calls addressing the marker
|
||||
/// version explicitly. The marker-creation replica DELETE carries no
|
||||
/// `versionId` query, so the version id is an exact discriminator.
|
||||
fn delayed_purge_attempts(target: &FakeS3Target, marker_version: &str) -> Vec<RequestRecord> {
|
||||
target
|
||||
.requests()
|
||||
.into_iter()
|
||||
.filter(|record| {
|
||||
record.operation == FakeTargetOperation::DeleteObject
|
||||
&& record.key.as_deref() == Some(DELAYED_PURGE_KEY)
|
||||
&& record.version_id.as_deref() == Some(marker_version)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn wait_for_target_marker_purged(
|
||||
target: &FakeS3Target,
|
||||
target_bucket: &str,
|
||||
max_wait: Duration,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let deadline = tokio::time::Instant::now() + max_wait;
|
||||
loop {
|
||||
let marker_present = target
|
||||
.stored_versions(target_bucket, DELAYED_PURGE_KEY)
|
||||
.iter()
|
||||
.any(|(_, delete_marker)| *delete_marker);
|
||||
if !marker_present {
|
||||
return Ok(());
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"target delete marker was never purged; target state: {:?}",
|
||||
target.stored_versions(target_bucket, DELAYED_PURGE_KEY)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// P1-21: the delayed purge's single target DELETE currently swallows failures
|
||||
/// (`let _ =`), so one transient target error strands the replicated marker on
|
||||
/// the target forever. Contract under test: a failed purge attempt is retried
|
||||
/// within the watch window and converges once the fault clears.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delayed_delete_marker_purge_retries_after_transient_target_failure() -> TestResult {
|
||||
init_logging();
|
||||
let source_bucket = "delayed-purge-retry-src";
|
||||
let target_bucket = "delayed-purge-retry-dst";
|
||||
let (target, _source_env, source_client) = start_delayed_purge_fixture(source_bucket, target_bucket).await?;
|
||||
|
||||
let marker_version = replicate_delete_marker(&target, target_bucket, &source_client, source_bucket).await?;
|
||||
|
||||
// Four scripted failures. Fault budget accounting (each journal record
|
||||
// consumes one fault, including the SDK's own per-request retries):
|
||||
// deleting the marker version fans out over the version-purge replication
|
||||
// channel (initial attempt + its fast in-memory MRF retries) plus the
|
||||
// delayed purge watcher's single pre-fix attempt — three target DELETE calls
|
||||
// in total today, empirically (see the exhaustion test's journal). Four
|
||||
// faults outlast all of them, so only a delayed-purge retry in a later
|
||||
// watch round can converge. If the SDK retry configuration ever changes,
|
||||
// re-derive this budget from a fresh journal capture.
|
||||
target.inject(
|
||||
FakeTargetOperation::DeleteObject,
|
||||
FakeTargetFault::Status(StatusCode::SERVICE_UNAVAILABLE),
|
||||
4,
|
||||
);
|
||||
|
||||
// Purge the marker at the source. The watcher spawned when the marker
|
||||
// replication completed moments ago observes the source marker vanish
|
||||
// within its watch window and drives the target purge.
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(DELAYED_PURGE_KEY)
|
||||
.version_id(&marker_version)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Tight window on purpose: a fixed delayed purge retries on 1s rounds and
|
||||
// converges within ~5s, while any straggling backoff retry from the other
|
||||
// channels would land later and must not be what turns this test green.
|
||||
wait_for_target_marker_purged(&target, target_bucket, Duration::from_secs(15)).await?;
|
||||
|
||||
let attempts = delayed_purge_attempts(&target, &marker_version);
|
||||
assert!(
|
||||
attempts.len() >= 2,
|
||||
"expected the faulted purge attempt plus at least one retry, got: {attempts:?}"
|
||||
);
|
||||
assert!(
|
||||
attempts.iter().any(|record| record.fault.is_none()),
|
||||
"expected a clean purge attempt after the fault script drained, got: {attempts:?}"
|
||||
);
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// P1-21 review follow-up: the watcher must purge the version the TARGET
|
||||
/// assigned to the replicated marker, not one derived from the source uuid.
|
||||
/// A target that mints its own version ids answers a source-derived purge
|
||||
/// with an idempotent 204, which used to look like success and strand the
|
||||
/// real marker on the target forever.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delayed_delete_marker_purge_uses_target_assigned_version() -> TestResult {
|
||||
init_logging();
|
||||
let source_bucket = "delayed-purge-mint-src";
|
||||
let target_bucket = "delayed-purge-mint-dst";
|
||||
let (target, _source_env, source_client) = start_delayed_purge_fixture(source_bucket, target_bucket).await?;
|
||||
// The target ignores the forwarded source-version-id header and mints its
|
||||
// own ids for both the object and the replicated delete marker.
|
||||
target.assign_own_version_ids(true);
|
||||
|
||||
let marker_version = replicate_delete_marker(&target, target_bucket, &source_client, source_bucket).await?;
|
||||
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(DELAYED_PURGE_KEY)
|
||||
.version_id(&marker_version)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// The replicated marker carries a target-minted version id, so nothing
|
||||
// but the recorded mapping can address it.
|
||||
wait_for_target_marker_purged(&target, target_bucket, Duration::from_secs(25)).await?;
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// P1-21: when every watch-window purge attempt fails, the purge intent must
|
||||
/// survive as a durable MRF entry and replay on the next startup; once the
|
||||
/// replayed purge succeeds, the entry must be acknowledged instead of being
|
||||
/// retained as Missed forever.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_delayed_delete_marker_purge_exhaustion_persists_to_mrf_and_replays_on_restart() -> TestResult {
|
||||
init_logging();
|
||||
let source_bucket = "delayed-purge-mrf-src";
|
||||
let target_bucket = "delayed-purge-mrf-dst";
|
||||
let (target, mut source_env, source_client) = start_delayed_purge_fixture(source_bucket, target_bucket).await?;
|
||||
|
||||
let marker_version = replicate_delete_marker(&target, target_bucket, &source_client, source_bucket).await?;
|
||||
|
||||
// Outlast the whole watch window: every in-process purge attempt fails.
|
||||
target.inject(
|
||||
FakeTargetOperation::DeleteObject,
|
||||
FakeTargetFault::Status(StatusCode::SERVICE_UNAVAILABLE),
|
||||
64,
|
||||
);
|
||||
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(DELAYED_PURGE_KEY)
|
||||
.version_id(&marker_version)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Let the watch window drain before restarting. The wall-clock length is
|
||||
// not 5x1s: every faulted attempt embeds the SDK's own per-request 503
|
||||
// retries (a few seconds each), so instead of a fixed sleep, wait until
|
||||
// the faulted attempts stop arriving (the watcher exhausted its rounds and
|
||||
// persisted the purge intent), then give the MRF persister its 100ms
|
||||
// flush interval.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
|
||||
let mut last_seen = delayed_purge_attempts(&target, &marker_version).len();
|
||||
let mut quiet_since = tokio::time::Instant::now();
|
||||
loop {
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
let seen = delayed_purge_attempts(&target, &marker_version).len();
|
||||
if seen != last_seen {
|
||||
last_seen = seen;
|
||||
quiet_since = tokio::time::Instant::now();
|
||||
}
|
||||
if last_seen > 0 && quiet_since.elapsed() >= Duration::from_secs(5) {
|
||||
break;
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!("purge attempts never quiesced (saw {last_seen}); journal: {:?}", target.requests()).into());
|
||||
}
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
|
||||
let marker_survives_faults = target
|
||||
.stored_versions(target_bucket, DELAYED_PURGE_KEY)
|
||||
.iter()
|
||||
.any(|(_, delete_marker)| *delete_marker);
|
||||
assert!(marker_survives_faults, "scripted faults must have blocked every in-process purge attempt");
|
||||
|
||||
target.clear_faults();
|
||||
let attempts_before_restart = delayed_purge_attempts(&target, &marker_version).len();
|
||||
|
||||
// Startup MRF replay must re-drive the purge and clean the target.
|
||||
source_env
|
||||
.restart_server_preserving_data(vec![], &delayed_purge_process_env())
|
||||
.await?;
|
||||
wait_for_target_marker_purged(&target, target_bucket, Duration::from_secs(30)).await?;
|
||||
let attempts_after_replay = delayed_purge_attempts(&target, &marker_version).len();
|
||||
assert!(
|
||||
attempts_after_replay > attempts_before_restart,
|
||||
"the restart replay must have issued the purge DELETE"
|
||||
);
|
||||
|
||||
// The successful replay must acknowledge the MRF entry: another restart may
|
||||
// not re-drive the purge again.
|
||||
source_env
|
||||
.restart_server_preserving_data(vec![], &delayed_purge_process_env())
|
||||
.await?;
|
||||
sleep(Duration::from_secs(5)).await;
|
||||
assert_eq!(
|
||||
delayed_purge_attempts(&target, &marker_version).len(),
|
||||
attempts_after_replay,
|
||||
"acknowledged purge-intent MRF entries must not replay again"
|
||||
);
|
||||
|
||||
target.shutdown().await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// --- P1-20 (backlog#1675): scanner existing-object compensation matrix ---
|
||||
//
|
||||
// Every case below inverts the order used by the rest of this file: objects
|
||||
// are written FIRST and the replication rule arrives afterwards, so the only
|
||||
// channel that can move the pre-existing objects is the data scanner's
|
||||
// existing-object resync pass. Negative cells ("never compensated") are
|
||||
// contracts and are asserted over multiple scanner cycles, always next to a
|
||||
// replicated control key that proves the scanner and the live path are
|
||||
// running — an absent key on a dead scanner proves nothing.
|
||||
|
||||
/// Envs + buckets only: versioning, the remote target, and the rule variant
|
||||
/// are wired by each test (the null-version case must PUT before the source
|
||||
/// bucket becomes versioned). The source runs with FAST_SCANNER_ENV so
|
||||
/// existing keys are rescanned within seconds instead of 16 dir cycles.
|
||||
async fn build_scanner_compensation_pair(
|
||||
source_bucket: &str,
|
||||
target_bucket: &str,
|
||||
) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), Box<dyn Error + Send + Sync>> {
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_process_env = replication_fast_env();
|
||||
source_process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_process_env.extend_from_slice(FAST_SCANNER_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_process_env).await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
|
||||
source_env
|
||||
.create_s3_client()
|
||||
.create_bucket()
|
||||
.bucket(source_bucket)
|
||||
.send()
|
||||
.await?;
|
||||
target_env
|
||||
.create_s3_client()
|
||||
.create_bucket()
|
||||
.bucket(target_bucket)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
Ok((source_env, target_env))
|
||||
}
|
||||
|
||||
/// P1-20: objects that already exist when a rule with
|
||||
/// ExistingObjectReplication=Enabled arrives are compensated by the scanner's
|
||||
/// existing-object resync pass, whatever wrote them — plain PUT, CopyObject,
|
||||
/// or Snowball auto-extract. The pinned exception is a null-version object
|
||||
/// (written before the bucket became versioned): the scanner heal gate skips
|
||||
/// nil-version objects entirely (`scanner_folder.rs` heal_replication), so it
|
||||
/// must NEVER be compensated.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scanner_compensates_existing_objects_across_write_paths() -> TestResult {
|
||||
init_logging();
|
||||
let source_bucket = "scanner-comp-src";
|
||||
let target_bucket = "scanner-comp-dst";
|
||||
let (source_env, target_env) = build_scanner_compensation_pair(source_bucket, target_bucket).await?;
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
// Null-version cell: PUT before versioning; the object keeps the nil
|
||||
// version id forever.
|
||||
let null_key = "pre-versioning-null.txt";
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(null_key)
|
||||
.body(ByteStream::from_static(b"null version payload"))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
|
||||
// Pre-existing objects from three write paths, all before any replication
|
||||
// config exists (their replication status stays Empty).
|
||||
let plain_key = "existing-plain.txt";
|
||||
let plain_payload = "existing plain payload";
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(plain_key)
|
||||
.body(ByteStream::from_static(plain_payload.as_bytes()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let copy_key = "existing-copy.txt";
|
||||
source_client
|
||||
.copy_object()
|
||||
.bucket(source_bucket)
|
||||
.key(copy_key)
|
||||
.copy_source(format!("{source_bucket}/{plain_key}"))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let member_key = "snowball/existing-member.txt";
|
||||
let member_payload: &[u8] = b"existing snowball member payload";
|
||||
let mut builder = tokio_tar::Builder::new(std::io::Cursor::new(Vec::new()));
|
||||
let mut header = tokio_tar::Header::new_gnu();
|
||||
header.set_size(member_payload.len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append_data(&mut header, member_key, std::io::Cursor::new(member_payload))
|
||||
.await?;
|
||||
let archive = builder.into_inner().await?.into_inner();
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("existing-members.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(archive))
|
||||
.send()
|
||||
.await?;
|
||||
// The extracted member must exist locally before the rule arrives, or it
|
||||
// would replicate through the live path instead of the scanner.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
loop {
|
||||
if source_client
|
||||
.head_object()
|
||||
.bucket(source_bucket)
|
||||
.key(member_key)
|
||||
.send()
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
break;
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err("snowball member was never extracted on the source".into());
|
||||
}
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
|
||||
// Only now wire the remote target and the Enabled rule.
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
// Control key written after the rule replicates through the live path.
|
||||
let control_key = "control-live.txt";
|
||||
let control_payload = "control live payload";
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(control_key)
|
||||
.body(ByteStream::from_static(control_payload.as_bytes()))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_replicated_object(&target_client, target_bucket, control_key, control_payload).await?;
|
||||
|
||||
// Scanner compensation for each pre-existing write path.
|
||||
wait_for_replicated_object(&target_client, target_bucket, plain_key, plain_payload).await?;
|
||||
wait_for_replicated_object(&target_client, target_bucket, copy_key, plain_payload).await?;
|
||||
wait_for_replicated_object(&target_client, target_bucket, member_key, std::str::from_utf8(member_payload)?).await?;
|
||||
|
||||
// Null-version contract: with every sibling compensated (scanner proven
|
||||
// live), the nil-version object must stay absent across further cycles.
|
||||
assert_replication_key_absent(&target_client, target_bucket, null_key, Duration::from_secs(6)).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// P1-20: ExistingObjectReplication=Disabled is a contract, not a delay — the
|
||||
/// scanner must NEVER compensate objects that predate the rule, while objects
|
||||
/// written after the rule replicate normally (the setting only gates the
|
||||
/// existing-object resync path).
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scanner_never_compensates_when_existing_object_replication_disabled() -> TestResult {
|
||||
init_logging();
|
||||
let source_bucket = "scanner-disabled-src";
|
||||
let target_bucket = "scanner-disabled-dst";
|
||||
let (source_env, mut target_env) = build_scanner_compensation_pair(source_bucket, target_bucket).await?;
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
|
||||
let existing_key = "existing-disabled.txt";
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(existing_key)
|
||||
.body(ByteStream::from_static(b"existing disabled payload"))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication_with_statuses(&source_env, source_bucket, &target_arn, "Enabled", None, "Disabled").await?;
|
||||
|
||||
// The live path is unaffected by the Disabled existing-object setting.
|
||||
let control_key = "control-live.txt";
|
||||
let control_payload = "control live payload";
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(control_key)
|
||||
.body(ByteStream::from_static(control_payload.as_bytes()))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_replicated_object(&target_client, target_bucket, control_key, control_payload).await?;
|
||||
|
||||
// Scanner-only witness. A live-path control key alone would let this test
|
||||
// pass while the existing-object scanner is disabled or wedged, so make
|
||||
// the scanner itself observable: an object whose replication FAILED while
|
||||
// the target was down can only be re-driven by the data scanner's
|
||||
// replication heal pass (see FAST_SCANNER_ENV), and that pass is NOT
|
||||
// gated by ExistingObjectReplication. The witness lives in the same
|
||||
// bucket and prefix as the pre-existing key, so a heal pass that reached
|
||||
// it necessarily walked the pre-existing key in the same scan.
|
||||
let witness_key = "scanner-witness.txt";
|
||||
let witness_payload = "scanner witness payload";
|
||||
target_env.stop_server();
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(witness_key)
|
||||
.body(ByteStream::from_static(witness_payload.as_bytes()))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_source_replication_status(&source_client, source_bucket, witness_key, "FAILED", false).await?;
|
||||
target_env.restart_server_preserving_data(vec![], &[]).await?;
|
||||
let target_client = target_env.create_s3_client();
|
||||
wait_for_replicated_object(&target_client, target_bucket, witness_key, witness_payload).await?;
|
||||
|
||||
// The scanner demonstrably swept this bucket; the pre-existing key must
|
||||
// still be absent, and stay absent over further cycles.
|
||||
assert_replication_key_absent(&target_client, target_bucket, existing_key, Duration::from_secs(6)).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -69,6 +69,7 @@ fn build_non_inline_writers(config: &BenchConfig) -> Vec<Option<BitrotWriterWrap
|
||||
fn bench_single_block_non_inline_fast_path(c: &mut Criterion) {
|
||||
let configs = vec![
|
||||
BenchConfig::new(4 * 1024, 4, 2, 128 * 1024),
|
||||
BenchConfig::new(16 * 1024, 4, 2, 128 * 1024),
|
||||
BenchConfig::new(64 * 1024, 4, 2, 128 * 1024),
|
||||
BenchConfig::new(128 * 1024, 4, 2, 128 * 1024),
|
||||
];
|
||||
@@ -112,7 +113,12 @@ fn bench_single_block_non_inline_fast_path(c: &mut Criterion) {
|
||||
rt.block_on(async {
|
||||
erasure
|
||||
.clone()
|
||||
.encode_single_block_non_inline(reader, &mut writers, config.data_shards)
|
||||
.encode_single_block_non_inline_with_size_hint(
|
||||
reader,
|
||||
&mut writers,
|
||||
config.data_shards,
|
||||
config.payload_size,
|
||||
)
|
||||
.await
|
||||
.expect("single block candidate benchmark");
|
||||
});
|
||||
|
||||
@@ -32,7 +32,7 @@ pub mod bucket {
|
||||
pub mod bucket_target_sys {
|
||||
pub use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
|
||||
TargetClient,
|
||||
TargetClient, append_version_id_query,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -281,7 +281,7 @@ pub mod config {
|
||||
pub mod com {
|
||||
pub use crate::config::com::{
|
||||
COMMA_SEPARATED_LISTS, CONFIG_PREFIX, ENV_CONFIG_RECOVER_ON_CORRUPTION, STORAGE_CLASS_SUB_SYS,
|
||||
ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config,
|
||||
ServerConfigCorruptError, ServerConfigSaveResult, ServerConfigSnapshot, delete_config, delete_config_no_lock,
|
||||
is_server_config_corrupt_error, lookup_configs, read_config, read_config_no_lock, read_config_with_metadata,
|
||||
read_config_without_migrate, read_config_without_migrate_no_lock, read_existing_server_config_no_lock,
|
||||
read_server_config_snapshot, save_config, save_config_no_lock, save_config_with_opts, save_server_config,
|
||||
@@ -308,6 +308,8 @@ pub mod config {
|
||||
}
|
||||
|
||||
pub mod data_usage {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::data_usage::seed_bucket_usage_memory_for_test;
|
||||
pub use crate::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
|
||||
@@ -409,10 +411,10 @@ pub mod object {
|
||||
pub use crate::object_api::{
|
||||
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
|
||||
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader,
|
||||
ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
|
||||
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
|
||||
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer,
|
||||
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
};
|
||||
pub use crate::store::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
|
||||
@@ -1450,7 +1450,7 @@ fn resolve_put_api_version_id(source_version_id: &str) -> Option<&str> {
|
||||
/// member, so the query is spliced in via `map_request`, which runs at
|
||||
/// `modify_before_signing`: the parameter becomes part of the SigV4 canonical
|
||||
/// request.
|
||||
fn append_version_id_query(uri: &str, version_id: &str) -> String {
|
||||
pub fn append_version_id_query(uri: &str, version_id: &str) -> String {
|
||||
let separator = if uri.contains('?') { '&' } else { '?' };
|
||||
format!("{uri}{separator}versionId={}", urlencoding::encode(version_id))
|
||||
}
|
||||
@@ -1861,6 +1861,9 @@ impl TargetClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// On success returns the version id the target assigned (from
|
||||
/// `x-amz-version-id`), letting callers audit the version-identity
|
||||
/// contract — a target that adopts the source version echoes it back.
|
||||
pub async fn put_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1868,7 +1871,7 @@ impl TargetClient {
|
||||
size: i64,
|
||||
body: ByteStream,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<(), S3ClientError> {
|
||||
) -> Result<Option<String>, S3ClientError> {
|
||||
let mut headers = opts.header();
|
||||
|
||||
let builder = self.client.put_object();
|
||||
@@ -1903,7 +1906,7 @@ impl TargetClient {
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Ok(output) => Ok(output.version_id().map(ToOwned::to_owned)),
|
||||
Err(e) => match e {
|
||||
SdkError::ServiceError(service_err) => {
|
||||
let err = service_err.into_err();
|
||||
|
||||
@@ -64,10 +64,41 @@ impl BucketDurabilityConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Default durability tier seeded into a newly created bucket's metadata
|
||||
/// (rustfs/backlog#1811). `relaxed` aligns new buckets with MinIO's default
|
||||
/// posture: object data is still fdatasynced, while xl.meta and directory-entry
|
||||
/// fsyncs follow the relaxed durability gate.
|
||||
pub const ENV_NEW_BUCKET_DURABILITY_MODE: &str = "RUSTFS_NEW_BUCKET_DURABILITY_MODE";
|
||||
pub const DEFAULT_NEW_BUCKET_DURABILITY_MODE: &str = BUCKET_DURABILITY_MODE_RELAXED;
|
||||
|
||||
/// The `durability.json` bytes to seed into a freshly created bucket's metadata.
|
||||
/// Empty means "no override" (the bucket then follows the global
|
||||
/// `RUSTFS_DURABILITY_MODE`); otherwise the serialized chosen tier. Operators
|
||||
/// can set `inherit` to disable the new-bucket override. Invalid values also
|
||||
/// fail closed to inherit the global mode instead of seeding a surprising tier.
|
||||
pub fn new_bucket_durability_config_json() -> Vec<u8> {
|
||||
let raw = std::env::var(ENV_NEW_BUCKET_DURABILITY_MODE).unwrap_or_else(|_| DEFAULT_NEW_BUCKET_DURABILITY_MODE.to_string());
|
||||
let mode = raw.trim();
|
||||
if mode.eq_ignore_ascii_case("inherit") || mode.is_empty() || !BucketDurabilityConfig::is_valid_mode(mode) {
|
||||
return Vec::new();
|
||||
}
|
||||
serde_json::to_vec(&BucketDurabilityConfig::new(mode)).expect("BucketDurabilityConfig serialization cannot fail")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn new_bucket_seeded_mode() -> Option<String> {
|
||||
let json = new_bucket_durability_config_json();
|
||||
if json.is_empty() {
|
||||
return None;
|
||||
}
|
||||
serde_json::from_slice::<BucketDurabilityConfig>(&json)
|
||||
.expect("new-bucket durability config must serialize")
|
||||
.normalized_mode()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn valid_modes_are_recognized() {
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("strict"));
|
||||
@@ -99,4 +130,33 @@ mod tests {
|
||||
let empty: BucketDurabilityConfig = serde_json::from_slice(b"{}").expect("deserialize empty");
|
||||
assert_eq!(empty.normalized_mode(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_default_seeds_relaxed_when_unset() {
|
||||
temp_env::with_var_unset(ENV_NEW_BUCKET_DURABILITY_MODE, || {
|
||||
assert_eq!(new_bucket_seeded_mode().as_deref(), Some(BUCKET_DURABILITY_MODE_RELAXED));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_default_honors_explicit_tiers() {
|
||||
for mode in [
|
||||
BUCKET_DURABILITY_MODE_STRICT,
|
||||
BUCKET_DURABILITY_MODE_RELAXED,
|
||||
BUCKET_DURABILITY_MODE_NONE,
|
||||
] {
|
||||
temp_env::with_var(ENV_NEW_BUCKET_DURABILITY_MODE, Some(mode), || {
|
||||
assert_eq!(new_bucket_seeded_mode().as_deref(), Some(mode));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_default_can_inherit_global_mode() {
|
||||
for mode in ["inherit", "", "bogus"] {
|
||||
temp_env::with_var(ENV_NEW_BUCKET_DURABILITY_MODE, Some(mode), || {
|
||||
assert_eq!(new_bucket_seeded_mode(), None);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,8 +78,8 @@ use rustfs_common::metrics::{
|
||||
};
|
||||
use rustfs_config::{
|
||||
DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_QUEUE_SEND_TIMEOUT_MS, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX,
|
||||
DEFAULT_TRANSITION_WORKERS_CAP, ENV_TRANSITION_QUEUE_CAPACITY, ENV_TRANSITION_QUEUE_SEND_TIMEOUT_MS, ENV_TRANSITION_WORKERS,
|
||||
ENV_TRANSITION_WORKERS_ABSOLUTE_MAX,
|
||||
DEFAULT_TRANSITION_WORKERS_CAP, ENV_MAX_EXPIRY_WORKERS, ENV_TRANSITION_QUEUE_CAPACITY, ENV_TRANSITION_QUEUE_SEND_TIMEOUT_MS,
|
||||
ENV_TRANSITION_WORKERS, ENV_TRANSITION_WORKERS_ABSOLUTE_MAX,
|
||||
};
|
||||
use rustfs_data_usage::TierStats;
|
||||
use rustfs_filemeta::{
|
||||
@@ -159,6 +159,8 @@ const TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL: StdDuration = StdDuration::f
|
||||
const TIER_FREE_VERSION_RECOVERY_JITTER_PERCENT: u64 = 10;
|
||||
const DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS: i64 = 5;
|
||||
const EXPIRY_WORKER_QUEUE_CAPACITY: usize = 1000;
|
||||
/// Maximum expiry workers used as a local fallback when runtime env is unset.
|
||||
const DEFAULT_EXPIRY_WORKERS_CAP: usize = 16;
|
||||
const DEFAULT_MANUAL_TRANSITION_JOB_RECOVERY_LIMIT: usize = 100;
|
||||
|
||||
// Phase 5 (backlog#939): lifecycle expiry/transition state moved into the
|
||||
@@ -204,6 +206,15 @@ fn resolve_transition_queue_send_timeout() -> StdDuration {
|
||||
)
|
||||
}
|
||||
|
||||
fn resolve_expiry_worker_count() -> usize {
|
||||
let fallback = std::cmp::min(num_cpus::get(), DEFAULT_EXPIRY_WORKERS_CAP);
|
||||
env::var(ENV_MAX_EXPIRY_WORKERS)
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.filter(|value| *value > 0)
|
||||
.unwrap_or(fallback)
|
||||
}
|
||||
|
||||
fn is_immediate_transition_source(src: &LcEventSrc) -> bool {
|
||||
matches!(
|
||||
src,
|
||||
@@ -2017,17 +2028,7 @@ fn is_slow_down(err: &Error) -> bool {
|
||||
}
|
||||
|
||||
pub async fn init_background_expiry(api: Arc<ECStore>) {
|
||||
let mut workers = get_env_usize("RUSTFS_MAX_EXPIRY_WORKERS", std::cmp::min(num_cpus::get(), 16));
|
||||
//globalILMConfig.getExpirationWorkers()
|
||||
if let Ok(env_expiration_workers) = env::var("_RUSTFS_ILM_EXPIRATION_WORKERS")
|
||||
&& let Ok(num_expirations) = env_expiration_workers.parse::<usize>()
|
||||
{
|
||||
workers = num_expirations;
|
||||
}
|
||||
|
||||
if workers == 0 {
|
||||
workers = get_env_usize("RUSTFS_DEFAULT_EXPIRY_WORKERS", 8);
|
||||
}
|
||||
let workers = resolve_expiry_worker_count();
|
||||
|
||||
ExpiryState::resize_workers(workers, api.clone()).await;
|
||||
let _ = spawn_tier_free_version_recovery_once(api.clone(), &TIER_FREE_VERSION_RECOVERY_STARTED);
|
||||
@@ -3318,6 +3319,9 @@ pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) {
|
||||
return;
|
||||
}
|
||||
};
|
||||
if configs.table_bucket_enabled {
|
||||
return;
|
||||
}
|
||||
let Some(lifecycle) = configs.lifecycle else {
|
||||
return;
|
||||
};
|
||||
@@ -3978,6 +3982,9 @@ async fn enqueue_expiry_for_existing_object_group(
|
||||
|
||||
pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
|
||||
let configs = metadata_boundary::get_expiry_configs(&api, bucket).await?;
|
||||
if configs.table_bucket_enabled {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(lc) = configs.lifecycle else {
|
||||
return Ok(());
|
||||
};
|
||||
@@ -4194,12 +4201,16 @@ pub async fn expire_transitioned_object(
|
||||
_src: &LcEventSrc,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Result<ObjectInfo, std::io::Error> {
|
||||
let publication_guard = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id)
|
||||
.await
|
||||
.ok_or_else(|| std::io::Error::other("lifecycle expiry is not allowed for this bucket"))?;
|
||||
let snapshot = lifecycle_delete_config_snapshot(&api, oi)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let (versioned, version_suspended) = snapshot.versioning_config().delete_state(&oi.name);
|
||||
let mut opts = transitioned_object_delete_opts(oi, lc_event.action, versioned, version_suspended, bucket_incarnation_id)
|
||||
.map_err(std::io::Error::other)?;
|
||||
opts.add_namespace_lock_guard(&publication_guard);
|
||||
opts.delete_replication_config_snapshot = Some(Arc::new(snapshot));
|
||||
//let tags = LcAuditEvent::new(src, lcEvent).Tags();
|
||||
if lc_event.action.delete_restored() {
|
||||
@@ -4789,6 +4800,43 @@ pub async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, o
|
||||
.await
|
||||
}
|
||||
|
||||
async fn lifecycle_expiry_publication_guard(
|
||||
api: &ECStore,
|
||||
oi: &ObjectInfo,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||
let result = async {
|
||||
let lock = api
|
||||
.new_ns_lock(&oi.bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await?;
|
||||
let guard = lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(Error::other)?;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(Error::other("table-bucket publication lock was lost before lifecycle delete admission"));
|
||||
}
|
||||
if !metadata_boundary::lifecycle_expiry_allowed(api, &oi.bucket, bucket_incarnation_id).await? {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(guard))
|
||||
}
|
||||
.await;
|
||||
match result {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_DELETE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
operation = "authorize_lifecycle_expiry",
|
||||
error = %err,
|
||||
"Lifecycle delete admission failed"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn apply_expiry_on_transitioned_object(
|
||||
api: Arc<ECStore>,
|
||||
oi: &ObjectInfo,
|
||||
@@ -4812,6 +4860,9 @@ pub async fn apply_expiry_on_non_transitioned_objects(
|
||||
_src: &LcEventSrc,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> bool {
|
||||
let Some(publication_guard) = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id).await else {
|
||||
return false;
|
||||
};
|
||||
let snapshot = match lifecycle_delete_config_snapshot(&api, oi).await {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(err) => {
|
||||
@@ -4837,6 +4888,7 @@ pub async fn apply_expiry_on_non_transitioned_objects(
|
||||
expected_bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_namespace_lock_guard(&publication_guard);
|
||||
|
||||
if lc_event.action.delete_versioned() {
|
||||
opts.version_id = oi.version_id.map(|v| v.to_string());
|
||||
@@ -5024,26 +5076,27 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc,
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX,
|
||||
DEFAULT_TRANSITION_WORKERS_CAP, EVENT_LIFECYCLE_EVALUATION_FAILED, EVENT_LIFECYCLE_EXPIRED_DETECTED,
|
||||
EVENT_LIFECYCLE_NOT_ENQUEUED, ExpiryState, ExpiryTask, FreeVersionTask, ManualTransitionJobRecoveryOutcome,
|
||||
ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, StaleMultipartUploadCandidate,
|
||||
TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL, TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL, TRANSITION_COMPLETE,
|
||||
TierFreeVersionRecoverySchedule, TransitionEnqueueOutcome, TransitionState, TransitionedObject, VersionReplicationScan,
|
||||
cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at,
|
||||
DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, DEFAULT_EXPIRY_WORKERS_CAP, DEFAULT_TRANSITION_QUEUE_CAPACITY,
|
||||
DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX, DEFAULT_TRANSITION_WORKERS_CAP, EVENT_LIFECYCLE_EVALUATION_FAILED,
|
||||
EVENT_LIFECYCLE_EXPIRED_DETECTED, EVENT_LIFECYCLE_NOT_ENQUEUED, ExpiryState, ExpiryTask, FreeVersionTask,
|
||||
ManualTransitionJobRecoveryOutcome, ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport,
|
||||
StaleMultipartUploadCandidate, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL, TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL,
|
||||
TRANSITION_COMPLETE, TierFreeVersionRecoverySchedule, TransitionEnqueueOutcome, TransitionState, TransitionedObject,
|
||||
VersionReplicationScan, cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at,
|
||||
enqueue_recovered_free_version_with_state, enqueue_transition_for_existing_objects_scoped,
|
||||
enqueue_transition_with_lifecycle, enqueue_transition_with_lifecycle_report, eval_action_from_lifecycle,
|
||||
jitter_tier_free_version_recovery_delay, lifecycle_action_blocked_by_replication,
|
||||
get_lock_acquire_timeout, jitter_tier_free_version_recovery_delay, lifecycle_action_blocked_by_replication,
|
||||
lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object, lifecycle_replication_blocks_action,
|
||||
lifecycle_rule_has_date_expiration, manual_transition_duration_elapsed, manual_transition_has_more_after_limit,
|
||||
manual_transition_recovery_progress_sink, manual_transition_version_marker, manual_transition_worker_failure_reason,
|
||||
mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate,
|
||||
persist_manual_transition_job_progress, persist_manual_transition_page_checkpoint, recover_manual_transition_job,
|
||||
recover_manual_transition_jobs, resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity,
|
||||
resolve_transition_queue_send_timeout, resolve_transition_worker_count, resolve_transition_workers_absolute_max,
|
||||
run_tier_free_version_recovery_loop, select_restore_s3_location, set_lifecycle_observability_observer,
|
||||
set_recovered_free_version_enqueue_observer, should_defer_date_expiry_for_recent_config_update,
|
||||
transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery,
|
||||
recover_manual_transition_jobs, resolve_expiry_worker_count, resolve_tier_free_version_recovery_enabled,
|
||||
resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, resolve_transition_worker_count,
|
||||
resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop, select_restore_s3_location,
|
||||
set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer,
|
||||
should_defer_date_expiry_for_recent_config_update, transitioned_cleanup_tuple, transitioned_object_delete_opts,
|
||||
wait_for_tier_free_version_recovery,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
use super::{delete_free_version_remote_object_then, encode_dir_object, get_transitioned_object_reader_with_tier_manager};
|
||||
@@ -5090,7 +5143,6 @@ mod tests {
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::warm_backend::WarmBackend as _;
|
||||
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
|
||||
@@ -5105,7 +5157,7 @@ mod tests {
|
||||
#[cfg(feature = "test-util")]
|
||||
use http::HeaderMap;
|
||||
use rustfs_common::metrics::{IlmAction, global_metrics};
|
||||
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
|
||||
use rustfs_config::{ENV_MAX_EXPIRY_WORKERS, ENV_TRANSITION_WORKERS_ABSOLUTE_MAX};
|
||||
use rustfs_data_usage::TierStats;
|
||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||
use s3s::dto::{
|
||||
@@ -7357,6 +7409,76 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
// SAFETY: this helper is only used from `#[serial]` tests and those tests run under a
|
||||
// single-thread runtime (`worker_threads = 1`), so no concurrent reader/writer can access
|
||||
// process environment while `env::set_var`/`env::remove_var` is active.
|
||||
#[allow(unsafe_code)]
|
||||
fn with_expiry_worker_env<F>(value: Option<&str>, test_fn: F)
|
||||
where
|
||||
F: FnOnce(),
|
||||
{
|
||||
let original = env::var_os(ENV_MAX_EXPIRY_WORKERS);
|
||||
|
||||
match value {
|
||||
Some(value) => unsafe {
|
||||
env::set_var(ENV_MAX_EXPIRY_WORKERS, value);
|
||||
},
|
||||
None => unsafe {
|
||||
env::remove_var(ENV_MAX_EXPIRY_WORKERS);
|
||||
},
|
||||
}
|
||||
|
||||
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(test_fn));
|
||||
|
||||
match original {
|
||||
Some(value) => unsafe {
|
||||
env::set_var(ENV_MAX_EXPIRY_WORKERS, value);
|
||||
},
|
||||
None => unsafe {
|
||||
env::remove_var(ENV_MAX_EXPIRY_WORKERS);
|
||||
},
|
||||
}
|
||||
|
||||
if let Err(e) = result {
|
||||
std::panic::resume_unwind(e);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn resolve_expiry_worker_count_uses_fallback_when_env_missing() {
|
||||
with_expiry_worker_env(None, || {
|
||||
let fallback = std::cmp::min(num_cpus::get(), DEFAULT_EXPIRY_WORKERS_CAP);
|
||||
assert_eq!(resolve_expiry_worker_count(), fallback);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn resolve_expiry_worker_count_honors_positive_env_value() {
|
||||
with_expiry_worker_env(Some("6"), || {
|
||||
assert_eq!(resolve_expiry_worker_count(), 6);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn resolve_expiry_worker_count_falls_back_for_zero_value() {
|
||||
with_expiry_worker_env(Some("0"), || {
|
||||
let fallback = std::cmp::min(num_cpus::get(), DEFAULT_EXPIRY_WORKERS_CAP);
|
||||
assert_eq!(resolve_expiry_worker_count(), fallback);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn resolve_expiry_worker_count_falls_back_for_invalid_value() {
|
||||
with_expiry_worker_env(Some("not-a-number"), || {
|
||||
let fallback = std::cmp::min(num_cpus::get(), DEFAULT_EXPIRY_WORKERS_CAP);
|
||||
assert_eq!(resolve_expiry_worker_count(), fallback);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn resolve_transition_queue_capacity_uses_default_when_env_missing() {
|
||||
@@ -10319,6 +10441,85 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn queued_lifecycle_expiry_does_not_delete_from_table_bucket() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("table-bucket-lifecycle-{}", Uuid::new_v4().simple());
|
||||
let object = "tables/table-id/data/part-00001.parquet";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
|
||||
let mut reader = PutObjReader::from_vec(b"referenced table data".to_vec());
|
||||
let object_info = ecstore
|
||||
.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("table data object should be created");
|
||||
|
||||
let publication_lock = ecstore
|
||||
.new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("table-bucket publication lock should be created");
|
||||
let enable_guard = publication_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("table-bucket enablement should acquire the publication lock");
|
||||
let expiry_store = ecstore.clone();
|
||||
let expiry_object = object_info.clone();
|
||||
let (expiry_started_tx, expiry_started_rx) = tokio::sync::oneshot::channel();
|
||||
let mut expiry = tokio::spawn(async move {
|
||||
let event = crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: IlmAction::DeleteAction,
|
||||
..Default::default()
|
||||
};
|
||||
let bucket_incarnation_id = expiry_store
|
||||
.bucket_incarnation_id_from_disk(&expiry_object.bucket)
|
||||
.await
|
||||
.expect("bucket incarnation should be available");
|
||||
expiry_started_tx.send(()).expect("lifecycle expiry start should be observed");
|
||||
super::apply_expiry_on_non_transitioned_objects(
|
||||
expiry_store,
|
||||
&expiry_object,
|
||||
&event,
|
||||
&LcEventSrc::Scanner,
|
||||
bucket_incarnation_id,
|
||||
)
|
||||
.await
|
||||
});
|
||||
expiry_started_rx.await.expect("lifecycle expiry should start");
|
||||
assert!(
|
||||
tokio::time::timeout(StdDuration::from_millis(100), &mut expiry)
|
||||
.await
|
||||
.is_err(),
|
||||
"queued lifecycle expiry must wait for table-bucket enablement"
|
||||
);
|
||||
|
||||
let sys = metadata_sys::bucket_metadata_sys_of(&ecstore.ctx).expect("metadata system should be initialized");
|
||||
let sys = sys.read().await.clone();
|
||||
let mut metadata = (*sys.get(&bucket).await.expect("bucket metadata should exist")).clone();
|
||||
metadata.table_bucket_config_json = br#"{"enabled":true}"#.to_vec();
|
||||
sys.persist_and_set(metadata)
|
||||
.await
|
||||
.expect("table bucket marker should be persisted");
|
||||
sys.reload_from_store(&bucket)
|
||||
.await
|
||||
.expect("table bucket marker should become authoritative");
|
||||
drop(enable_guard);
|
||||
assert!(
|
||||
!tokio::time::timeout(StdDuration::from_secs(2), expiry)
|
||||
.await
|
||||
.expect("queued lifecycle expiry should resume after enablement")
|
||||
.expect("queued lifecycle expiry task should join"),
|
||||
"a queued lifecycle task must be rejected after the bucket becomes table-enabled"
|
||||
);
|
||||
assert!(
|
||||
ecstore
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.is_ok(),
|
||||
"table data must remain readable after lifecycle admission rejects the delete"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_object_lifecycle_skips_current_expiration_for_explicit_legal_hold() {
|
||||
let lc = latest_expiration_lifecycle();
|
||||
|
||||
@@ -18,6 +18,7 @@ use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::bucket::metadata::BucketMetadata;
|
||||
use crate::bucket::metadata_sys::{self, ObjectLockConfigState};
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
@@ -26,16 +27,37 @@ pub(crate) struct LifecycleExpiryConfigs {
|
||||
pub(crate) lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
|
||||
pub(crate) object_lock: Option<Arc<ObjectLockConfiguration>>,
|
||||
pub(crate) bucket_incarnation_id: Uuid,
|
||||
pub(crate) table_bucket_enabled: bool,
|
||||
}
|
||||
|
||||
pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result<LifecycleExpiryConfigs> {
|
||||
let bucket_incarnation_id = api.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
async fn get_authoritative_metadata(
|
||||
api: &crate::store::ECStore,
|
||||
bucket: &str,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Result<Arc<BucketMetadata>> {
|
||||
let sys = metadata_sys::bucket_metadata_sys_of(&api.ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
let metadata = sys.get_authoritative_metadata(bucket).await?;
|
||||
if !metadata.bucket_incarnation_sidecar || metadata.bucket_incarnation_id != bucket_incarnation_id {
|
||||
return Err(Error::other(format!("bucket lifecycle metadata is not authoritative: {bucket}")));
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
pub(crate) async fn lifecycle_expiry_allowed(
|
||||
api: &crate::store::ECStore,
|
||||
bucket: &str,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Result<bool> {
|
||||
Ok(!get_authoritative_metadata(api, bucket, bucket_incarnation_id)
|
||||
.await?
|
||||
.table_bucket_enabled())
|
||||
}
|
||||
|
||||
pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result<LifecycleExpiryConfigs> {
|
||||
let bucket_incarnation_id = api.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let metadata = get_authoritative_metadata(api, bucket, bucket_incarnation_id).await?;
|
||||
let table_bucket_enabled = metadata.table_bucket_enabled();
|
||||
|
||||
let lifecycle = if metadata.lifecycle_config.is_none() && !metadata.lifecycle_config_xml.is_empty() {
|
||||
return Err(Error::other("persisted bucket lifecycle configuration is invalid"));
|
||||
@@ -51,6 +73,7 @@ pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str
|
||||
lifecycle: None,
|
||||
object_lock: None,
|
||||
bucket_incarnation_id,
|
||||
table_bucket_enabled,
|
||||
});
|
||||
}
|
||||
let object_lock = match metadata_sys::object_lock_config_state_from_authoritative_metadata(&metadata)? {
|
||||
@@ -65,6 +88,7 @@ pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str
|
||||
lifecycle,
|
||||
object_lock,
|
||||
bucket_incarnation_id,
|
||||
table_bucket_enabled,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -125,6 +149,7 @@ mod tests {
|
||||
let lifecycle = lifecycle_config();
|
||||
metadata.lifecycle_config_xml = crate::bucket::utils::serialize(&lifecycle).unwrap();
|
||||
metadata.lifecycle_config = Some(lifecycle);
|
||||
metadata.table_bucket_config_json = br#"{"enabled":true}"#.to_vec();
|
||||
metadata_sys::set_new_bucket_metadata_in(&store_a.ctx, metadata)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -132,7 +157,14 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(get_expiry_configs(&store_a, bucket).await.unwrap().lifecycle.is_some());
|
||||
let configs = get_expiry_configs(&store_a, bucket).await.unwrap();
|
||||
assert!(configs.lifecycle.is_some());
|
||||
assert!(configs.table_bucket_enabled);
|
||||
assert!(
|
||||
!lifecycle_expiry_allowed(&store_a, bucket, configs.bucket_incarnation_id)
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
assert!(get_expiry_configs(&store_b, bucket).await.unwrap().lifecycle.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -425,6 +425,15 @@ impl BucketMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
/// Metadata for a physically new user bucket. Existing or fabricated legacy
|
||||
/// metadata must use [`Self::new`] so upgrades do not rewrite their
|
||||
/// durability posture.
|
||||
pub fn new_with_default_durability(name: &str) -> Self {
|
||||
let mut metadata = Self::new(name);
|
||||
metadata.durability_config_json = super::durability::new_bucket_durability_config_json();
|
||||
metadata
|
||||
}
|
||||
|
||||
pub fn save_file_path(&self) -> String {
|
||||
format!("{}/{}/{}", BUCKET_META_PREFIX, self.name.as_str(), BUCKET_METADATA_FILE)
|
||||
}
|
||||
@@ -1302,7 +1311,7 @@ mod test {
|
||||
assert!(bm.object_locking(), "object lock active via parsed config");
|
||||
}
|
||||
|
||||
/// backlog#580: KNOWN GAP (weisd 2026-03-06 "inline_data 前缀不同"). RustFS's
|
||||
/// backlog#580: KNOWN GAP (flagged 2026-03-06: "inline_data 前缀不同"). RustFS's
|
||||
/// inline-data extraction does not yet recover the object body from a
|
||||
/// MinIO-written bucket-metadata object: `into_fileinfo(read_data=true).data`
|
||||
/// returns bytes that are not the `.metadata.bin` blob (no `format|version`
|
||||
@@ -1310,7 +1319,7 @@ mod test {
|
||||
/// inline-data framing is handled on the read path.
|
||||
/// backlog#580: prove RustFS reads a MinIO-written **inlined** bucket-metadata
|
||||
/// object end-to-end. MinIO stores inline data as `[bitrot hash][object body]`
|
||||
/// (the "`inline_data` 前缀不同" that weisd flagged on 2026-03-06 is that
|
||||
/// (the "`inline_data` 前缀不同" gap flagged on 2026-03-06 is that
|
||||
/// bitrot prefix, not a format incompatibility). Running the raw inline shard
|
||||
/// through RustFS's `BitrotReader` with the default `HighwayHash256S` must
|
||||
/// verify the checksum and yield the exact `.metadata.bin` blob.
|
||||
@@ -1378,6 +1387,43 @@ mod test {
|
||||
assert_ne!(old.bucket_incarnation_id, new.bucket_incarnation_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regular_bucket_metadata_constructor_does_not_seed_durability() {
|
||||
temp_env::with_var_unset(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, || {
|
||||
let metadata = BucketMetadata::new("legacy-or-fabricated");
|
||||
assert!(metadata.durability_config_json.is_empty());
|
||||
assert!(metadata.durability_config().is_none());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_metadata_constructor_seeds_default_durability() {
|
||||
temp_env::with_var_unset(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, || {
|
||||
let metadata = BucketMetadata::new_with_default_durability("new-user-bucket");
|
||||
assert_eq!(
|
||||
metadata.durability_config().and_then(|cfg| cfg.normalized_mode()).as_deref(),
|
||||
Some(crate::bucket::durability::BUCKET_DURABILITY_MODE_RELAXED)
|
||||
);
|
||||
|
||||
let encoded = metadata.marshal_msg().expect("marshal metadata");
|
||||
let decoded = BucketMetadata::unmarshal(&encoded).expect("unmarshal metadata");
|
||||
assert_eq!(decoded.durability_config_json, metadata.durability_config_json);
|
||||
assert_eq!(
|
||||
decoded.durability_config().and_then(|cfg| cfg.normalized_mode()).as_deref(),
|
||||
Some(crate::bucket::durability::BUCKET_DURABILITY_MODE_RELAXED)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_metadata_constructor_can_inherit_global_durability() {
|
||||
temp_env::with_var(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, Some("inherit"), || {
|
||||
let metadata = BucketMetadata::new_with_default_durability("strict-fleet-new-bucket");
|
||||
assert!(metadata.durability_config_json.is_empty());
|
||||
assert!(metadata.durability_config().is_none());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn site_replication_config_updates_cannot_replace_bucket_incarnation() {
|
||||
let mut metadata = BucketMetadata::new("site-replication-update");
|
||||
|
||||
@@ -288,6 +288,13 @@ pub(crate) fn bucket_metadata_sys_of(ctx: &crate::runtime::instance::InstanceCon
|
||||
get_bucket_metadata_sys()
|
||||
}
|
||||
|
||||
pub(crate) fn require_bucket_metadata_sys_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
) -> Result<Arc<RwLock<BucketMetadataSys>>> {
|
||||
ctx.bucket_metadata_sys()
|
||||
.ok_or_else(|| Error::other("bucket metadata sys not initialized for this instance"))
|
||||
}
|
||||
|
||||
pub(crate) async fn object_store_in(ctx: &crate::runtime::instance::InstanceContext) -> Result<Arc<ECStore>> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
Ok(sys.read().await.api.clone())
|
||||
@@ -376,6 +383,15 @@ pub async fn update(bucket: &str, config_file: &str, data: Vec<u8>) -> Result<Of
|
||||
Box::pin(update_with_sys(get_bucket_metadata_sys()?, bucket, config_file, data)).await
|
||||
}
|
||||
|
||||
pub(crate) async fn update_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
Box::pin(update_with_sys(require_bucket_metadata_sys_in(ctx)?, bucket, config_file, data)).await
|
||||
}
|
||||
|
||||
pub async fn delete(bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
|
||||
delete_with_sys(get_bucket_metadata_sys()?, bucket, config_file).await
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
use super::metadata_sys::get_bucket_metadata_sys;
|
||||
use crate::error::{Result, StorageError};
|
||||
use crate::store::ECStore;
|
||||
use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
|
||||
|
||||
pub struct PolicySys {}
|
||||
@@ -27,6 +28,10 @@ impl PolicySys {
|
||||
Self::is_allowed_with_policy(args, Self::get(args.bucket).await).await
|
||||
}
|
||||
|
||||
pub async fn try_is_allowed_for_store(store: &ECStore, args: &BucketPolicyArgs<'_>) -> Result<bool> {
|
||||
Self::is_allowed_with_policy(args, store.get_bucket_policy(args.bucket).await.map(|(policy, _)| policy)).await
|
||||
}
|
||||
|
||||
async fn is_allowed_with_policy(args: &BucketPolicyArgs<'_>, policy: Result<BucketPolicy>) -> Result<bool> {
|
||||
match policy {
|
||||
Ok(policy) => Ok(policy.is_allowed(args).await),
|
||||
|
||||
@@ -2567,6 +2567,12 @@ pub trait ReplicationPoolTrait: std::fmt::Debug {
|
||||
async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission;
|
||||
async fn queue_replica_delete_task(&self, ri: DeletedObjectReplicationInfo) -> ReplicationQueueAdmission;
|
||||
async fn queue_replica_delete_batch(&self, deletes: &[DeletedObjectReplicationInfo]) -> ReplicationBatchAdmission;
|
||||
/// Persist one entry straight to the durable MRF journal, bypassing the
|
||||
/// live worker queues. For failures whose source state is already gone —
|
||||
/// e.g. exhausted delete-marker purges — where only a startup replay can
|
||||
/// retry, and live re-dispatch would loop unboundedly against a down
|
||||
/// target.
|
||||
async fn persist_mrf_entry(&self, entry: MrfReplicateEntry) -> ReplicationQueueAdmission;
|
||||
async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize);
|
||||
async fn get_bucket_resync_status(&self, bucket: &str) -> Result<BucketReplicationResyncStatus, EcstoreError>;
|
||||
async fn cancel_bucket_resync(&self, opts: ResyncOpts) -> Result<(), EcstoreError>;
|
||||
@@ -2607,6 +2613,10 @@ impl<S: ReplicationStorage> ReplicationPoolTrait for ReplicationPool<S> {
|
||||
self.queue_replica_delete_batch(deletes).await
|
||||
}
|
||||
|
||||
async fn persist_mrf_entry(&self, entry: MrfReplicateEntry) -> ReplicationQueueAdmission {
|
||||
self.queue_mrf_save_admission(entry, "delete_marker_purge").await
|
||||
}
|
||||
|
||||
async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize) {
|
||||
self.resize(priority, max_workers, max_l_workers).await;
|
||||
}
|
||||
|
||||
@@ -18,9 +18,10 @@ use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Result, is_err_object_not_found, is_err_version_not_found};
|
||||
use super::replication_event_sink::{EventArgs, send_event, send_local_event};
|
||||
use super::replication_filemeta_boundary::{
|
||||
NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedInfos,
|
||||
ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType, ReplicationType, VersionPurgeStatusType,
|
||||
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
MrfReplicateEntry, NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo,
|
||||
ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType, ReplicationType,
|
||||
ReplicationWorkerOperation, VersionPurgeStatusType, get_replication_state, parse_replicate_decision,
|
||||
replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
};
|
||||
use super::replication_lock_boundary::ReplicationLockTiming;
|
||||
use super::replication_logging::{EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION_RESYNC};
|
||||
@@ -33,7 +34,7 @@ use super::replication_object_decision_boundary::{
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, should_retry_delete_marker_purge,
|
||||
};
|
||||
use super::replication_queue_boundary::DeletedObjectReplicationInfo;
|
||||
use super::replication_queue_boundary::{DeletedObjectReplicationInfo, ReplicationQueueAdmission};
|
||||
use super::replication_resync_boundary::ResyncStatusType;
|
||||
use super::replication_resync_boundary::{
|
||||
BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus, encode_resync_file, is_version_id_mismatch,
|
||||
@@ -63,6 +64,7 @@ use futures::stream::StreamExt;
|
||||
use http::HeaderMap;
|
||||
use http_body::Frame;
|
||||
use http_body_util::StreamBody;
|
||||
use metrics::counter;
|
||||
#[cfg(test)]
|
||||
use rmp_serde;
|
||||
use rustfs_s3_types::EventName;
|
||||
@@ -72,10 +74,10 @@ use rustfs_utils::http::{
|
||||
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
|
||||
#[cfg(test)]
|
||||
use s3s::dto::ReplicationConfiguration;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, LazyLock, Mutex as StdMutex};
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
use tokio::io::AsyncRead;
|
||||
@@ -100,6 +102,10 @@ const EVENT_REPLICATION_FORCE_DELETE_SKIPPED: &str = "replication_force_delete_s
|
||||
const EVENT_RESYNC_TASK_FAILED: &str = "replication_resync_task_failed";
|
||||
const EVENT_RESYNC_TARGET_OPERATION_FAILED: &str = "replication_resync_target_operation_failed";
|
||||
const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_channel_failed";
|
||||
const EVENT_DELETE_MARKER_PURGE_FAILED: &str = "replication_delete_marker_purge_failed";
|
||||
const EVENT_DELETE_MARKER_PURGE_MRF: &str = "replication_delete_marker_purge_mrf";
|
||||
const METRIC_DELETE_MARKER_PURGE_TOTAL: &str = "rustfs_replication_delete_marker_purge_total";
|
||||
const EVENT_REPLICATION_VERSION_IDENTITY_DRIFT: &str = "replication_version_identity_drift";
|
||||
const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
|
||||
"dispatch failure",
|
||||
"timeouterror",
|
||||
@@ -181,6 +187,57 @@ fn is_head_proxy_failure(err: &SdkError<HeadObjectError>) -> bool {
|
||||
should_count_head_proxy_failure(is_not_found, code, raw_status)
|
||||
}
|
||||
|
||||
const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_identity_drift_total";
|
||||
|
||||
/// Targets that already produced a version-identity-drift warning this
|
||||
/// process lifetime, by ARN. Deduping is advisory only (the metric still
|
||||
/// counts every drifting PUT), so a reconfigured target re-warning only
|
||||
/// after a restart is acceptable.
|
||||
static VERSION_IDENTITY_WARNED_ARNS: LazyLock<StdMutex<HashSet<String>>> = LazyLock::new(|| StdMutex::new(HashSet::new()));
|
||||
|
||||
/// Runtime half of the P1-19 version-identity contract (the explicit probe
|
||||
/// lives in replication-check's VersionFidelity phase): every replication PUT
|
||||
/// response reveals whether the target adopted the source version id. A
|
||||
/// target minting its own ids silently breaks version-addressed deletes and
|
||||
/// heal, so surface it — once per target — instead of letting the divergence
|
||||
/// accumulate unseen.
|
||||
/// Pure drift judgment: the contract only applies when the source addressed a
|
||||
/// real (non-nil) version uuid, and drift means the target answered with
|
||||
/// anything else — including nothing at all.
|
||||
fn version_identity_drifted(source_version_id: &str, assigned_version_id: Option<&str>) -> bool {
|
||||
if source_version_id.is_empty() {
|
||||
return false;
|
||||
}
|
||||
// A nil source uuid travels as the literal "null" (unversioned-source
|
||||
// semantics); no identity contract applies to it.
|
||||
if Uuid::parse_str(source_version_id).map(|uuid| uuid.is_nil()).unwrap_or(true) {
|
||||
return false;
|
||||
}
|
||||
assigned_version_id != Some(source_version_id)
|
||||
}
|
||||
|
||||
fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &str, assigned_version_id: Option<&str>) {
|
||||
if !version_identity_drifted(source_version_id, assigned_version_id) {
|
||||
return;
|
||||
}
|
||||
counter!(METRIC_VERSION_IDENTITY_DRIFT_TOTAL).increment(1);
|
||||
let mut warned = VERSION_IDENTITY_WARNED_ARNS
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if warned.insert(tgt_client.arn.clone()) {
|
||||
warn!(
|
||||
event = EVENT_REPLICATION_VERSION_IDENTITY_DRIFT,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
arn = %tgt_client.arn,
|
||||
endpoint = %tgt_client.endpoint,
|
||||
sent_version_id = %source_version_id,
|
||||
assigned_version_id = assigned_version_id.unwrap_or("<none>"),
|
||||
"Replication target does not adopt source version ids; version-addressed replication cannot converge (run ?replication-check for details)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_proxy_request(bucket: &str, api: &str, is_err: bool) {
|
||||
if let Some(stats) = runtime_sources::replication_stats() {
|
||||
stats.inc_proxy(bucket, api, is_err).await;
|
||||
@@ -1271,7 +1328,12 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
reason = "source_version_missing",
|
||||
"Skipping stale delete-marker replication"
|
||||
);
|
||||
return true;
|
||||
// The marker is gone at the source, but a replica of it may
|
||||
// already exist on the targets (a live race, or an MRF
|
||||
// purge-intent replay landing here on purpose). Purge instead
|
||||
// of just skipping; the result decides whether an MRF replay
|
||||
// may acknowledge the entry.
|
||||
return purge_stale_delete_marker_targets(&bucket, &dobj).await;
|
||||
}
|
||||
Err(err) => {
|
||||
source_state_verified = false;
|
||||
@@ -1485,29 +1547,6 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
let is_version_purge = is_version_delete_replication(&dobj.delete_object);
|
||||
|
||||
let requires_delayed_purge = should_retry_delete_marker_purge(&dobj.delete_object);
|
||||
if requires_delayed_purge {
|
||||
let bucket_clone = bucket.clone();
|
||||
let dobj_clone = dobj.clone();
|
||||
let dsc_clone = dsc.clone();
|
||||
let storage_clone = storage.clone();
|
||||
tokio::spawn(async move {
|
||||
for _ in 0..5 {
|
||||
if let Some(delete_marker_version_id) = dobj_clone.delete_object.delete_marker_version_id
|
||||
&& source_delete_marker_missing(
|
||||
&*storage_clone,
|
||||
&bucket_clone,
|
||||
&dobj_clone.delete_object.object_name,
|
||||
delete_marker_version_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
replicate_delete_marker_purge_to_targets(&bucket_clone, &dobj_clone, &dsc_clone).await;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(TokioDuration::from_secs(1)).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let (replication_status, prev_status) = if !is_version_purge {
|
||||
(
|
||||
@@ -1550,6 +1589,24 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
drs.replication_timestamp = Some(OffsetDateTime::now_utc());
|
||||
}
|
||||
|
||||
if requires_delayed_purge {
|
||||
// Hand the watcher the MERGED replication state: `drs` folds this
|
||||
// round's per-target results into the previous state, including the
|
||||
// version ids the targets assigned to the markers they just created.
|
||||
// Spawning with the pre-merge `dobj` made the purge fall back to a
|
||||
// source-derived id, which a target that mints its own ids answers
|
||||
// with an idempotent 204 — the intent was then dropped while the
|
||||
// real marker stayed behind.
|
||||
let bucket_clone = bucket.clone();
|
||||
let mut dobj_clone = dobj.clone();
|
||||
dobj_clone.delete_object.replication_state = Some(drs.clone());
|
||||
let dsc_clone = dsc.clone();
|
||||
let storage_clone = storage.clone();
|
||||
tokio::spawn(async move {
|
||||
watch_and_purge_source_delete_marker(bucket_clone, dobj_clone, dsc_clone, storage_clone).await;
|
||||
});
|
||||
}
|
||||
|
||||
let event_name = if replication_status == ReplicationStatusType::Completed {
|
||||
EventName::ObjectReplicationComplete.to_string()
|
||||
} else {
|
||||
@@ -1608,12 +1665,36 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
}
|
||||
};
|
||||
|
||||
replicate_delete_outcome(
|
||||
expected_targets,
|
||||
rinfos.targets.len(),
|
||||
state_persisted,
|
||||
source_state_verified,
|
||||
&replication_status,
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether a delete replication fully succeeded — the MRF replay acknowledges
|
||||
/// (drops) an entry exactly when this returns true.
|
||||
///
|
||||
/// The delayed purge is deliberately NOT an input: holding the outcome hostage
|
||||
/// to it (`&& !requires_delayed_purge`) forced `false` for every delete-marker
|
||||
/// entry and retained them all in the durable MRF journal forever. Purge
|
||||
/// failures persist their own purge-intent entry instead
|
||||
/// (`watch_and_purge_source_delete_marker`), and replays of those entries
|
||||
/// report purge success through `purge_stale_delete_marker_targets`.
|
||||
fn replicate_delete_outcome(
|
||||
expected_targets: usize,
|
||||
replicated_targets: usize,
|
||||
state_persisted: bool,
|
||||
source_state_verified: bool,
|
||||
replication_status: &ReplicationStatusType,
|
||||
) -> bool {
|
||||
expected_targets > 0
|
||||
&& rinfos.targets.len() == expected_targets
|
||||
&& replicated_targets == expected_targets
|
||||
&& state_persisted
|
||||
&& source_state_verified
|
||||
&& !requires_delayed_purge
|
||||
&& replication_status == ReplicationStatusType::Completed
|
||||
&& *replication_status == ReplicationStatusType::Completed
|
||||
}
|
||||
|
||||
async fn source_delete_marker_missing<S: EcstoreObjectOperations>(
|
||||
@@ -1663,48 +1744,286 @@ fn delete_marker_purge_version_id(
|
||||
})
|
||||
}
|
||||
|
||||
async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedObjectReplicationInfo, dsc: &ReplicateDecision) {
|
||||
/// One purge pass over the eligible targets. Returns the ARNs that must be
|
||||
/// retried: the remote DELETE failed, or the target client was unavailable
|
||||
/// (e.g. a runtime cache miss). Inconsistent recorded version mappings are a
|
||||
/// deliberate refusal — retrying cannot make guessing a version id safe — so
|
||||
/// they are logged and excluded from the retry set.
|
||||
async fn replicate_delete_marker_purge_to_targets(
|
||||
bucket: &str,
|
||||
dobj: &DeletedObjectReplicationInfo,
|
||||
dsc: &ReplicateDecision,
|
||||
retry_arns: Option<&[String]>,
|
||||
) -> Vec<String> {
|
||||
let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id else {
|
||||
return;
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let target_arns = dobj.admitted_target_arns();
|
||||
let mut failed_arns = Vec::new();
|
||||
for tgt_entry in dsc.targets_map.values() {
|
||||
if !tgt_entry.replicate {
|
||||
continue;
|
||||
}
|
||||
let target_arns = dobj.admitted_target_arns();
|
||||
if !target_arns.is_empty() && !target_arns.iter().any(|arn| arn == &tgt_entry.arn) {
|
||||
continue;
|
||||
}
|
||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &tgt_entry.arn).await else {
|
||||
if let Some(retry_arns) = retry_arns
|
||||
&& !retry_arns.iter().any(|arn| arn == &tgt_entry.arn)
|
||||
{
|
||||
continue;
|
||||
};
|
||||
|
||||
}
|
||||
// Decide the version first: refusing to guess is a per-target
|
||||
// FAILURE, not a silent skip. Reporting it as success would let the
|
||||
// watcher and the MRF replay drop the purge intent while the marker
|
||||
// is still on the target — the leak stays visible instead (the
|
||||
// entry is retained and keeps warning) until an operator repairs
|
||||
// the metadata.
|
||||
let Some(purge_version_id) = delete_marker_purge_version_id(
|
||||
dobj.delete_object.replication_state.as_ref(),
|
||||
&tgt_entry.arn,
|
||||
delete_marker_version_id,
|
||||
) else {
|
||||
warn!(
|
||||
event = EVENT_DELETE_MARKER_PURGE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
arn = tgt_entry.arn,
|
||||
"Skipping delete-marker purge: recorded target version metadata is inconsistent"
|
||||
reason = "recorded_target_version_inconsistent",
|
||||
"Delete-marker purge refused: recorded target version metadata is inconsistent"
|
||||
);
|
||||
counter!(METRIC_DELETE_MARKER_PURGE_TOTAL, "state" => "refused").increment(1);
|
||||
failed_arns.push(tgt_entry.arn.clone());
|
||||
continue;
|
||||
};
|
||||
|
||||
let _ = tgt_client
|
||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &tgt_entry.arn).await else {
|
||||
warn!(
|
||||
event = EVENT_DELETE_MARKER_PURGE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
arn = tgt_entry.arn,
|
||||
reason = "target_client_missing",
|
||||
"Delete-marker purge attempt failed"
|
||||
);
|
||||
counter!(METRIC_DELETE_MARKER_PURGE_TOTAL, "state" => "failed").increment(1);
|
||||
failed_arns.push(tgt_entry.arn.clone());
|
||||
continue;
|
||||
};
|
||||
|
||||
match tgt_client
|
||||
.remove_object(
|
||||
&tgt_client.bucket,
|
||||
&dobj.delete_object.object_name,
|
||||
purge_version_id,
|
||||
replication_delete_marker_purge_remove_options(dobj.delete_object.delete_marker_mtime),
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
counter!(METRIC_DELETE_MARKER_PURGE_TOTAL, "state" => "purged").increment(1);
|
||||
}
|
||||
// The marker version is already gone on the target: the purge goal
|
||||
// is met. Strict S3 targets 404 here (RustFS/MinIO answer 204);
|
||||
// treating it as a failure would retain the intent entry forever.
|
||||
Err(error) if matches!(error.code.as_deref(), Some("NoSuchKey" | "NoSuchVersion")) => {
|
||||
counter!(METRIC_DELETE_MARKER_PURGE_TOTAL, "state" => "purged").increment(1);
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(
|
||||
event = EVENT_DELETE_MARKER_PURGE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
arn = tgt_entry.arn,
|
||||
error = %error,
|
||||
reason = "target_delete_failed",
|
||||
"Delete-marker purge attempt failed"
|
||||
);
|
||||
counter!(METRIC_DELETE_MARKER_PURGE_TOTAL, "state" => "failed").increment(1);
|
||||
mark_replication_target_offline_if_needed(&tgt_client, &error).await;
|
||||
failed_arns.push(tgt_entry.arn.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
failed_arns
|
||||
}
|
||||
|
||||
const DELETE_MARKER_PURGE_WATCH_ROUNDS: usize = 5;
|
||||
const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_secs(1);
|
||||
|
||||
/// Watch the source delete marker for a short window after its replication.
|
||||
///
|
||||
/// KNOWN NON-DURABLE WINDOW: this task is detached, so a process exit inside
|
||||
/// the watch window loses an intent that has not been persisted yet. The
|
||||
/// window predates this code (the previous implementation had no durable
|
||||
/// channel at all, and no replay half either), so nothing regresses — closing
|
||||
/// it needs a write-ahead intent recorded before the parent delete is
|
||||
/// acknowledged, which is tracked as follow-up rather than done here: every
|
||||
/// delete-marker replication would pay a journal write for a purge that
|
||||
/// almost never happens.
|
||||
///
|
||||
/// If the marker disappears (deleted before or while the replica landed),
|
||||
/// purge the replicated marker from the targets, retrying failed targets on
|
||||
/// later rounds. When the window drains with targets still dirty, persist the
|
||||
/// purge intent as a durable MRF entry so the next startup replays it through
|
||||
/// `purge_stale_delete_marker_targets`.
|
||||
async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
|
||||
bucket: String,
|
||||
dobj: DeletedObjectReplicationInfo,
|
||||
dsc: ReplicateDecision,
|
||||
storage: Arc<S>,
|
||||
) {
|
||||
let Some(delete_marker_version_id) = dobj.delete_object.delete_marker_version_id else {
|
||||
return;
|
||||
};
|
||||
|
||||
// `pending` is None until the source marker is observed missing; after the
|
||||
// first purge pass it holds the targets that still need a successful purge.
|
||||
let mut pending: Option<Vec<String>> = None;
|
||||
for round in 0..DELETE_MARKER_PURGE_WATCH_ROUNDS {
|
||||
pending = match pending.take() {
|
||||
None => {
|
||||
if source_delete_marker_missing(&*storage, &bucket, &dobj.delete_object.object_name, delete_marker_version_id)
|
||||
.await
|
||||
{
|
||||
Some(replicate_delete_marker_purge_to_targets(&bucket, &dobj, &dsc, None).await)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Some(failed_arns) => Some(replicate_delete_marker_purge_to_targets(&bucket, &dobj, &dsc, Some(&failed_arns)).await),
|
||||
};
|
||||
if matches!(pending.as_deref(), Some([])) {
|
||||
return;
|
||||
}
|
||||
if round + 1 < DELETE_MARKER_PURGE_WATCH_ROUNDS {
|
||||
tokio::time::sleep(DELETE_MARKER_PURGE_WATCH_INTERVAL).await;
|
||||
}
|
||||
}
|
||||
if let Some(failed_arns) = pending.filter(|failed_arns| !failed_arns.is_empty()) {
|
||||
enqueue_delete_marker_purge_mrf(&dobj, failed_arns).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Shape an exhausted purge intent as a marker-creation delete entry. Replay
|
||||
/// reconstructs it with `delete_marker: true`, finds the source marker gone,
|
||||
/// and funnels into the stale-marker branch of `replicate_delete_with_outcome`
|
||||
/// — which re-runs the purge without touching source state and reports purge
|
||||
/// success as the replay outcome.
|
||||
fn delete_marker_purge_mrf_entry(dobj: &DeletedObjectReplicationInfo, failed_arns: Vec<String>) -> MrfReplicateEntry {
|
||||
let mut entry = dobj.to_mrf_entry();
|
||||
entry.delete_marker = true;
|
||||
entry.version_id = None;
|
||||
entry.retry_count = 0;
|
||||
entry.target_arns = failed_arns;
|
||||
entry
|
||||
}
|
||||
|
||||
async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, failed_arns: Vec<String>) {
|
||||
let arns = failed_arns.join(",");
|
||||
let miss_reason = match runtime_sources::replication_pool() {
|
||||
None => Some("replication_pool_unavailable"),
|
||||
Some(pool) => match pool.persist_mrf_entry(delete_marker_purge_mrf_entry(dobj, failed_arns)).await {
|
||||
ReplicationQueueAdmission::Queued => None,
|
||||
_ => Some("mrf_save_unavailable"),
|
||||
},
|
||||
};
|
||||
match miss_reason {
|
||||
None => {
|
||||
warn!(
|
||||
event = EVENT_DELETE_MARKER_PURGE_MRF,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = dobj.bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
arns,
|
||||
state = "queued",
|
||||
"Delete-marker purge exhausted its watch window; intent persisted to the MRF journal"
|
||||
);
|
||||
counter!(METRIC_DELETE_MARKER_PURGE_TOTAL, "state" => "mrf_queued").increment(1);
|
||||
}
|
||||
Some(reason) => {
|
||||
warn!(
|
||||
event = EVENT_DELETE_MARKER_PURGE_MRF,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = dobj.bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
arns,
|
||||
state = "missed",
|
||||
reason,
|
||||
"Delete-marker purge intent could not be persisted for retry"
|
||||
);
|
||||
counter!(METRIC_DELETE_MARKER_PURGE_TOTAL, "state" => "mrf_missed").increment(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The marker vanished at the source while its replication was still pending
|
||||
/// (a live race), or this is an MRF purge-intent replay. Any marker already
|
||||
/// replicated to a target must still be purged; run bounded retry passes and
|
||||
/// report the result so an MRF replay only acknowledges the entry once every
|
||||
/// target is clean. Live callers persist a fresh purge intent on failure;
|
||||
/// replay callers (`ReplicationType::Heal`) rely on Missed retention instead,
|
||||
/// so the journal does not accumulate duplicate entries.
|
||||
///
|
||||
/// Heal callers retry for the full watch window because the startup MRF
|
||||
/// processor runs before bucket metadata (and thus target clients) finishes
|
||||
/// initializing — the first pass can see `target_client_missing` and a later
|
||||
/// round resolves the client; the replay loop is serial and startup-only, so
|
||||
/// blocking it for up to the window per dirty entry is acceptable. Live
|
||||
/// callers run on replication workers where a down target would pin a worker
|
||||
/// for the whole window, so they attempt once and lean on the durable intent
|
||||
/// entry instead.
|
||||
async fn purge_stale_delete_marker_targets(bucket: &str, dobj: &DeletedObjectReplicationInfo) -> bool {
|
||||
let decision_str = dobj
|
||||
.delete_object
|
||||
.replication_state
|
||||
.as_ref()
|
||||
.map(|state| state.replicate_decision_str.clone())
|
||||
.unwrap_or_default();
|
||||
let dsc = match parse_replicate_decision(bucket, &decision_str) {
|
||||
Ok(dsc) => dsc,
|
||||
Err(error) => {
|
||||
warn!(
|
||||
event = EVENT_DELETE_MARKER_PURGE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
error = %error,
|
||||
reason = "replicate_decision_parse_failed",
|
||||
"Delete-marker purge attempt failed"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let rounds = if dobj.op_type == ReplicationType::Heal {
|
||||
DELETE_MARKER_PURGE_WATCH_ROUNDS
|
||||
} else {
|
||||
1
|
||||
};
|
||||
let mut failed_arns = replicate_delete_marker_purge_to_targets(bucket, dobj, &dsc, None).await;
|
||||
for _ in 1..rounds {
|
||||
if failed_arns.is_empty() {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(DELETE_MARKER_PURGE_WATCH_INTERVAL).await;
|
||||
failed_arns = replicate_delete_marker_purge_to_targets(bucket, dobj, &dsc, Some(&failed_arns)).await;
|
||||
}
|
||||
if failed_arns.is_empty() {
|
||||
return true;
|
||||
}
|
||||
if dobj.op_type != ReplicationType::Heal {
|
||||
enqueue_delete_marker_purge_mrf(dobj, failed_arns).await;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) -> bool {
|
||||
@@ -2605,6 +2924,13 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let result = tgt_client
|
||||
.put_object(&tgt_client.bucket, &object, transfer_size, byte_stream, &put_opts)
|
||||
.await
|
||||
.map(|assigned_version_id| {
|
||||
audit_target_version_identity(
|
||||
&tgt_client,
|
||||
&put_opts.internal.source_version_id,
|
||||
assigned_version_id.as_deref(),
|
||||
)
|
||||
})
|
||||
.map_err(|e| std::io::Error::other(e.to_string()));
|
||||
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
|
||||
if has_tagging_replication {
|
||||
@@ -3012,6 +3338,13 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let result = tgt_client
|
||||
.put_object(&tgt_client.bucket, &object, transfer_size, byte_stream, &put_opts)
|
||||
.await
|
||||
.map(|assigned_version_id| {
|
||||
audit_target_version_identity(
|
||||
&tgt_client,
|
||||
&put_opts.internal.source_version_id,
|
||||
assigned_version_id.as_deref(),
|
||||
)
|
||||
})
|
||||
.map_err(|e| std::io::Error::other(e.to_string()));
|
||||
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
|
||||
if has_tagging_replication {
|
||||
@@ -3203,21 +3536,34 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
|
||||
|
||||
let actual_size = replication_multipart_complete_actual_size(&object_info.user_defined);
|
||||
|
||||
cli.complete_multipart_upload(
|
||||
dst_bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
uploaded_parts,
|
||||
&replication_complete_multipart_options(actual_size, object_info.etag.clone().unwrap_or_default(), object_info.mod_time),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
let completed = cli
|
||||
.complete_multipart_upload(
|
||||
dst_bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
uploaded_parts,
|
||||
&replication_complete_multipart_options(
|
||||
actual_size,
|
||||
object_info.etag.clone().unwrap_or_default(),
|
||||
object_info.mod_time,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
|
||||
// Multipart decides the target version at initiate time and only reveals
|
||||
// it on completion, so this is where the identity contract is observable
|
||||
// for this path. A target can mirror PutObject version ids and still mint
|
||||
// its own here, which would leave multipart deletes and heals addressing
|
||||
// a version that never existed.
|
||||
audit_target_version_identity(&cli, &put_opts.internal.source_version_id, completed.version_id());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::replication_filemeta_boundary::ReplicateTargetDecision;
|
||||
use super::super::replication_target_boundary::{BucketTarget, BucketTargets};
|
||||
use super::*;
|
||||
use s3s::dto::{
|
||||
@@ -3257,6 +3603,27 @@ mod tests {
|
||||
ReplicationTargetStore::register_test_target(target).await;
|
||||
}
|
||||
|
||||
/// P1-19 runtime spot-check exemption matrix: drift only applies when the
|
||||
/// source addressed a real version uuid.
|
||||
#[test]
|
||||
fn test_version_identity_drift_judgment() {
|
||||
let source = "6fa459ea-ee8a-3ca4-894e-db77e160355e";
|
||||
for (sent, got, expected) in [
|
||||
(source, Some(source), false),
|
||||
(source, Some("0e304ce5-33e9-4b8a-9b12-9e40a53e6ded"), true),
|
||||
(source, None, true),
|
||||
("", None, false),
|
||||
("null", Some("anything"), false),
|
||||
("00000000-0000-0000-0000-000000000000", Some("anything"), false),
|
||||
] {
|
||||
assert_eq!(
|
||||
version_identity_drifted(sent, got),
|
||||
expected,
|
||||
"sent {sent:?} got {got:?} must judge drift = {expected}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resync_admission_configuration_is_bounded() {
|
||||
assert_eq!(ENV_REPL_RESYNC_MAX_JOBS, "RUSTFS_REPL_RESYNC_MAX_JOBS");
|
||||
@@ -3581,6 +3948,101 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// P1-21 regression guard for the outcome formula. A fully successful
|
||||
/// delete-marker replication must acknowledge its MRF entry: the formula
|
||||
/// once carried `&& !requires_delayed_purge`, which pinned every
|
||||
/// delete-marker entry to Missed and retained the whole backlog forever.
|
||||
/// (Deterministically staging a marker-creation entry in the durable
|
||||
/// journal from e2e would require saturating the worker queues, so the
|
||||
/// formula is pinned here instead; the purge-intent replay half is pinned
|
||||
/// by the delayed-purge e2e pair.)
|
||||
#[test]
|
||||
fn test_replicate_delete_outcome_is_not_held_hostage_by_the_delayed_purge() {
|
||||
assert!(
|
||||
replicate_delete_outcome(1, 1, true, true, &ReplicationStatusType::Completed),
|
||||
"a completed delete-marker replication must be acknowledgeable even though a delayed purge watch is pending"
|
||||
);
|
||||
assert!(!replicate_delete_outcome(0, 0, true, true, &ReplicationStatusType::Completed));
|
||||
assert!(!replicate_delete_outcome(2, 1, true, true, &ReplicationStatusType::Completed));
|
||||
assert!(!replicate_delete_outcome(1, 1, false, true, &ReplicationStatusType::Completed));
|
||||
assert!(!replicate_delete_outcome(1, 1, true, false, &ReplicationStatusType::Completed));
|
||||
assert!(!replicate_delete_outcome(1, 1, true, true, &ReplicationStatusType::Failed));
|
||||
}
|
||||
|
||||
/// P1-21 review follow-up: a target whose recorded marker version is
|
||||
/// inconsistent must be reported as a per-target FAILURE. Treating the
|
||||
/// refusal as success let the watcher and the MRF replay drop the purge
|
||||
/// intent while the marker was still on the target.
|
||||
#[tokio::test]
|
||||
async fn test_delete_marker_purge_reports_corrupt_recorded_version_as_failure() {
|
||||
let arn = format!("arn:rustfs:replication:us-east-1:corrupt:{}", Uuid::new_v4());
|
||||
let mut dsc = ReplicateDecision::new();
|
||||
dsc.set(ReplicateTargetDecision::new(arn.clone(), true, false));
|
||||
|
||||
let mut state = ReplicationState {
|
||||
target_delete_marker_version_ids_corrupt: true,
|
||||
..Default::default()
|
||||
};
|
||||
state.targets.insert(arn.clone(), ReplicationStatusType::Completed);
|
||||
|
||||
let dobj = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "doc.txt".to_string(),
|
||||
delete_marker: true,
|
||||
delete_marker_version_id: Some(Uuid::new_v4()),
|
||||
replication_state: Some(state),
|
||||
..Default::default()
|
||||
},
|
||||
bucket: "bucket-a".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// No target client is registered: the refusal must be decided from
|
||||
// the recorded metadata alone, before any remote call is attempted.
|
||||
let failed = replicate_delete_marker_purge_to_targets("bucket-a", &dobj, &dsc, None).await;
|
||||
|
||||
assert_eq!(
|
||||
failed,
|
||||
vec![arn],
|
||||
"a refused purge must stay in the failed set so the intent is never acknowledged"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_marker_purge_mrf_entry_replays_through_the_stale_marker_branch() {
|
||||
let delete_marker_version_id = Uuid::new_v4();
|
||||
let dobj = DeletedObjectReplicationInfo {
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "doc.txt".to_string(),
|
||||
// A version-purge flavored source event: the entry must still
|
||||
// be reshaped as a marker-creation delete so replay funnels
|
||||
// into the stale-marker branch instead of re-running the full
|
||||
// delete replication (whose source-state stamping would fail
|
||||
// against the already-purged version).
|
||||
delete_marker: false,
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
delete_marker_version_id: Some(delete_marker_version_id),
|
||||
..Default::default()
|
||||
},
|
||||
bucket: "bucket-a".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let entry = delete_marker_purge_mrf_entry(&dobj, vec!["arn:a".to_string()]);
|
||||
|
||||
assert!(entry.delete_marker, "purge intents must replay as marker-creation deletes");
|
||||
assert_eq!(entry.version_id, None, "the purged data version must not leak into the replay");
|
||||
assert_eq!(entry.delete_marker_version_id, Some(delete_marker_version_id));
|
||||
assert_eq!(
|
||||
entry.target_arns,
|
||||
vec!["arn:a".to_string()],
|
||||
"only the targets whose purge failed may be retried"
|
||||
);
|
||||
assert_eq!(entry.retry_count, 0);
|
||||
assert_eq!(entry.bucket, "bucket-a");
|
||||
assert_eq!(entry.object, "doc.txt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_retryable_delete_replication_head_error_allows_delete_marker_head_responses() {
|
||||
assert!(
|
||||
|
||||
@@ -1,171 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::{HeaderMap, StatusCode};
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Body;
|
||||
use hyper::body::Bytes;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::http_resp_to_error_response,
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
||||
};
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn set_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
|
||||
if policy == "" {
|
||||
return self.remove_bucket_policy(bucket_name).await;
|
||||
}
|
||||
|
||||
self.put_bucket_policy(bucket_name, policy).await
|
||||
}
|
||||
|
||||
pub async fn put_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("policy".to_string(), "".to_string());
|
||||
|
||||
let mut req_metadata = RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
query_values: url_values,
|
||||
content_body: ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())),
|
||||
content_length: policy.len() as i64,
|
||||
object_name: "".to_string(),
|
||||
custom_header: HeaderMap::new(),
|
||||
content_md5_base64: "".to_string(),
|
||||
content_sha256_hex: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
};
|
||||
|
||||
let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
|
||||
//defer closeResponse(resp)
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
|
||||
//if resp != nil {
|
||||
if resp_status != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(
|
||||
resp_status,
|
||||
&h,
|
||||
vec![],
|
||||
bucket_name,
|
||||
"",
|
||||
)));
|
||||
}
|
||||
//}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn remove_bucket_policy(&self, bucket_name: &str) -> Result<(), std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("policy".to_string(), "".to_string());
|
||||
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::DELETE,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
query_values: url_values,
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
object_name: "".to_string(),
|
||||
custom_header: HeaderMap::new(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
//defer closeResponse(resp)
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
|
||||
if resp_status != StatusCode::NO_CONTENT {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(
|
||||
resp_status,
|
||||
&h,
|
||||
vec![],
|
||||
bucket_name,
|
||||
"",
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result<String, std::io::Error> {
|
||||
let bucket_policy = self.get_bucket_policy_inner(bucket_name).await?;
|
||||
Ok(bucket_policy)
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("policy".to_string(), "".to_string());
|
||||
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::GET,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
query_values: url_values,
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
object_name: "".to_string(),
|
||||
custom_header: HeaderMap::new(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
let policy = String::from_utf8_lossy(&body_vec).to_string();
|
||||
Ok(policy)
|
||||
}
|
||||
}
|
||||
@@ -1,199 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::http_resp_to_error_response,
|
||||
api_get_options::GetObjectOptions,
|
||||
transition_api::{ObjectInfo, ReaderImpl, RequestMetadata, TransitionClient},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use http_body_util::BodyExt;
|
||||
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
||||
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
|
||||
use s3s::dto::Owner;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Grantee {
|
||||
pub id: String,
|
||||
pub display_name: String,
|
||||
pub uri: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Grant {
|
||||
pub grantee: Grantee,
|
||||
pub permission: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct AccessControlList {
|
||||
pub grant: Vec<Grant>,
|
||||
pub permission: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub struct AccessControlPolicy {
|
||||
#[serde(skip)]
|
||||
owner: Owner,
|
||||
pub access_control_list: AccessControlList,
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn get_object_acl(&self, bucket_name: &str, object_name: &str) -> Result<ObjectInfo, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("acl".to_string(), "".to_string());
|
||||
let mut resp = self
|
||||
.execute_method(
|
||||
http::Method::GET,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
query_values: url_values,
|
||||
custom_header: HeaderMap::new(),
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
|
||||
if resp_status != http::StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(
|
||||
resp_status,
|
||||
&h,
|
||||
body_vec,
|
||||
bucket_name,
|
||||
object_name,
|
||||
)));
|
||||
}
|
||||
|
||||
let mut res = match quick_xml::de::from_str::<AccessControlPolicy>(&String::from_utf8(body_vec).unwrap()) {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
let mut obj_info = self
|
||||
.stat_object(bucket_name, object_name, &GetObjectOptions::default())
|
||||
.await?;
|
||||
|
||||
obj_info.owner.display_name = res.owner.display_name.clone();
|
||||
obj_info.owner.id = res.owner.id.clone();
|
||||
|
||||
//obj_info.grant.extend(res.access_control_list.grant);
|
||||
|
||||
let canned_acl = get_canned_acl(&res);
|
||||
if canned_acl != "" {
|
||||
obj_info
|
||||
.metadata
|
||||
.insert("X-Amz-Acl", HeaderValue::from_str(&canned_acl).unwrap());
|
||||
return Ok(obj_info);
|
||||
}
|
||||
|
||||
let grant_acl = get_amz_grant_acl(&res);
|
||||
/*for (k, v) in grant_acl {
|
||||
obj_info.metadata.insert(HeaderName::from_bytes(k.as_bytes()).unwrap(), HeaderValue::from_str(&v.to_string()).unwrap());
|
||||
}*/
|
||||
|
||||
Ok(obj_info)
|
||||
}
|
||||
}
|
||||
|
||||
fn get_canned_acl(ac_policy: &AccessControlPolicy) -> String {
|
||||
let grants = ac_policy.access_control_list.grant.clone();
|
||||
|
||||
if grants.len() == 1 {
|
||||
if grants[0].grantee.uri == "" && grants[0].permission == "FULL_CONTROL" {
|
||||
return "private".to_string();
|
||||
}
|
||||
} else if grants.len() == 2 {
|
||||
for g in grants {
|
||||
if g.grantee.uri == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && &g.permission == "READ" {
|
||||
return "authenticated-read".to_string();
|
||||
}
|
||||
if g.grantee.uri == "http://acs.amazonaws.com/groups/global/AllUsers" && &g.permission == "READ" {
|
||||
return "public-read".to_string();
|
||||
}
|
||||
if g.permission == "READ" && g.grantee.id == ac_policy.owner.id.clone().unwrap() {
|
||||
return "bucket-owner-read".to_string();
|
||||
}
|
||||
}
|
||||
} else if grants.len() == 3 {
|
||||
for g in grants {
|
||||
if g.grantee.uri == "http://acs.amazonaws.com/groups/global/AllUsers" && g.permission == "WRITE" {
|
||||
return "public-read-write".to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
"".to_string()
|
||||
}
|
||||
|
||||
pub fn get_amz_grant_acl(ac_policy: &AccessControlPolicy) -> HashMap<String, Vec<String>> {
|
||||
let grants = ac_policy.access_control_list.grant.clone();
|
||||
let mut res = HashMap::<String, Vec<String>>::new();
|
||||
|
||||
for g in grants {
|
||||
let mut id = "id=".to_string();
|
||||
id.push_str(&g.grantee.id);
|
||||
let permission: &str = &g.permission;
|
||||
match permission {
|
||||
"READ" => {
|
||||
res.entry("X-Amz-Grant-Read".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
"WRITE" => {
|
||||
res.entry("X-Amz-Grant-Write".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
"READ_ACP" => {
|
||||
res.entry("X-Amz-Grant-Read-Acp".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
"WRITE_ACP" => {
|
||||
res.entry("X-Amz-Grant-Write-Acp".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
"FULL_CONTROL" => {
|
||||
res.entry("X-Amz-Grant-Full-Control".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
@@ -1,266 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::client::constants::{GET_OBJECT_ATTRIBUTES_MAX_PARTS, GET_OBJECT_ATTRIBUTES_TAGS, ISO8601_DATEFORMAT};
|
||||
use crate::client::{
|
||||
api_get_object_acl::AccessControlPolicy,
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
||||
};
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Body;
|
||||
use hyper::body::Bytes;
|
||||
use hyper::body::Incoming;
|
||||
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
||||
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
|
||||
use s3s::header::{X_AMZ_MAX_PARTS, X_AMZ_OBJECT_ATTRIBUTES, X_AMZ_PART_NUMBER_MARKER, X_AMZ_VERSION_ID};
|
||||
|
||||
pub struct ObjectAttributesOptions {
|
||||
pub max_parts: i64,
|
||||
pub version_id: String,
|
||||
pub part_number_marker: i64,
|
||||
//server_side_encryption: encrypt::ServerSide,
|
||||
}
|
||||
|
||||
pub struct ObjectAttributes {
|
||||
pub version_id: String,
|
||||
pub last_modified: OffsetDateTime,
|
||||
pub object_attributes_response: ObjectAttributesResponse,
|
||||
}
|
||||
|
||||
impl ObjectAttributes {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
version_id: "".to_string(),
|
||||
last_modified: OffsetDateTime::now_utc(),
|
||||
object_attributes_response: ObjectAttributesResponse::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub struct Checksum {
|
||||
checksum_crc32: String,
|
||||
checksum_crc32c: String,
|
||||
checksum_sha1: String,
|
||||
checksum_sha256: String,
|
||||
}
|
||||
|
||||
impl Checksum {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
checksum_crc32: "".to_string(),
|
||||
checksum_crc32c: "".to_string(),
|
||||
checksum_sha1: "".to_string(),
|
||||
checksum_sha256: "".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub struct ObjectParts {
|
||||
pub parts_count: i64,
|
||||
pub part_number_marker: i64,
|
||||
pub next_part_number_marker: i64,
|
||||
pub max_parts: i64,
|
||||
is_truncated: bool,
|
||||
parts: Vec<ObjectAttributePart>,
|
||||
}
|
||||
|
||||
impl ObjectParts {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
parts_count: 0,
|
||||
part_number_marker: 0,
|
||||
next_part_number_marker: 0,
|
||||
max_parts: 0,
|
||||
is_truncated: false,
|
||||
parts: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub struct ObjectAttributesResponse {
|
||||
pub etag: String,
|
||||
pub storage_class: String,
|
||||
pub object_size: i64,
|
||||
pub checksum: Checksum,
|
||||
pub object_parts: ObjectParts,
|
||||
}
|
||||
|
||||
impl ObjectAttributesResponse {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
etag: "".to_string(),
|
||||
storage_class: "".to_string(),
|
||||
object_size: 0,
|
||||
checksum: Checksum::new(),
|
||||
object_parts: ObjectParts::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
struct ObjectAttributePart {
|
||||
checksum_crc32: String,
|
||||
checksum_crc32c: String,
|
||||
checksum_sha1: String,
|
||||
checksum_sha256: String,
|
||||
part_number: i64,
|
||||
size: i64,
|
||||
}
|
||||
|
||||
impl ObjectAttributes {
|
||||
pub async fn parse_response(&mut self, h: &HeaderMap, body_vec: Vec<u8>) -> Result<(), std::io::Error> {
|
||||
let last_modified = h
|
||||
.get("Last-Modified")
|
||||
.ok_or_else(|| std::io::Error::other("missing Last-Modified header"))?
|
||||
.to_str()
|
||||
.map_err(|e| std::io::Error::other(format!("invalid Last-Modified header: {e}")))?;
|
||||
let mod_time = OffsetDateTime::parse(last_modified, ISO8601_DATEFORMAT)
|
||||
.map_err(|e| std::io::Error::other(format!("invalid Last-Modified date: {e}")))?;
|
||||
self.last_modified = mod_time;
|
||||
|
||||
let version_id = h
|
||||
.get(X_AMZ_VERSION_ID)
|
||||
.ok_or_else(|| std::io::Error::other("missing version ID header"))?
|
||||
.to_str()
|
||||
.map_err(|e| std::io::Error::other(format!("invalid version ID header: {e}")))?;
|
||||
self.version_id = version_id.to_string();
|
||||
|
||||
let body_str = String::from_utf8(body_vec).map_err(|e| std::io::Error::other(format!("invalid UTF-8 body: {e}")))?;
|
||||
let mut response = match quick_xml::de::from_str::<ObjectAttributesResponse>(&body_str) {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
self.object_attributes_response = response;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn get_object_attributes(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
opts: ObjectAttributesOptions,
|
||||
) -> Result<ObjectAttributes, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("attributes".to_string(), "".to_string());
|
||||
if opts.version_id != "" {
|
||||
url_values.insert("versionId".to_string(), opts.version_id);
|
||||
}
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
X_AMZ_OBJECT_ATTRIBUTES,
|
||||
HeaderValue::from_str(GET_OBJECT_ATTRIBUTES_TAGS).expect("valid header value"),
|
||||
);
|
||||
|
||||
if opts.part_number_marker > 0 {
|
||||
headers.insert(
|
||||
X_AMZ_PART_NUMBER_MARKER,
|
||||
HeaderValue::from_str(&opts.part_number_marker.to_string()).expect("valid header value"),
|
||||
);
|
||||
}
|
||||
|
||||
if opts.max_parts > 0 {
|
||||
headers.insert(
|
||||
X_AMZ_MAX_PARTS,
|
||||
HeaderValue::from_str(&opts.max_parts.to_string()).expect("valid header value"),
|
||||
);
|
||||
} else {
|
||||
headers.insert(
|
||||
X_AMZ_MAX_PARTS,
|
||||
HeaderValue::from_str(&GET_OBJECT_ATTRIBUTES_MAX_PARTS.to_string()).expect("valid header value"),
|
||||
);
|
||||
}
|
||||
|
||||
/*if opts.server_side_encryption.is_some() {
|
||||
opts.server_side_encryption.Marshal(headers);
|
||||
}*/
|
||||
|
||||
let mut resp = self
|
||||
.execute_method(
|
||||
http::Method::HEAD,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
query_values: url_values,
|
||||
custom_header: headers,
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
content_md5_base64: "".to_string(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
let has_etag = h.get("ETag").and_then(|v| v.to_str().ok()).unwrap_or("");
|
||||
if !has_etag.is_empty() {
|
||||
return Err(std::io::Error::other(
|
||||
"get_object_attributes is not supported by the current endpoint version",
|
||||
));
|
||||
}
|
||||
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
|
||||
if resp_status != http::StatusCode::OK {
|
||||
let err_body =
|
||||
String::from_utf8(body_vec).map_err(|e| std::io::Error::other(format!("invalid UTF-8 error body: {e}")))?;
|
||||
let mut er = match quick_xml::de::from_str::<AccessControlPolicy>(&err_body) {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
return Err(std::io::Error::other(er.access_control_list.permission));
|
||||
}
|
||||
|
||||
let mut oa = ObjectAttributes::new();
|
||||
oa.parse_response(&h, body_vec).await?;
|
||||
|
||||
Ok(oa)
|
||||
}
|
||||
}
|
||||
@@ -1,159 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[cfg(not(windows))]
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
use tokio::fs::{self, OpenOptions};
|
||||
use tokio::io::{AsyncSeekExt, AsyncWriteExt, SeekFrom};
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::err_invalid_argument, api_get_options::GetObjectOptions, transition_api::TransitionClient,
|
||||
};
|
||||
|
||||
async fn prepare_download_target(file_path: &Path) -> io::Result<()> {
|
||||
match fs::metadata(file_path).await {
|
||||
Ok(metadata) if metadata.is_dir() => {
|
||||
return Err(io::Error::other(err_invalid_argument("filename is a directory.")));
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
if let Some(parent) = file_path.parent()
|
||||
&& !parent.as_os_str().is_empty()
|
||||
{
|
||||
fs::create_dir_all(parent).await?;
|
||||
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
let mut permissions = fs::metadata(parent).await?.permissions();
|
||||
permissions.set_mode(0o700);
|
||||
fs::set_permissions(parent, permissions).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn build_part_path(file_path: &Path) -> PathBuf {
|
||||
PathBuf::from(format!("{}.part.rustfs", file_path.display()))
|
||||
}
|
||||
|
||||
async fn open_download_part_file(file_part_path: &Path) -> io::Result<tokio::fs::File> {
|
||||
let mut options = OpenOptions::new();
|
||||
options.create(true).truncate(false).read(true).write(true);
|
||||
|
||||
#[cfg(not(windows))]
|
||||
options.mode(0o600);
|
||||
|
||||
options.open(file_part_path).await
|
||||
}
|
||||
|
||||
async fn cleanup_part_file(file_part_path: &Path) {
|
||||
let _ = fs::remove_file(file_part_path).await;
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn fget_object(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
file_path: &str,
|
||||
mut opts: GetObjectOptions,
|
||||
) -> Result<(), io::Error> {
|
||||
let file_path = Path::new(file_path);
|
||||
prepare_download_target(file_path).await?;
|
||||
|
||||
let file_part_path = build_part_path(file_path);
|
||||
let mut file_part = open_download_part_file(&file_part_path).await?;
|
||||
let existing_len = file_part.metadata().await?.len();
|
||||
if existing_len > 0 {
|
||||
opts.set_range(existing_len as i64, 0)?;
|
||||
file_part.seek(SeekFrom::Start(existing_len)).await?;
|
||||
}
|
||||
|
||||
let (_object_info, _headers, mut object_reader) = self.get_object_inner(bucket_name, object_name, &opts).await?;
|
||||
if let Err(err) = tokio::io::copy(&mut object_reader, &mut file_part).await {
|
||||
cleanup_part_file(&file_part_path).await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if let Err(err) = file_part.flush().await {
|
||||
cleanup_part_file(&file_part_path).await;
|
||||
return Err(err);
|
||||
}
|
||||
drop(file_part);
|
||||
|
||||
if let Err(err) = fs::rename(&file_part_path, file_path).await {
|
||||
cleanup_part_file(&file_part_path).await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepare_download_target_allows_missing_file_and_creates_parent_dirs() {
|
||||
let dir = tempdir().expect("temp dir");
|
||||
let target = dir.path().join("nested").join("object.bin");
|
||||
|
||||
prepare_download_target(&target)
|
||||
.await
|
||||
.expect("missing target should be accepted");
|
||||
|
||||
assert!(target.parent().expect("parent").exists(), "parent directory should be created");
|
||||
assert!(
|
||||
fs::metadata(&target).await.is_err(),
|
||||
"preparing the target should not create the final file eagerly"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepare_download_target_rejects_directory_paths() {
|
||||
let dir = tempdir().expect("temp dir");
|
||||
let target_dir = dir.path().join("download-dir");
|
||||
fs::create_dir_all(&target_dir).await.expect("target dir");
|
||||
|
||||
let err = prepare_download_target(&target_dir)
|
||||
.await
|
||||
.expect_err("directory targets must be rejected");
|
||||
|
||||
assert!(err.to_string().contains("directory"), "unexpected error for directory target: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn open_download_part_file_creates_part_file() {
|
||||
let dir = tempdir().expect("temp dir");
|
||||
let target = dir.path().join("object.bin");
|
||||
let part_path = build_part_path(&target);
|
||||
|
||||
let file = open_download_part_file(&part_path)
|
||||
.await
|
||||
.expect("part file should be created");
|
||||
drop(file);
|
||||
|
||||
assert!(part_path.exists(), "part file should exist after creation");
|
||||
}
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::{err_invalid_argument, http_resp_to_error_response},
|
||||
api_get_object_acl::AccessControlList,
|
||||
api_get_options::GetObjectOptions,
|
||||
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Body;
|
||||
use hyper::body::Bytes;
|
||||
use s3s::dto::RestoreRequest;
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::BufReader;
|
||||
|
||||
const TIER_STANDARD: &str = "Standard";
|
||||
const TIER_BULK: &str = "Bulk";
|
||||
const TIER_EXPEDITED: &str = "Expedited";
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Encryption {
|
||||
pub encryption_type: String,
|
||||
pub kms_context: String,
|
||||
pub kms_key_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct MetadataEntry {
|
||||
pub name: String,
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize)]
|
||||
pub struct S3 {
|
||||
pub access_control_list: AccessControlList,
|
||||
pub bucket_name: String,
|
||||
pub prefix: String,
|
||||
pub canned_acl: String,
|
||||
pub encryption: Encryption,
|
||||
pub storage_class: String,
|
||||
//tagging: Tags,
|
||||
pub user_metadata: MetadataEntry,
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn restore_object(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
version_id: &str,
|
||||
restore_req: &RestoreRequest,
|
||||
) -> Result<(), std::io::Error> {
|
||||
/*let restore_request = match quick_xml::se::to_string(restore_req) {
|
||||
Ok(buf) => buf,
|
||||
Err(e) => {
|
||||
return Err(std::io::Error::other(e));
|
||||
}
|
||||
};*/
|
||||
let restore_request = "".to_string();
|
||||
let restore_request_bytes = restore_request.as_bytes().to_vec();
|
||||
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("restore".to_string(), "".to_string());
|
||||
if version_id != "" {
|
||||
url_values.insert("versionId".to_string(), version_id.to_string());
|
||||
}
|
||||
|
||||
let restore_request_buffer = Bytes::from(restore_request_bytes.clone());
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::HEAD,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
query_values: url_values,
|
||||
custom_header: HeaderMap::new(),
|
||||
content_sha256_hex: "".to_string(), //sum_sha256_hex(&restore_request_bytes),
|
||||
content_md5_base64: "".to_string(), //sum_md5_base64(&restore_request_bytes),
|
||||
content_body: ReaderImpl::Body(restore_request_buffer),
|
||||
content_length: restore_request_bytes.len() as i64,
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
if resp_status != http::StatusCode::ACCEPTED && resp_status != http::StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(
|
||||
resp_status,
|
||||
&h,
|
||||
body_vec,
|
||||
bucket_name,
|
||||
"",
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,3 @@ pub const TOTAL_WORKERS: i64 = 4;
|
||||
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
|
||||
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
|
||||
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
|
||||
|
||||
pub const GET_OBJECT_ATTRIBUTES_TAGS: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts";
|
||||
pub const GET_OBJECT_ATTRIBUTES_MAX_PARTS: i64 = 1000;
|
||||
|
||||
@@ -16,12 +16,8 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod admin_handler_utils;
|
||||
pub mod api_bucket_policy;
|
||||
pub mod api_error_response;
|
||||
pub mod api_get_object;
|
||||
pub mod api_get_object_acl;
|
||||
pub mod api_get_object_attributes;
|
||||
pub mod api_get_object_file;
|
||||
pub mod api_get_options;
|
||||
pub mod api_list;
|
||||
pub mod api_put_object;
|
||||
@@ -29,7 +25,6 @@ pub mod api_put_object_common;
|
||||
pub mod api_put_object_multipart;
|
||||
pub mod api_put_object_streaming;
|
||||
pub mod api_remove;
|
||||
pub mod api_restore;
|
||||
pub mod api_s3_datatypes;
|
||||
pub mod api_stat;
|
||||
pub mod bucket_cache;
|
||||
|
||||
@@ -1006,16 +1006,6 @@ impl TransitionCore {
|
||||
client.abort_multipart_upload(bucket_name, object, upload_id).await
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result<String, std::io::Error> {
|
||||
let client = self.0.clone();
|
||||
client.get_bucket_policy(bucket_name).await
|
||||
}
|
||||
|
||||
pub async fn put_bucket_policy(&self, bucket_name: &str, bucket_policy: &str) -> Result<(), std::io::Error> {
|
||||
let client = self.0.clone();
|
||||
client.put_bucket_policy(bucket_name, bucket_policy).await
|
||||
}
|
||||
|
||||
pub async fn get_object(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
|
||||
@@ -15,12 +15,12 @@
|
||||
#[cfg(test)]
|
||||
use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
|
||||
use crate::cluster::rpc::http_auth::{
|
||||
RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER, RPC_BOOT_EPOCH_HEADER,
|
||||
RPC_BOOT_EPOCH_PROOF_HEADER, RPC_CONTENT_SHA256_HEADER, TIMESTAMP_HEADER,
|
||||
};
|
||||
use crate::cluster::rpc::{
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, verify_tonic_boot_epoch_response,
|
||||
AuthenticatedPeerReplayCapabilities, RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER,
|
||||
RPC_CONTENT_SHA256_HEADER, RPC_REPLAY_CACHE_CAPABILITY_HEADER, RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
|
||||
RollingMutationBodyDigest, TIMESTAMP_HEADER, internode_rpc_body_digest_strict,
|
||||
verify_tonic_peer_replay_capabilities_response,
|
||||
};
|
||||
use crate::cluster::rpc::{gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience};
|
||||
#[cfg(test)]
|
||||
use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
|
||||
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
|
||||
@@ -233,7 +233,22 @@ pub struct ReplayScopeChannel<S> {
|
||||
/// The channel type used by internode clients after v2 authentication and replay-scope handling.
|
||||
pub type AuthenticatedChannel = ReplayScopeChannel<Channel>;
|
||||
|
||||
static PEER_BOOT_EPOCHS: LazyLock<Mutex<HashMap<String, Uuid>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum PeerReplayCapability {
|
||||
Capable { boot_epoch: Uuid },
|
||||
Revoked,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||
struct PeerReplayState {
|
||||
boot_epoch: Option<Uuid>,
|
||||
cache_capability: Option<PeerReplayCapability>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct PeerReplayStateSnapshot(PeerReplayState);
|
||||
|
||||
static PEER_REPLAY_STATES: LazyLock<Mutex<HashMap<String, PeerReplayState>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
impl<S> ReplayScopeChannel<S> {
|
||||
fn new(inner: S, audience: Option<String>) -> Self {
|
||||
@@ -241,13 +256,67 @@ impl<S> ReplayScopeChannel<S> {
|
||||
}
|
||||
}
|
||||
|
||||
fn cached_peer_boot_epoch(audience: &str) -> Option<Uuid> {
|
||||
PEER_BOOT_EPOCHS.lock().ok().and_then(|epochs| epochs.get(audience).copied())
|
||||
fn peer_replay_state(audience: &str) -> PeerReplayState {
|
||||
PEER_REPLAY_STATES
|
||||
.lock()
|
||||
.ok()
|
||||
.and_then(|states| states.get(audience).copied())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
fn remember_peer_boot_epoch(audience: String, epoch: Uuid) {
|
||||
if let Ok(mut epochs) = PEER_BOOT_EPOCHS.lock() {
|
||||
epochs.insert(audience, epoch);
|
||||
fn apply_peer_replay_response(
|
||||
audience: String,
|
||||
sent_state: PeerReplayState,
|
||||
response: std::io::Result<AuthenticatedPeerReplayCapabilities>,
|
||||
) {
|
||||
if let Ok(mut states) = PEER_REPLAY_STATES.lock() {
|
||||
let current_state = states.get(&audience).copied().unwrap_or_default();
|
||||
let mut next_state = current_state;
|
||||
if let Ok(response) = &response
|
||||
&& sent_state.boot_epoch == current_state.boot_epoch
|
||||
{
|
||||
next_state.boot_epoch = Some(response.boot_epoch);
|
||||
}
|
||||
|
||||
if sent_state.boot_epoch == current_state.boot_epoch {
|
||||
let response_capability = response
|
||||
.as_ref()
|
||||
.ok()
|
||||
.filter(|response| response.dynamic_replay_cache)
|
||||
.map(|response| response.boot_epoch);
|
||||
match (sent_state.cache_capability, current_state.cache_capability, response_capability) {
|
||||
(None, None, Some(boot_epoch))
|
||||
| (Some(PeerReplayCapability::Revoked), Some(PeerReplayCapability::Revoked), Some(boot_epoch)) => {
|
||||
next_state.cache_capability = Some(PeerReplayCapability::Capable { boot_epoch });
|
||||
}
|
||||
(
|
||||
Some(PeerReplayCapability::Capable {
|
||||
boot_epoch: sent_boot_epoch,
|
||||
}),
|
||||
Some(PeerReplayCapability::Capable {
|
||||
boot_epoch: current_boot_epoch,
|
||||
}),
|
||||
Some(response_boot_epoch),
|
||||
) if sent_boot_epoch == current_boot_epoch => {
|
||||
next_state.cache_capability = Some(PeerReplayCapability::Capable {
|
||||
boot_epoch: response_boot_epoch,
|
||||
});
|
||||
}
|
||||
(
|
||||
Some(PeerReplayCapability::Capable {
|
||||
boot_epoch: sent_boot_epoch,
|
||||
}),
|
||||
Some(PeerReplayCapability::Capable {
|
||||
boot_epoch: current_boot_epoch,
|
||||
}),
|
||||
None,
|
||||
) if sent_boot_epoch == current_boot_epoch => {
|
||||
next_state.cache_capability = Some(PeerReplayCapability::Revoked);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
states.insert(audience, next_state);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -276,6 +345,11 @@ where
|
||||
== Some(RPC_AUTH_VERSION_V2)
|
||||
});
|
||||
let challenge = authenticated.then(Uuid::new_v4);
|
||||
let sent_state = request
|
||||
.extensions()
|
||||
.get::<PeerReplayStateSnapshot>()
|
||||
.map(|snapshot| snapshot.0)
|
||||
.unwrap_or_default();
|
||||
if let (Some(audience), Some(challenge)) = (self.audience.as_deref(), challenge) {
|
||||
// The challenge is independently HMAC-authenticated by the response proof. It is not
|
||||
// part of v2 so old peers ignore it, while a new peer can safely advertise its epoch.
|
||||
@@ -284,7 +358,7 @@ where
|
||||
challenge.to_string().parse().expect("UUID must be a valid header value"),
|
||||
);
|
||||
if let (Some(boot_epoch), Some(timestamp), Some(content_sha256)) = (
|
||||
cached_peer_boot_epoch(audience),
|
||||
sent_state.boot_epoch,
|
||||
request.headers().get(TIMESTAMP_HEADER).and_then(|value| value.to_str().ok()),
|
||||
request
|
||||
.headers()
|
||||
@@ -303,16 +377,21 @@ where
|
||||
Box::pin(async move {
|
||||
let response = future.await?;
|
||||
if let (Some(audience), Some(challenge)) = (audience, challenge) {
|
||||
match verify_tonic_boot_epoch_response(&audience, challenge, response.headers()) {
|
||||
Ok(epoch) => remember_peer_boot_epoch(audience, epoch),
|
||||
Err(error)
|
||||
if response.headers().contains_key(RPC_BOOT_EPOCH_HEADER)
|
||||
|| response.headers().contains_key(RPC_BOOT_EPOCH_PROOF_HEADER) =>
|
||||
{
|
||||
debug!(error = %error, "peer boot epoch response proof was rejected")
|
||||
}
|
||||
Err(_) => {}
|
||||
let response_state = verify_tonic_peer_replay_capabilities_response(&audience, challenge, response.headers());
|
||||
if let Err(error) = &response_state
|
||||
&& (response.headers().contains_key(RPC_REPLAY_CACHE_CAPABILITY_HEADER)
|
||||
|| response.headers().contains_key(RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER))
|
||||
{
|
||||
debug!(
|
||||
event = "internode_rpc_capability_proof_rejected",
|
||||
component = "ecstore",
|
||||
subsystem = "rpc_client",
|
||||
result = "rejected",
|
||||
error = %error,
|
||||
"internode RPC capability proof rejected"
|
||||
)
|
||||
}
|
||||
apply_peer_replay_response(audience, sent_state, response_state);
|
||||
}
|
||||
Ok(response)
|
||||
})
|
||||
@@ -321,6 +400,7 @@ where
|
||||
|
||||
pub struct TonicSignatureInterceptor {
|
||||
audience: Option<String>,
|
||||
body_digest_strict: bool,
|
||||
}
|
||||
|
||||
impl tonic::service::Interceptor for TonicSignatureInterceptor {
|
||||
@@ -337,9 +417,31 @@ impl tonic::service::Interceptor for TonicSignatureInterceptor {
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
// RUSTFS_COMPAT_TODO(disk-mutation-body-digest): use cache-free v2 for peers without an authenticated boot epoch. Remove after every supported peer advertises the authenticated dynamic replay-cache capability and body-digest strict mode is the default.
|
||||
// beta.11 verifies v2 body digests but stores their nonces in a fixed-size cache.
|
||||
let rolling_mutation = req.extensions().get::<RollingMutationBodyDigest>().is_some();
|
||||
let peer_state = PEER_REPLAY_STATES
|
||||
.lock()
|
||||
.map_err(|_| tonic::Status::unauthenticated("RPC peer capability state unavailable"))?
|
||||
.get(audience)
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
let content_sha256 = if content_sha256.is_some() {
|
||||
if peer_state.cache_capability == Some(PeerReplayCapability::Revoked) {
|
||||
return Err(tonic::Status::unauthenticated("RPC peer replay capability changed"));
|
||||
}
|
||||
if rolling_mutation && !self.body_digest_strict && peer_state.boot_epoch.is_none() {
|
||||
None
|
||||
} else {
|
||||
content_sha256
|
||||
}
|
||||
} else {
|
||||
content_sha256
|
||||
};
|
||||
let headers = gen_tonic_signature_headers(audience, method.service(), method.method(), content_sha256)
|
||||
.map_err(|_| tonic::Status::unauthenticated("No valid auth token"))?;
|
||||
req.metadata_mut().as_mut().extend(headers);
|
||||
req.extensions_mut().insert(PeerReplayStateSnapshot(peer_state));
|
||||
inject_trace_context_into_metadata(req.metadata_mut());
|
||||
inject_request_id_into_metadata(req.metadata_mut());
|
||||
Ok(req)
|
||||
@@ -347,7 +449,10 @@ impl tonic::service::Interceptor for TonicSignatureInterceptor {
|
||||
}
|
||||
|
||||
pub fn gen_tonic_signature_interceptor() -> TonicSignatureInterceptor {
|
||||
TonicSignatureInterceptor { audience: None }
|
||||
TonicSignatureInterceptor {
|
||||
audience: None,
|
||||
body_digest_strict: internode_rpc_body_digest_strict(),
|
||||
}
|
||||
}
|
||||
|
||||
pub struct NoOpInterceptor;
|
||||
@@ -409,6 +514,7 @@ mod tests {
|
||||
#[derive(Clone)]
|
||||
struct EpochProofService {
|
||||
audience: String,
|
||||
include_capability: bool,
|
||||
seen_headers: std::sync::Arc<Mutex<Vec<http::HeaderMap>>>,
|
||||
}
|
||||
|
||||
@@ -430,29 +536,97 @@ mod tests {
|
||||
.expect("client challenge must be syntactically valid")
|
||||
.expect("authenticated client request must carry a boot epoch challenge");
|
||||
let mut response = HttpResponse::new(());
|
||||
response.headers_mut().extend(
|
||||
tonic_boot_epoch_response_headers(&self.audience, challenge)
|
||||
.expect("test server must be able to sign an epoch proof"),
|
||||
);
|
||||
let mut headers = tonic_boot_epoch_response_headers(&self.audience, challenge)
|
||||
.expect("test server must be able to sign an epoch proof");
|
||||
if !self.include_capability {
|
||||
headers.remove(RPC_REPLAY_CACHE_CAPABILITY_HEADER);
|
||||
headers.remove(RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER);
|
||||
}
|
||||
response.headers_mut().extend(headers);
|
||||
std::future::ready(Ok(response))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct MissingProofService;
|
||||
|
||||
impl Service<HttpRequest<()>> for MissingProofService {
|
||||
type Response = HttpResponse<()>;
|
||||
type Error = std::convert::Infallible;
|
||||
type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, _request: HttpRequest<()>) -> Self::Future {
|
||||
std::future::ready(Ok(HttpResponse::new(())))
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_test_rpc_secret() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
}
|
||||
|
||||
fn test_request() -> tonic::Request<()> {
|
||||
test_request_for("Ping")
|
||||
}
|
||||
|
||||
fn test_request_for(method: &'static str) -> tonic::Request<()> {
|
||||
let mut request = tonic::Request::new(());
|
||||
request
|
||||
.extensions_mut()
|
||||
.insert(tonic::GrpcMethod::new("node_service.NodeService", "Ping"));
|
||||
.insert(tonic::GrpcMethod::new("node_service.NodeService", method));
|
||||
request
|
||||
}
|
||||
|
||||
fn test_interceptor() -> TonicSignatureInterceptor {
|
||||
test_interceptor_for("node-a:9000", false)
|
||||
}
|
||||
|
||||
fn test_interceptor_for(audience: &str, body_digest_strict: bool) -> TonicSignatureInterceptor {
|
||||
TonicSignatureInterceptor {
|
||||
audience: Some("node-a:9000".to_string()),
|
||||
audience: Some(audience.to_string()),
|
||||
body_digest_strict,
|
||||
}
|
||||
}
|
||||
|
||||
fn clear_peer_capability(audience: &str) {
|
||||
PEER_REPLAY_STATES
|
||||
.lock()
|
||||
.expect("peer capability cache lock must not be poisoned")
|
||||
.remove(audience);
|
||||
}
|
||||
|
||||
fn rolling_mutation_request(method: &'static str) -> tonic::Request<()> {
|
||||
let mut request = tonic::Request::new(rustfs_protos::proto_gen::node_service::GenerallyLockRequest {
|
||||
args: "canonical mutation request".to_string(),
|
||||
});
|
||||
request
|
||||
.extensions_mut()
|
||||
.insert(tonic::GrpcMethod::new("node_service.NodeService", method));
|
||||
crate::cluster::rpc::set_tonic_rolling_mutation_body_digest(&mut request).expect("test mutation digest must be attached");
|
||||
request.map(|_| ())
|
||||
}
|
||||
|
||||
fn replay_scope_request(audience: &str, method: &'static str) -> HttpRequest<()> {
|
||||
let mut request = HttpRequest::builder()
|
||||
.uri(format!("/node_service.NodeService/{method}"))
|
||||
.body(())
|
||||
.expect("test RPC request must build");
|
||||
request.headers_mut().extend(
|
||||
gen_tonic_signature_headers(audience, "node_service.NodeService", method, None).expect("v2 test headers must mint"),
|
||||
);
|
||||
request
|
||||
.extensions_mut()
|
||||
.insert(PeerReplayStateSnapshot(peer_replay_state(audience)));
|
||||
request
|
||||
}
|
||||
|
||||
fn authenticated_peer_response(boot_epoch: Uuid, dynamic_replay_cache: bool) -> AuthenticatedPeerReplayCapabilities {
|
||||
AuthenticatedPeerReplayCapabilities {
|
||||
boot_epoch,
|
||||
dynamic_replay_cache,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -567,6 +741,431 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_peer_mutations_use_cache_free_unsigned_v2() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "legacy-body-digest-client-test:9000";
|
||||
clear_peer_capability(audience);
|
||||
let mut interceptor = test_interceptor_for(audience, false);
|
||||
for method in ["Lock", "WriteAll"] {
|
||||
let request = interceptor
|
||||
.call(rolling_mutation_request(method))
|
||||
.expect("interceptor call should succeed");
|
||||
|
||||
assert_eq!(
|
||||
request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("UNSIGNED-PAYLOAD")
|
||||
);
|
||||
assert_eq!(
|
||||
request
|
||||
.metadata()
|
||||
.get("x-rustfs-rpc-nonce")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("unsigned")
|
||||
);
|
||||
assert!(
|
||||
crate::cluster::rpc::verify_tonic_rpc_signature(
|
||||
audience,
|
||||
&format!("/node_service.NodeService/{method}"),
|
||||
request.metadata().as_ref(),
|
||||
)
|
||||
.is_ok(),
|
||||
"the cache-free request must retain valid audience- and method-bound v2 authentication"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_peer_exact_body_contract_remains_body_bound() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "exact-body-contract-client-test:9000";
|
||||
clear_peer_capability(audience);
|
||||
let mut interceptor = test_interceptor_for(audience, false);
|
||||
let mut request = test_request_for("ScannerActivity");
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, b"exact scanner activity body")
|
||||
.expect("test exact body digest must be attached");
|
||||
let expected_digest = request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("test request must carry its digest")
|
||||
.to_string();
|
||||
|
||||
let request = interceptor.call(request).expect("interceptor call should succeed");
|
||||
|
||||
assert_eq!(
|
||||
request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(expected_digest.as_str())
|
||||
);
|
||||
assert!(
|
||||
crate::cluster::rpc::verify_tonic_rpc_signature(
|
||||
audience,
|
||||
"/node_service.NodeService/ScannerActivity",
|
||||
request.metadata().as_ref(),
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_peer_iam_mutation_helper_remains_body_bound() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "exact-iam-mutation-client-test:9000";
|
||||
clear_peer_capability(audience);
|
||||
let mut interceptor = test_interceptor_for(audience, false);
|
||||
let mut request = tonic::Request::new(rustfs_protos::proto_gen::node_service::DeleteUserRequest {
|
||||
access_key: "target-access-key".to_string(),
|
||||
});
|
||||
request
|
||||
.extensions_mut()
|
||||
.insert(tonic::GrpcMethod::new("node_service.NodeService", "DeleteUser"));
|
||||
crate::cluster::rpc::set_tonic_mutation_body_digest(&mut request).expect("test IAM mutation digest must be attached");
|
||||
let request = request.map(|_| ());
|
||||
let expected_digest = request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("test IAM mutation must carry its digest")
|
||||
.to_string();
|
||||
|
||||
let request = interceptor.call(request).expect("interceptor call should succeed");
|
||||
|
||||
assert_eq!(
|
||||
request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(expected_digest.as_str())
|
||||
);
|
||||
assert!(
|
||||
crate::cluster::rpc::verify_tonic_rpc_signature(
|
||||
audience,
|
||||
"/node_service.NodeService/DeleteUser",
|
||||
request.metadata().as_ref(),
|
||||
)
|
||||
.is_ok(),
|
||||
"IAM mutations must remain body-bound before capability discovery"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authenticated_replay_cache_capability_enables_body_binding() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "body-digest-capable-client-test:9000";
|
||||
clear_peer_capability(audience);
|
||||
let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
|
||||
let service = EpochProofService {
|
||||
audience: audience.to_string(),
|
||||
include_capability: true,
|
||||
seen_headers,
|
||||
};
|
||||
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
|
||||
futures::executor::block_on(channel.call(replay_scope_request(audience, "Ping")))
|
||||
.expect("authenticated capability probe must complete");
|
||||
|
||||
let mut interceptor = test_interceptor_for(audience, false);
|
||||
let request = rolling_mutation_request("Lock");
|
||||
let expected_digest = request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("test mutation must carry its digest")
|
||||
.to_string();
|
||||
|
||||
let request = interceptor.call(request).expect("interceptor call should succeed");
|
||||
|
||||
assert_eq!(
|
||||
request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(expected_digest.as_str())
|
||||
);
|
||||
let nonce = request
|
||||
.metadata()
|
||||
.get("x-rustfs-rpc-nonce")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| Uuid::parse_str(value).ok())
|
||||
.expect("capable peer body-bound mutation must carry a UUID nonce");
|
||||
assert!(!nonce.is_nil());
|
||||
assert!(
|
||||
crate::cluster::rpc::verify_tonic_rpc_signature(
|
||||
audience,
|
||||
"/node_service.NodeService/Lock",
|
||||
request.metadata().as_ref(),
|
||||
)
|
||||
.is_ok(),
|
||||
"the body-bound request must retain valid audience- and method-bound v2 authentication"
|
||||
);
|
||||
clear_peer_capability(audience);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_capability_proof_does_not_enable_body_binding() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "invalid-capability-client-test:9000";
|
||||
clear_peer_capability(audience);
|
||||
let service = EpochProofService {
|
||||
audience: "wrong-capability-audience:9000".to_string(),
|
||||
include_capability: true,
|
||||
seen_headers: std::sync::Arc::new(Mutex::new(Vec::new())),
|
||||
};
|
||||
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
|
||||
futures::executor::block_on(channel.call(replay_scope_request(audience, "Ping")))
|
||||
.expect("invalid capability response must still complete");
|
||||
|
||||
let mut interceptor = test_interceptor_for(audience, false);
|
||||
let request = interceptor
|
||||
.call(rolling_mutation_request("Lock"))
|
||||
.expect("legacy-compatible mutation must still be signed");
|
||||
|
||||
assert_eq!(
|
||||
request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("UNSIGNED-PAYLOAD")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_boot_proof_keeps_mutations_body_bound_and_enables_non_ping_v3() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "legacy-boot-proof-client-test:9000";
|
||||
clear_peer_capability(audience);
|
||||
let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
|
||||
let service = EpochProofService {
|
||||
audience: audience.to_string(),
|
||||
include_capability: false,
|
||||
seen_headers: seen_headers.clone(),
|
||||
};
|
||||
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
|
||||
futures::executor::block_on(channel.call(replay_scope_request(audience, "Ping")))
|
||||
.expect("legacy boot proof response must complete");
|
||||
|
||||
let state = peer_replay_state(audience);
|
||||
assert!(state.boot_epoch.is_some(), "authenticated legacy proof must enable replay-scoped v3");
|
||||
assert_eq!(state.cache_capability, None);
|
||||
|
||||
let mut interceptor = test_interceptor_for(audience, false);
|
||||
let request = rolling_mutation_request("Lock");
|
||||
let expected_digest = request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("test mutation must carry its digest")
|
||||
.to_string();
|
||||
let request = interceptor.call(request).expect("legacy-compatible mutation must be signed");
|
||||
assert_eq!(
|
||||
request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(expected_digest.as_str())
|
||||
);
|
||||
let (metadata, extensions, body) = request.into_parts();
|
||||
let mut request = HttpRequest::new(body);
|
||||
*request.uri_mut() = "/node_service.NodeService/Lock".parse().expect("test RPC URI must parse");
|
||||
*request.headers_mut() = metadata.into_headers();
|
||||
*request.extensions_mut() = extensions;
|
||||
futures::executor::block_on(channel.call(request)).expect("legacy strict-compatible lock request must complete");
|
||||
|
||||
let headers = seen_headers.lock().expect("test header capture lock must not be poisoned");
|
||||
assert!(
|
||||
headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
|
||||
"authenticated legacy boot proof must enable v3 on a non-Ping request"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reordered_capability_responses_cannot_undo_newer_state() {
|
||||
let audience = "reordered-capability-client-test:9000";
|
||||
let epoch_one = Uuid::new_v4();
|
||||
let epoch_two = Uuid::new_v4();
|
||||
clear_peer_capability(audience);
|
||||
|
||||
let unknown = PeerReplayState::default();
|
||||
apply_peer_replay_response(audience.to_string(), unknown, Ok(authenticated_peer_response(epoch_one, true)));
|
||||
apply_peer_replay_response(audience.to_string(), unknown, Err(std::io::Error::other("delayed legacy response")));
|
||||
let epoch_one_state = PeerReplayState {
|
||||
boot_epoch: Some(epoch_one),
|
||||
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch: epoch_one }),
|
||||
};
|
||||
assert_eq!(peer_replay_state(audience), epoch_one_state);
|
||||
|
||||
apply_peer_replay_response(audience.to_string(), epoch_one_state, Err(std::io::Error::other("rollback response")));
|
||||
apply_peer_replay_response(audience.to_string(), epoch_one_state, Ok(authenticated_peer_response(epoch_one, true)));
|
||||
assert_eq!(
|
||||
peer_replay_state(audience),
|
||||
PeerReplayState {
|
||||
boot_epoch: Some(epoch_one),
|
||||
cache_capability: Some(PeerReplayCapability::Revoked),
|
||||
}
|
||||
);
|
||||
|
||||
let revoked = peer_replay_state(audience);
|
||||
apply_peer_replay_response(audience.to_string(), revoked, Ok(authenticated_peer_response(epoch_two, true)));
|
||||
apply_peer_replay_response(audience.to_string(), epoch_one_state, Ok(authenticated_peer_response(epoch_one, true)));
|
||||
assert_eq!(
|
||||
peer_replay_state(audience),
|
||||
PeerReplayState {
|
||||
boot_epoch: Some(epoch_two),
|
||||
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch: epoch_two }),
|
||||
}
|
||||
);
|
||||
clear_peer_capability(audience);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_capability_response_cannot_cross_a_new_boot_epoch() {
|
||||
let audience = "cross-epoch-capability-client-test:9000";
|
||||
let epoch_one = Uuid::new_v4();
|
||||
let epoch_two = Uuid::new_v4();
|
||||
let epoch_three = Uuid::new_v4();
|
||||
clear_peer_capability(audience);
|
||||
|
||||
let revoked_epoch_one = PeerReplayState {
|
||||
boot_epoch: Some(epoch_one),
|
||||
cache_capability: Some(PeerReplayCapability::Revoked),
|
||||
};
|
||||
PEER_REPLAY_STATES
|
||||
.lock()
|
||||
.expect("peer replay state lock must not be poisoned")
|
||||
.insert(audience.to_string(), revoked_epoch_one);
|
||||
|
||||
apply_peer_replay_response(
|
||||
audience.to_string(),
|
||||
revoked_epoch_one,
|
||||
Ok(authenticated_peer_response(epoch_three, false)),
|
||||
);
|
||||
apply_peer_replay_response(audience.to_string(), revoked_epoch_one, Ok(authenticated_peer_response(epoch_two, true)));
|
||||
|
||||
assert_eq!(
|
||||
peer_replay_state(audience),
|
||||
PeerReplayState {
|
||||
boot_epoch: Some(epoch_three),
|
||||
cache_capability: Some(PeerReplayCapability::Revoked),
|
||||
},
|
||||
"a stale dynamic-cache proof must not cross a newer authenticated boot epoch"
|
||||
);
|
||||
clear_peer_capability(audience);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interceptor_snapshot_prevents_delayed_legacy_response_from_revoking_capability() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "capability-snapshot-client-test:9000";
|
||||
clear_peer_capability(audience);
|
||||
let boot_epoch = Uuid::new_v4();
|
||||
let mut interceptor = test_interceptor_for(audience, false);
|
||||
let request = interceptor
|
||||
.call(rolling_mutation_request("Lock"))
|
||||
.expect("legacy-compatible request must pass the interceptor");
|
||||
assert_eq!(
|
||||
request
|
||||
.extensions()
|
||||
.get::<PeerReplayStateSnapshot>()
|
||||
.map(|snapshot| snapshot.0),
|
||||
Some(PeerReplayState::default()),
|
||||
"interceptor must preserve its unknown-state admission snapshot"
|
||||
);
|
||||
|
||||
let capable_state = PeerReplayState {
|
||||
boot_epoch: Some(boot_epoch),
|
||||
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch }),
|
||||
};
|
||||
PEER_REPLAY_STATES
|
||||
.lock()
|
||||
.expect("peer capability cache lock must not be poisoned")
|
||||
.insert(audience.to_string(), capable_state);
|
||||
let (metadata, extensions, body) = request.into_parts();
|
||||
let mut request = HttpRequest::new(body);
|
||||
*request.uri_mut() = "/node_service.NodeService/Lock".parse().expect("test RPC URI must parse");
|
||||
*request.headers_mut() = metadata.into_headers();
|
||||
*request.extensions_mut() = extensions;
|
||||
let mut channel = ReplayScopeChannel::new(MissingProofService, Some(audience.to_string()));
|
||||
|
||||
futures::executor::block_on(channel.call(request)).expect("in-flight request response must complete");
|
||||
|
||||
assert_eq!(peer_replay_state(audience), capable_state);
|
||||
clear_peer_capability(audience);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_mode_keeps_unknown_peer_mutations_body_bound() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "strict-body-digest-client-test:9000";
|
||||
clear_peer_capability(audience);
|
||||
let mut interceptor = test_interceptor_for(audience, true);
|
||||
let request = rolling_mutation_request("WriteAll");
|
||||
let expected_digest = request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("test mutation must carry its digest")
|
||||
.to_string();
|
||||
|
||||
let request = interceptor.call(request).expect("interceptor call should succeed");
|
||||
|
||||
assert_eq!(
|
||||
request
|
||||
.metadata()
|
||||
.get("x-rustfs-content-sha256")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some(expected_digest.as_str())
|
||||
);
|
||||
let nonce = request
|
||||
.metadata()
|
||||
.get("x-rustfs-rpc-nonce")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| Uuid::parse_str(value).ok())
|
||||
.expect("strict body-bound mutation must carry a UUID nonce");
|
||||
assert!(!nonce.is_nil());
|
||||
assert!(
|
||||
crate::cluster::rpc::verify_tonic_rpc_signature(
|
||||
audience,
|
||||
"/node_service.NodeService/WriteAll",
|
||||
request.metadata().as_ref(),
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_capability_after_pin_fails_closed() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "revoked-capability-client-test:9000";
|
||||
let boot_epoch = Uuid::new_v4();
|
||||
PEER_REPLAY_STATES
|
||||
.lock()
|
||||
.expect("peer capability cache lock must not be poisoned")
|
||||
.insert(
|
||||
audience.to_string(),
|
||||
PeerReplayState {
|
||||
boot_epoch: Some(boot_epoch),
|
||||
cache_capability: Some(PeerReplayCapability::Capable { boot_epoch }),
|
||||
},
|
||||
);
|
||||
let mut channel = ReplayScopeChannel::new(MissingProofService, Some(audience.to_string()));
|
||||
futures::executor::block_on(channel.call(replay_scope_request(audience, "Ping")))
|
||||
.expect("legacy response must complete before capability rejection");
|
||||
|
||||
let mut interceptor = test_interceptor_for(audience, false);
|
||||
let error = interceptor
|
||||
.call(rolling_mutation_request("Lock"))
|
||||
.expect_err("a peer that loses its pinned capability must fail closed");
|
||||
|
||||
assert_eq!(error.code(), tonic::Code::Unauthenticated);
|
||||
assert_eq!(error.message(), "RPC peer replay capability changed");
|
||||
clear_peer_capability(audience);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_signature_interceptor_binds_audience_from_peer_uri() {
|
||||
let interceptor = TonicInterceptor::Signature(gen_tonic_signature_interceptor())
|
||||
@@ -583,27 +1182,15 @@ mod tests {
|
||||
fn replay_scope_channel_uses_epoch_proof_before_sending_v3() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "replay-scope-client-test:9000";
|
||||
PEER_BOOT_EPOCHS
|
||||
.lock()
|
||||
.expect("peer epoch cache lock must not be poisoned")
|
||||
.remove(audience);
|
||||
clear_peer_capability(audience);
|
||||
let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
|
||||
let service = EpochProofService {
|
||||
audience: audience.to_string(),
|
||||
include_capability: true,
|
||||
seen_headers: seen_headers.clone(),
|
||||
};
|
||||
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
|
||||
let make_request = || {
|
||||
let mut request = HttpRequest::builder()
|
||||
.uri("/node_service.NodeService/Ping")
|
||||
.body(())
|
||||
.expect("test RPC request must build");
|
||||
request.headers_mut().extend(
|
||||
gen_tonic_signature_headers(audience, "node_service.NodeService", "Ping", None)
|
||||
.expect("v2 test headers must mint"),
|
||||
);
|
||||
request
|
||||
};
|
||||
let make_request = || replay_scope_request(audience, "Ping");
|
||||
|
||||
futures::executor::block_on(channel.call(make_request())).expect("first request must complete");
|
||||
futures::executor::block_on(channel.call(make_request())).expect("second request must complete");
|
||||
@@ -619,10 +1206,7 @@ mod tests {
|
||||
headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
|
||||
"the second request must carry the replay-scoped v3 signature"
|
||||
);
|
||||
PEER_BOOT_EPOCHS
|
||||
.lock()
|
||||
.expect("peer epoch cache lock must not be poisoned")
|
||||
.remove(audience);
|
||||
clear_peer_capability(audience);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -40,8 +40,11 @@ use http::{HeaderMap, HeaderValue, Method, Uri};
|
||||
use rustfs_credentials::{DEFAULT_SECRET_KEY, RPC_SECRET_REQUIRED_MESSAGE};
|
||||
use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token};
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_OPERATION_GRPC_OTHER, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics,
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_FORCE_UNLOCK, INTERNODE_OPERATION_GRPC_LOCK,
|
||||
INTERNODE_OPERATION_GRPC_LOCK_BATCH, INTERNODE_OPERATION_GRPC_OTHER, INTERNODE_OPERATION_GRPC_READ_ALL,
|
||||
INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_REFRESH,
|
||||
INTERNODE_OPERATION_GRPC_UNLOCK, INTERNODE_OPERATION_GRPC_UNLOCK_BATCH, INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics,
|
||||
};
|
||||
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
|
||||
use rustfs_utils::get_env_bool;
|
||||
@@ -70,10 +73,14 @@ pub const RPC_REPLAY_SCOPE_NONCE_HEADER: &str = "x-rustfs-rpc-replay-nonce";
|
||||
pub const RPC_BOOT_EPOCH_HEADER: &str = "x-rustfs-rpc-boot-epoch";
|
||||
pub const RPC_BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
|
||||
pub const RPC_BOOT_EPOCH_PROOF_HEADER: &str = "x-rustfs-rpc-boot-epoch-proof";
|
||||
pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_HEADER: &str = "x-rustfs-rpc-replay-cache-capability";
|
||||
pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER: &str = "x-rustfs-rpc-replay-cache-capability-proof";
|
||||
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
|
||||
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
|
||||
const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
|
||||
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
|
||||
const RPC_REPLAY_CACHE_CAPABILITY_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-replay-cache-capability-proof-v1\0";
|
||||
const RPC_REPLAY_CACHE_CAPABILITY_V1: &str = "dynamic-replay-cache-v1";
|
||||
const HTTP_PUT_FILE_AUTH_DOMAIN: &[u8] = b"rustfs-http-put-file-auth-v1\0";
|
||||
const HTTP_PUT_FILE_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-http-put-file-capability-v1\0";
|
||||
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
@@ -82,8 +89,9 @@ const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
|
||||
const REPLAY_CACHE_RETENTION: Duration = Duration::from_secs(601);
|
||||
const REPLAY_CACHE_RETENTION_SECS: usize = 601;
|
||||
const REPLAY_CACHE_ENTRY_BYTES_ESTIMATE: u64 = 128;
|
||||
const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 8;
|
||||
const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 2048;
|
||||
// Keep 16 CPU / 32 GiB field nodes at the 32M cap without requiring an env override.
|
||||
const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
|
||||
const REPLAY_CACHE_AUTO_RPC_RPS_PER_CPU: usize = 4096;
|
||||
const REPLAY_CACHE_AUTO_MAX_CAPACITY: usize = 33_554_432;
|
||||
const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
|
||||
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
|
||||
@@ -99,6 +107,10 @@ static INTERNODE_RPC_BODY_DIGEST_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
|
||||
)
|
||||
});
|
||||
|
||||
pub(crate) fn internode_rpc_body_digest_strict() -> bool {
|
||||
*INTERNODE_RPC_BODY_DIGEST_STRICT
|
||||
}
|
||||
static INTERNODE_RPC_REPLAY_SCOPE_STRICT: LazyLock<bool> = LazyLock::new(|| {
|
||||
get_env_bool(
|
||||
rustfs_config::ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
|
||||
@@ -340,6 +352,7 @@ struct RpcNonceCacheMetrics<'a> {
|
||||
expired: usize,
|
||||
entries: usize,
|
||||
capacity: usize,
|
||||
record_scope: Option<RpcReplayCacheMetricScope<'a>>,
|
||||
overflow_scope: Option<RpcReplayCacheMetricScope<'a>>,
|
||||
}
|
||||
|
||||
@@ -350,6 +363,13 @@ fn publish_nonce_cache_metrics(metrics: Option<RpcNonceCacheMetrics<'_>>) {
|
||||
let internode_metrics = global_internode_metrics();
|
||||
internode_metrics.record_replay_cache_evictions("expired", metrics.expired);
|
||||
internode_metrics.record_replay_cache_state(metrics.entries, metrics.capacity);
|
||||
if let Some(scope) = metrics.record_scope {
|
||||
internode_metrics.record_replay_cache_record_for_operation_and_backend_path(
|
||||
scope.operation,
|
||||
scope.backend,
|
||||
scope.rpc_path,
|
||||
);
|
||||
}
|
||||
if let Some(scope) = metrics.overflow_scope {
|
||||
internode_metrics.record_replay_cache_overflow_for_operation_and_backend_path(
|
||||
scope.operation,
|
||||
@@ -385,6 +405,7 @@ impl RpcNonceCache {
|
||||
expired,
|
||||
entries: self.nonces.len(),
|
||||
capacity: record.capacity,
|
||||
record_scope: None,
|
||||
overflow_scope: None,
|
||||
};
|
||||
if self.nonces.contains(&record.nonce) {
|
||||
@@ -409,6 +430,7 @@ impl RpcNonceCache {
|
||||
Ok(()),
|
||||
Some(RpcNonceCacheMetrics {
|
||||
entries: self.nonces.len(),
|
||||
record_scope: Some(record.metric_scope),
|
||||
..metrics
|
||||
}),
|
||||
)
|
||||
@@ -776,6 +798,50 @@ fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_e
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC boot epoch proof"))
|
||||
}
|
||||
|
||||
fn update_replay_cache_capability_proof(mac: &mut HmacSha256, audience: &str, challenge: Uuid, boot_epoch: Uuid) {
|
||||
mac.update(RPC_REPLAY_CACHE_CAPABILITY_PROOF_DOMAIN);
|
||||
for part in [
|
||||
audience.as_bytes(),
|
||||
b"|",
|
||||
challenge.as_bytes(),
|
||||
b"|",
|
||||
boot_epoch.as_bytes(),
|
||||
b"|",
|
||||
RPC_REPLAY_CACHE_CAPABILITY_V1.as_bytes(),
|
||||
] {
|
||||
mac.update(part);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_replay_cache_capability_proof(
|
||||
secret: &str,
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
boot_epoch: Uuid,
|
||||
) -> std::io::Result<String> {
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_replay_cache_capability_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_replay_cache_capability_proof(
|
||||
secret: &str,
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
boot_epoch: Uuid,
|
||||
proof: &str,
|
||||
) -> std::io::Result<()> {
|
||||
let proof = general_purpose::STANDARD
|
||||
.decode(proof)
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC replay cache capability proof"))?;
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_replay_cache_capability_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
mac.verify_slice(&proof)
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC replay cache capability proof"))
|
||||
}
|
||||
|
||||
fn non_nil_uuid(value: &str, name: &str) -> std::io::Result<Uuid> {
|
||||
let value = Uuid::parse_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name}")))?;
|
||||
(!value.is_nil())
|
||||
@@ -858,15 +924,34 @@ pub fn tonic_boot_epoch_challenge(headers: &HeaderMap) -> std::io::Result<Option
|
||||
/// Build the authenticated response headers for a client boot-epoch challenge.
|
||||
pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std::io::Result<HeaderMap> {
|
||||
let boot_epoch = tonic_rpc_boot_epoch();
|
||||
let proof = generate_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch)?;
|
||||
let secret = get_shared_secret()?;
|
||||
let proof = generate_boot_epoch_proof(&secret, audience, challenge, boot_epoch)?;
|
||||
let capability_proof = generate_replay_cache_capability_proof(&secret, audience, challenge, boot_epoch)?;
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
|
||||
headers.insert(RPC_BOOT_EPOCH_PROOF_HEADER, header_value(&proof, RPC_BOOT_EPOCH_PROOF_HEADER)?);
|
||||
headers.insert(
|
||||
RPC_REPLAY_CACHE_CAPABILITY_HEADER,
|
||||
HeaderValue::from_static(RPC_REPLAY_CACHE_CAPABILITY_V1),
|
||||
);
|
||||
headers.insert(
|
||||
RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
|
||||
header_value(&capability_proof, RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER)?,
|
||||
);
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// Verify the server boot-epoch response for a challenge generated by this client.
|
||||
pub fn verify_tonic_boot_epoch_response(audience: &str, challenge: Uuid, headers: &HeaderMap) -> std::io::Result<Uuid> {
|
||||
verify_tonic_boot_epoch_response_with_secret(&get_shared_secret()?, audience, challenge, headers)
|
||||
}
|
||||
|
||||
fn verify_tonic_boot_epoch_response_with_secret(
|
||||
secret: &str,
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
headers: &HeaderMap,
|
||||
) -> std::io::Result<Uuid> {
|
||||
let boot_epoch = headers
|
||||
.get(RPC_BOOT_EPOCH_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
@@ -876,10 +961,47 @@ pub fn verify_tonic_boot_epoch_response(audience: &str, challenge: Uuid, headers
|
||||
.get(RPC_BOOT_EPOCH_PROOF_HEADER)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch proof"))?;
|
||||
verify_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch, proof)?;
|
||||
verify_boot_epoch_proof(secret, audience, challenge, boot_epoch, proof)?;
|
||||
Ok(boot_epoch)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct AuthenticatedPeerReplayCapabilities {
|
||||
pub(crate) boot_epoch: Uuid,
|
||||
pub(crate) dynamic_replay_cache: bool,
|
||||
}
|
||||
|
||||
pub(crate) fn verify_tonic_peer_replay_capabilities_response(
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
headers: &HeaderMap,
|
||||
) -> std::io::Result<AuthenticatedPeerReplayCapabilities> {
|
||||
let secret = get_shared_secret()?;
|
||||
let boot_epoch = verify_tonic_boot_epoch_response_with_secret(&secret, audience, challenge, headers)?;
|
||||
let capability = headers.get(RPC_REPLAY_CACHE_CAPABILITY_HEADER);
|
||||
let proof = headers.get(RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER);
|
||||
if capability.is_none() && proof.is_none() {
|
||||
return Ok(AuthenticatedPeerReplayCapabilities {
|
||||
boot_epoch,
|
||||
dynamic_replay_cache: false,
|
||||
});
|
||||
}
|
||||
let capability = capability
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC replay cache capability"))?;
|
||||
if capability != RPC_REPLAY_CACHE_CAPABILITY_V1 {
|
||||
return Err(std::io::Error::other("Unsupported RPC replay cache capability"));
|
||||
}
|
||||
let proof = proof
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC replay cache capability proof"))?;
|
||||
verify_replay_cache_capability_proof(&secret, audience, challenge, boot_epoch, proof)?;
|
||||
Ok(AuthenticatedPeerReplayCapabilities {
|
||||
boot_epoch,
|
||||
dynamic_replay_cache: true,
|
||||
})
|
||||
}
|
||||
|
||||
fn valid_content_sha256(value: &str) -> bool {
|
||||
value == UNSIGNED_PAYLOAD
|
||||
|| (value.len() == 64
|
||||
@@ -913,7 +1035,15 @@ fn tonic_rpc_metric_operation(path: &str) -> &'static str {
|
||||
match parse_tonic_rpc_path(path).ok().map(|(_, rpc_method)| rpc_method) {
|
||||
Some("ReadAll") => INTERNODE_OPERATION_GRPC_READ_ALL,
|
||||
Some("ReadMultiple") => INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
Some("ReadVersion") => INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
Some("BatchReadVersion") => INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
Some("WriteAll") => INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
Some("Lock") => INTERNODE_OPERATION_GRPC_LOCK,
|
||||
Some("UnLock") => INTERNODE_OPERATION_GRPC_UNLOCK,
|
||||
Some("LockBatch") => INTERNODE_OPERATION_GRPC_LOCK_BATCH,
|
||||
Some("UnLockBatch") => INTERNODE_OPERATION_GRPC_UNLOCK_BATCH,
|
||||
Some("Refresh") => INTERNODE_OPERATION_GRPC_REFRESH,
|
||||
Some("ForceUnLock") => INTERNODE_OPERATION_GRPC_FORCE_UNLOCK,
|
||||
_ => INTERNODE_OPERATION_GRPC_OTHER,
|
||||
}
|
||||
}
|
||||
@@ -1082,6 +1212,23 @@ pub fn set_tonic_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(
|
||||
set_tonic_canonical_body_digest(request, &canonical_body)
|
||||
}
|
||||
|
||||
pub fn set_tonic_rolling_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(
|
||||
request: &mut tonic::Request<T>,
|
||||
) -> std::io::Result<()> {
|
||||
set_tonic_mutation_body_digest(request)?;
|
||||
request.extensions_mut().insert(RollingMutationBodyDigest);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn set_tonic_rolling_canonical_body_digest<T>(request: &mut tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
set_tonic_canonical_body_digest(request, canonical_body)?;
|
||||
request.extensions_mut().insert(RollingMutationBodyDigest);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) struct RollingMutationBodyDigest;
|
||||
|
||||
pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
let version = request
|
||||
.metadata()
|
||||
@@ -1118,7 +1265,7 @@ pub fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canoni
|
||||
/// including v1-downgraded ones. It converges independently of the signature-strict switch
|
||||
/// (<https://github.com/rustfs/backlog/issues/1327>).
|
||||
pub fn verify_tonic_mutation_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
verify_tonic_mutation_body_digest_with_strictness(request, canonical_body, *INTERNODE_RPC_BODY_DIGEST_STRICT)
|
||||
verify_tonic_mutation_body_digest_with_strictness(request, canonical_body, internode_rpc_body_digest_strict())
|
||||
}
|
||||
|
||||
/// [`verify_tonic_mutation_body_digest`] with the strict gate injected as a parameter, so both
|
||||
@@ -2171,6 +2318,23 @@ mod tests {
|
||||
assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cache_capability_proof_binds_audience_challenge_epoch_and_value() {
|
||||
ensure_test_rpc_secret();
|
||||
let challenge = Uuid::new_v4();
|
||||
let headers = tonic_boot_epoch_response_headers("node-a:9000", challenge).expect("capability headers should build");
|
||||
let capabilities = verify_tonic_peer_replay_capabilities_response("node-a:9000", challenge, &headers)
|
||||
.expect("matching capability proof should verify");
|
||||
assert_eq!(capabilities.boot_epoch, tonic_rpc_boot_epoch());
|
||||
assert!(capabilities.dynamic_replay_cache);
|
||||
assert!(verify_tonic_peer_replay_capabilities_response("node-b:9000", challenge, &headers).is_err());
|
||||
assert!(verify_tonic_peer_replay_capabilities_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
|
||||
|
||||
let mut changed_capability = headers;
|
||||
changed_capability.insert(RPC_REPLAY_CACHE_CAPABILITY_HEADER, HeaderValue::from_static("dynamic-replay-cache-v2"));
|
||||
assert!(verify_tonic_peer_replay_capabilities_response("node-a:9000", challenge, &changed_capability).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tonic_rpc_auth_failure_reason_maps_security_relevant_errors() {
|
||||
for (message, reason) in [
|
||||
@@ -2457,10 +2621,42 @@ mod tests {
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/ReadMultiple"),
|
||||
INTERNODE_OPERATION_GRPC_READ_MULTIPLE
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/ReadVersion"),
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/BatchReadVersion"),
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/WriteAll"),
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/Lock"),
|
||||
INTERNODE_OPERATION_GRPC_LOCK
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/UnLock"),
|
||||
INTERNODE_OPERATION_GRPC_UNLOCK
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/LockBatch"),
|
||||
INTERNODE_OPERATION_GRPC_LOCK_BATCH
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/UnLockBatch"),
|
||||
INTERNODE_OPERATION_GRPC_UNLOCK_BATCH
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/Refresh"),
|
||||
INTERNODE_OPERATION_GRPC_REFRESH
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/ForceUnLock"),
|
||||
INTERNODE_OPERATION_GRPC_FORCE_UNLOCK
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/SignalService"),
|
||||
INTERNODE_OPERATION_GRPC_OTHER
|
||||
@@ -2499,21 +2695,27 @@ mod tests {
|
||||
|
||||
assert_eq!(decision.source, ReplayCacheCapacitySource::Auto);
|
||||
assert_eq!(decision.memory_basis, Some(MemoryBasis::Host));
|
||||
assert_eq!(decision.memory_based_capacity, 10_737_418);
|
||||
assert_eq!(decision.cpu_based_capacity, 9_846_784);
|
||||
assert_eq!(decision.capacity, 9_846_784);
|
||||
assert_eq!(decision.memory_based_capacity, 17_448_304);
|
||||
assert_eq!(decision.cpu_based_capacity, 19_693_568);
|
||||
assert_eq!(decision.capacity, 17_448_304);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_cache_capacity_auto_uses_resource_model_on_larger_nodes() {
|
||||
fn replay_cache_capacity_auto_uses_32m_on_field_sized_nodes() {
|
||||
let gib = 1024_u64 * 1024 * 1024;
|
||||
let decision =
|
||||
replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Absent, 16, Some(32 * gib), Some(MemoryBasis::Host));
|
||||
|
||||
assert_eq!(decision.source, ReplayCacheCapacitySource::Auto);
|
||||
assert_eq!(decision.memory_based_capacity, 21_474_836);
|
||||
assert_eq!(decision.cpu_based_capacity, 19_693_568);
|
||||
assert_eq!(decision.capacity, 19_693_568);
|
||||
assert_eq!(decision.memory_based_capacity, 34_896_609);
|
||||
assert_eq!(decision.cpu_based_capacity, 39_387_136);
|
||||
assert_eq!(decision.capacity, REPLAY_CACHE_AUTO_MAX_CAPACITY);
|
||||
|
||||
let observed_field_node =
|
||||
replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Absent, 16, Some(31 * gib), Some(MemoryBasis::Host));
|
||||
assert_eq!(observed_field_node.memory_based_capacity, 33_806_090);
|
||||
assert_eq!(observed_field_node.cpu_based_capacity, 39_387_136);
|
||||
assert_eq!(observed_field_node.capacity, REPLAY_CACHE_AUTO_MAX_CAPACITY);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2545,7 +2747,7 @@ mod tests {
|
||||
let decision = replay_cache_capacity_decision(rustfs_utils::EnvParseOutcome::Invalid, 8, None, None);
|
||||
|
||||
assert_eq!(decision.source, ReplayCacheCapacitySource::AutoInvalidEnv);
|
||||
assert_eq!(decision.capacity, 9_846_784);
|
||||
assert_eq!(decision.capacity, 19_693_568);
|
||||
}
|
||||
|
||||
fn check_test_nonce_record(cache: &mut RpcNonceCache, record: RpcNonceRecord<'_>) -> std::io::Result<()> {
|
||||
@@ -2554,6 +2756,13 @@ mod tests {
|
||||
result
|
||||
}
|
||||
|
||||
fn check_test_nonce_record_with_metrics<'a>(
|
||||
cache: &mut RpcNonceCache,
|
||||
record: RpcNonceRecord<'a>,
|
||||
) -> (std::io::Result<()>, Option<RpcNonceCacheMetrics<'a>>) {
|
||||
cache.check_and_record(record)
|
||||
}
|
||||
|
||||
fn test_nonce_record(
|
||||
nonce: Uuid,
|
||||
signed_at: i64,
|
||||
@@ -2597,6 +2806,48 @@ mod tests {
|
||||
assert!(cache.nonces.contains(&nonce_b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonce_cache_metrics_mark_successful_records_only() {
|
||||
let now = Instant::now();
|
||||
let expiry = now.checked_add(REPLAY_CACHE_RETENTION).expect("test expiry should fit");
|
||||
let nonce_a = Uuid::new_v4();
|
||||
let nonce_b = Uuid::new_v4();
|
||||
let mut cache = RpcNonceCache::default();
|
||||
|
||||
let (recorded, metrics) =
|
||||
check_test_nonce_record_with_metrics(&mut cache, test_nonce_record(nonce_a, 100, now, 100, expiry, 1));
|
||||
recorded.expect("first nonce should be recorded");
|
||||
let metrics = metrics.expect("successful nonce should publish metrics");
|
||||
let record_scope = metrics.record_scope.expect("successful nonce should carry record scope");
|
||||
assert_eq!(record_scope.operation, INTERNODE_OPERATION_GRPC_READ_ALL);
|
||||
assert_eq!(record_scope.backend, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
assert_eq!(record_scope.rpc_path, "/node_service.NodeService/ReadAll");
|
||||
assert!(metrics.overflow_scope.is_none());
|
||||
|
||||
let (replay, metrics) =
|
||||
check_test_nonce_record_with_metrics(&mut cache, test_nonce_record(nonce_a, 100, now, 100, expiry, 1));
|
||||
assert_eq!(
|
||||
replay.expect_err("duplicate nonce must fail closed").to_string(),
|
||||
"RPC request replay detected"
|
||||
);
|
||||
let metrics = metrics.expect("replay rejection should still publish cache state");
|
||||
assert!(metrics.record_scope.is_none());
|
||||
assert!(metrics.overflow_scope.is_none());
|
||||
|
||||
let (overflow, metrics) =
|
||||
check_test_nonce_record_with_metrics(&mut cache, test_nonce_record(nonce_b, 100, now, 100, expiry, 1));
|
||||
assert_eq!(
|
||||
overflow.expect_err("full cache must fail closed").to_string(),
|
||||
"RPC replay cache capacity exceeded"
|
||||
);
|
||||
let metrics = metrics.expect("overflow should publish cache state");
|
||||
assert!(metrics.record_scope.is_none());
|
||||
let overflow_scope = metrics.overflow_scope.expect("overflow should keep diagnostic scope");
|
||||
assert_eq!(overflow_scope.operation, INTERNODE_OPERATION_GRPC_READ_ALL);
|
||||
assert_eq!(overflow_scope.backend, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
assert_eq!(overflow_scope.rpc_path, "/node_service.NodeService/ReadAll");
|
||||
}
|
||||
|
||||
// The `rpc_body_digest_fallback_counter` serial group covers every test that drives (or
|
||||
// asserts on) the process-global body-digest fallback counter, so exact-delta assertions
|
||||
// cannot race with each other.
|
||||
|
||||
@@ -34,11 +34,12 @@ pub use client::{
|
||||
pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
set_tonic_mutation_body_digest, sign_ns_scanner_capability, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
|
||||
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
|
||||
sign_ns_scanner_capability, sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
|
||||
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_put_file_auth_trailer,
|
||||
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
|
||||
@@ -16,7 +16,6 @@ use crate::cluster::rpc::client::{
|
||||
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
|
||||
node_service_time_out_client, node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
|
||||
};
|
||||
use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
|
||||
use crate::cluster::rpc::internode_data_transport::{
|
||||
InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest,
|
||||
WriteStreamRequest,
|
||||
@@ -123,7 +122,7 @@ fn attach_mutation_body_digest<T>(
|
||||
op: &'static str,
|
||||
) -> Result<()> {
|
||||
let canonical_body = canonical_body.map_err(|_| Error::other(format!("{op} request length cannot be represented")))?;
|
||||
set_tonic_canonical_body_digest(request, &canonical_body).map_err(Error::other)
|
||||
crate::cluster::rpc::set_tonic_rolling_canonical_body_digest(request, &canonical_body).map_err(Error::other)
|
||||
}
|
||||
|
||||
fn decode_volume_infos(volume_infos: Vec<String>) -> Result<Vec<VolumeInfo>> {
|
||||
@@ -3029,6 +3028,22 @@ mod tests {
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
#[test]
|
||||
fn disk_mutation_digest_marks_rolling_compatibility() {
|
||||
let mut request = Request::new(());
|
||||
|
||||
attach_mutation_body_digest(&mut request, Ok(b"canonical disk mutation".to_vec()), "WriteAll")
|
||||
.expect("disk mutation digest must be attached");
|
||||
|
||||
assert!(
|
||||
request
|
||||
.extensions()
|
||||
.get::<crate::cluster::rpc::http_auth::RollingMutationBodyDigest>()
|
||||
.is_some(),
|
||||
"remote-disk mutations must reach the cache-free compatibility gate"
|
||||
);
|
||||
}
|
||||
|
||||
// `#[serial(internode_metrics)]` marks every test that observes
|
||||
// `global_internode_metrics()`. Those counters are a process-wide singleton:
|
||||
// some of these tests snapshot a counter, run one decode, and assert on the
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use crate::cluster::rpc::client::{
|
||||
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::set_tonic_mutation_body_digest;
|
||||
use crate::cluster::rpc::set_tonic_rolling_mutation_body_digest;
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use rustfs_lock::{
|
||||
@@ -33,6 +33,10 @@ use tonic::Request;
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(request: &mut Request<T>) -> std::io::Result<()> {
|
||||
set_tonic_rolling_mutation_body_digest(request)
|
||||
}
|
||||
|
||||
/// Remote lock client implementation
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RemoteClient {
|
||||
@@ -319,7 +323,7 @@ impl LockClient for RemoteClient {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
|
||||
let resp = match self.execute_rpc("lock", &resource_summary, client.lock(req)).await {
|
||||
Ok(resp) => resp.into_inner(),
|
||||
@@ -358,7 +362,7 @@ impl LockClient for RemoteClient {
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
|
||||
let resp = match self
|
||||
.execute_rpc("lock_batch", &resource_summary, client.lock_batch(req))
|
||||
@@ -400,7 +404,7 @@ impl LockClient for RemoteClient {
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = unlock_request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest { args: request_string });
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let resp = self
|
||||
.execute_rpc("release", &resource_summary, client.un_lock(req))
|
||||
.await?
|
||||
@@ -427,7 +431,7 @@ impl LockClient for RemoteClient {
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
|
||||
let resp = self
|
||||
.execute_rpc("release_batch", &resource_summary, client.un_lock_batch(req))
|
||||
@@ -450,7 +454,7 @@ impl LockClient for RemoteClient {
|
||||
args: serde_json::to_string(&refresh_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let resp = self
|
||||
.execute_rpc("refresh", &resource_summary, client.refresh(req))
|
||||
.await?
|
||||
@@ -470,7 +474,7 @@ impl LockClient for RemoteClient {
|
||||
args: serde_json::to_string(&force_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let resp = self
|
||||
.execute_rpc("force_release", &resource_summary, client.force_un_lock(req))
|
||||
.await?
|
||||
@@ -495,7 +499,7 @@ impl LockClient for RemoteClient {
|
||||
args: serde_json::to_string(&status_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut req)?;
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
|
||||
// Try exclusive lock first with very short timeout
|
||||
let resp = match self.execute_rpc("check_status", &resource_summary, client.lock(req)).await {
|
||||
@@ -510,7 +514,7 @@ impl LockClient for RemoteClient {
|
||||
args: serde_json::to_string(&status_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut release_req)?;
|
||||
attach_lock_mutation_body_digest(&mut release_req)?;
|
||||
let _ = self
|
||||
.execute_rpc("check_status_release", &resource_summary, client.un_lock(release_req))
|
||||
.await;
|
||||
@@ -626,6 +630,31 @@ mod tests {
|
||||
.with_priority(LockPriority::Normal)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lock_mutation_helper_marks_single_and_batch_requests_for_rolling_auth() {
|
||||
let mut single = Request::new(GenerallyLockRequest {
|
||||
args: "single-lock".to_string(),
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut single).expect("single lock digest must be attached");
|
||||
assert!(
|
||||
single
|
||||
.extensions()
|
||||
.get::<crate::cluster::rpc::http_auth::RollingMutationBodyDigest>()
|
||||
.is_some()
|
||||
);
|
||||
|
||||
let mut batch = Request::new(BatchGenerallyLockRequest {
|
||||
args: vec!["batch-lock".to_string()],
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut batch).expect("batch lock digest must be attached");
|
||||
assert!(
|
||||
batch
|
||||
.extensions()
|
||||
.get::<crate::cluster::rpc::http_auth::RollingMutationBodyDigest>()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn test_remote_client_acquire_lock_uses_rpc_timeout_and_evicts_connection() {
|
||||
|
||||
@@ -584,6 +584,44 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
/// `delete_config` with `no_lock` set — for callers already holding the
|
||||
/// config object's namespace lock (e.g. inside `with_config_object_write_lock`),
|
||||
/// where the locked variant would self-deadlock.
|
||||
pub async fn delete_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
match api
|
||||
.delete_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
file,
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) => {
|
||||
if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) {
|
||||
Err(Error::ConfigNotFound)
|
||||
} else {
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(skip(api))]
|
||||
pub async fn delete_config<S>(api: Arc<S>, file: &str) -> Result<()>
|
||||
where
|
||||
|
||||
@@ -1638,7 +1638,7 @@ fn preserve_unknown_dirty_usage(
|
||||
Some(preserved)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
async fn replace_bucket_usage_memory_from_authoritative(bucket: &str, usage: BucketUsageInfo, refresh_started_at: SystemTime) {
|
||||
let mut cache = memory_cache().write().await;
|
||||
if let Some(existing) = cache.get(bucket)
|
||||
@@ -1650,6 +1650,19 @@ async fn replace_bucket_usage_memory_from_authoritative(bucket: &str, usage: Buc
|
||||
cache.insert(bucket.to_string(), cached_bucket_usage_from_backend(usage, refresh_started_at, true));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn seed_bucket_usage_memory_for_test(bucket: &str, size: u64) {
|
||||
replace_bucket_usage_memory_from_authoritative(
|
||||
bucket,
|
||||
BucketUsageInfo {
|
||||
size,
|
||||
..Default::default()
|
||||
},
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Fast in-memory update for immediate quota and admin usage consistency.
|
||||
pub async fn record_bucket_object_write_memory(bucket: &str, previous_current_size: Option<u64>, new_size: u64) {
|
||||
record_bucket_object_write_memory_inner(bucket, previous_current_size, new_size, false).await;
|
||||
@@ -2137,30 +2150,6 @@ pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str
|
||||
Ok(d)
|
||||
}
|
||||
|
||||
#[instrument(skip(cache))]
|
||||
pub async fn save_data_usage_cache(cache: &DataUsageCache, name: &str) -> crate::error::Result<()> {
|
||||
use crate::config::com::save_config;
|
||||
use crate::disk::BUCKET_META_PREFIX;
|
||||
use std::path::Path;
|
||||
|
||||
let Some(store) = runtime_sources::object_store_handle() else {
|
||||
return Err(Error::other("errServerNotInitialized"));
|
||||
};
|
||||
let buf = cache.marshal_msg().map_err(Error::other)?;
|
||||
let buf_clone = buf.clone();
|
||||
|
||||
let store_clone = store.clone();
|
||||
|
||||
let name = Path::new(BUCKET_META_PREFIX).join(name).to_string_lossy().to_string();
|
||||
|
||||
let name_clone = name.clone();
|
||||
tokio::spawn(async move {
|
||||
let _ = save_config(store_clone, &format!("{}{}", name_clone, ".bkp"), buf_clone).await;
|
||||
});
|
||||
save_config(store, &name, buf).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Persist the current in-memory compression total to the backend.
|
||||
/// Resets the debounce counter so the next auto-persist won't fire
|
||||
/// immediately after this manual flush (intended for shutdown paths).
|
||||
|
||||
@@ -12,7 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
use crate::cluster::rpc::{
|
||||
ScannerBucketListing, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
};
|
||||
use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend_cached};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::{
|
||||
@@ -23,6 +25,7 @@ use crate::{
|
||||
|
||||
use crate::data_usage::load_data_usage_cache;
|
||||
use crate::storage_api_contracts::admin::StorageAdminApi;
|
||||
use crate::storage_api_contracts::bucket::BucketOptions;
|
||||
use rustfs_common::heal_channel::DriveState;
|
||||
use rustfs_madmin::{
|
||||
BackendDisks, Disk, ErasureSetInfo, ITEM_INITIALIZING, ITEM_OFFLINE, ITEM_ONLINE, ITEM_UNKNOWN, InfoMessage, MemStats,
|
||||
@@ -74,6 +77,19 @@ fn apply_data_usage_result(
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_bucket_namespace_count(result: Result<ScannerBucketListing>, buckets: &mut rustfs_madmin::Buckets) {
|
||||
if let Ok(listing) = result
|
||||
&& listing.topology_complete
|
||||
{
|
||||
let count = listing.buckets.iter().filter(|bucket| !bucket.name.starts_with('.')).count();
|
||||
let Ok(count) = u64::try_from(count) else {
|
||||
return;
|
||||
};
|
||||
buckets.count = count;
|
||||
buckets.error = None;
|
||||
}
|
||||
}
|
||||
|
||||
// pub const ITEM_OFFLINE: &str = "offline";
|
||||
// pub const ITEM_INITIALIZING: &str = "initializing";
|
||||
// pub const ITEM_ONLINE: &str = "online";
|
||||
@@ -285,6 +301,18 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
&mut delete_markers,
|
||||
&mut usage,
|
||||
);
|
||||
if buckets.error.is_some() {
|
||||
apply_bucket_namespace_count(
|
||||
store
|
||||
.list_bucket_for_scanner(&BucketOptions {
|
||||
cached: true,
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await,
|
||||
&mut buckets,
|
||||
);
|
||||
}
|
||||
|
||||
let after3 = OffsetDateTime::now_utc();
|
||||
|
||||
@@ -705,12 +733,13 @@ mod tests {
|
||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::storage_api_contracts::bucket::BucketInfo;
|
||||
use rustfs_madmin::{Disk, ITEM_OFFLINE, ITEM_ONLINE, ITEM_UNKNOWN, ServerProperties};
|
||||
|
||||
use super::{
|
||||
DATA_USAGE_ROOT, DATA_USAGE_UNAVAILABLE_ERROR, apply_data_usage_result, apply_erasure_set_usage,
|
||||
get_local_server_property, get_online_offline_disks_stats, get_server_info, reconcile_servers_with_endpoint_topology,
|
||||
server_topology_completeness_report,
|
||||
DATA_USAGE_ROOT, DATA_USAGE_UNAVAILABLE_ERROR, apply_bucket_namespace_count, apply_data_usage_result,
|
||||
apply_erasure_set_usage, get_local_server_property, get_online_offline_disks_stats, get_server_info,
|
||||
reconcile_servers_with_endpoint_topology, server_topology_completeness_report,
|
||||
};
|
||||
|
||||
fn disk_with_state(endpoint: &str, state: &str) -> Disk {
|
||||
@@ -960,6 +989,75 @@ mod tests {
|
||||
assert_eq!(usage.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_bucket_namespace_count_survives_unavailable_data_usage() {
|
||||
let mut buckets = rustfs_madmin::Buckets {
|
||||
count: 0,
|
||||
error: Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
|
||||
};
|
||||
|
||||
apply_bucket_namespace_count(
|
||||
Ok(crate::cluster::rpc::ScannerBucketListing {
|
||||
buckets: vec![
|
||||
BucketInfo {
|
||||
name: "bucket-a".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
BucketInfo {
|
||||
name: ".rustfs.sys".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
BucketInfo {
|
||||
name: "bucket-b".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
set_buckets: Vec::new(),
|
||||
topology_complete: true,
|
||||
}),
|
||||
&mut buckets,
|
||||
);
|
||||
|
||||
assert_eq!(buckets.count, 2);
|
||||
assert_eq!(buckets.error, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incomplete_bucket_namespace_lookup_preserves_usage_state() {
|
||||
let mut buckets = rustfs_madmin::Buckets {
|
||||
count: 7,
|
||||
error: Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
|
||||
};
|
||||
|
||||
apply_bucket_namespace_count(
|
||||
Ok(crate::cluster::rpc::ScannerBucketListing {
|
||||
buckets: vec![BucketInfo {
|
||||
name: "bucket-a".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
set_buckets: Vec::new(),
|
||||
topology_complete: false,
|
||||
}),
|
||||
&mut buckets,
|
||||
);
|
||||
|
||||
assert_eq!(buckets.count, 7);
|
||||
assert_eq!(buckets.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_bucket_namespace_lookup_preserves_usage_state() {
|
||||
let mut buckets = rustfs_madmin::Buckets {
|
||||
count: 7,
|
||||
error: Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
|
||||
};
|
||||
|
||||
apply_bucket_namespace_count(Err(crate::error::Error::DiskNotFound), &mut buckets);
|
||||
|
||||
assert_eq!(buckets.count, 7);
|
||||
assert_eq!(buckets.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incomplete_erasure_set_cache_is_not_reported_as_zero() {
|
||||
let mut cache = rustfs_data_usage::DataUsageCache::default();
|
||||
|
||||
@@ -331,7 +331,14 @@ impl From<std::io::Error> for DiskError {
|
||||
}
|
||||
match e.downcast::<DiskError>() {
|
||||
Ok(disk_error) => disk_error,
|
||||
Err(io_error) => DiskError::Io(io_error),
|
||||
// Mirror `From<io::Error> for StorageError`: a StorageError boxed
|
||||
// through `From<StorageError> for io::Error` must recover its typed
|
||||
// classification instead of degrading to `DiskError::Io`, which
|
||||
// quorum aggregation (`reduce_errs`) would count as a distinct error.
|
||||
Err(io_error) => match io_error.downcast::<crate::error::StorageError>() {
|
||||
Ok(storage_error) => storage_error.into(),
|
||||
Err(io_error) => DiskError::Io(io_error),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -953,6 +960,27 @@ mod tests {
|
||||
assert_eq!(original_disk_error, recovered_disk_error);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_io_error_with_storage_error_inside() {
|
||||
use crate::error::StorageError;
|
||||
|
||||
// An io::Error boxing a disk-representable StorageError (as produced by
|
||||
// `From<StorageError> for io::Error`) must recover the typed DiskError
|
||||
// variant instead of degrading to an opaque DiskError::Io.
|
||||
let io_with_storage_error: std::io::Error = StorageError::FaultyRemoteDisk.into();
|
||||
let recovered: DiskError = io_with_storage_error.into();
|
||||
assert_eq!(recovered, DiskError::FaultyRemoteDisk);
|
||||
|
||||
let io_with_storage_error: std::io::Error = StorageError::FileAccessDenied.into();
|
||||
let recovered: DiskError = io_with_storage_error.into();
|
||||
assert_eq!(recovered, DiskError::FileAccessDenied);
|
||||
|
||||
// A StorageError with no DiskError analog stays an opaque Io error.
|
||||
let io_with_bucket_error: std::io::Error = StorageError::BucketNotFound("bucket".to_string()).into();
|
||||
let recovered: DiskError = io_with_bucket_error.into();
|
||||
assert!(matches!(recovered, DiskError::Io(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_io_error_different_kinds() {
|
||||
use std::io::ErrorKind;
|
||||
|
||||
@@ -2027,6 +2027,9 @@ type InlinePreparationHook = Box<dyn FnOnce() + Send>;
|
||||
static INLINE_PREPARATION_BEFORE_BACKUP: std::sync::LazyLock<std::sync::Mutex<HashMap<String, InlinePreparationHook>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
#[cfg(test)]
|
||||
static INLINE_BEFORE_FILE_SYNC_ADMISSION: std::sync::LazyLock<std::sync::Mutex<HashMap<String, InlinePreparationHook>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
#[cfg(test)]
|
||||
static RENAME_DATA_AFTER_FIRST_PUBLICATION: std::sync::LazyLock<std::sync::Mutex<HashMap<String, InlinePreparationHook>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
#[cfg(test)]
|
||||
@@ -2091,6 +2094,14 @@ fn set_inline_preparation_before_backup(dst_path: &str, hook: impl FnOnce() + Se
|
||||
.insert(dst_path.to_string(), Box::new(hook));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn set_inline_before_file_sync_admission(dst_path: &str, hook: impl FnOnce() + Send + 'static) {
|
||||
INLINE_BEFORE_FILE_SYNC_ADMISSION
|
||||
.lock()
|
||||
.expect("test admission hook lock should not be poisoned")
|
||||
.insert(dst_path.to_string(), Box::new(hook));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn set_rename_data_after_first_publication(dst_path: &str, hook: impl FnOnce() + Send + 'static) {
|
||||
RENAME_DATA_AFTER_FIRST_PUBLICATION
|
||||
@@ -2236,6 +2247,17 @@ fn run_inline_preparation_before_backup(dst_path: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn run_inline_before_file_sync_admission(dst_path: &str) {
|
||||
let hook = INLINE_BEFORE_FILE_SYNC_ADMISSION
|
||||
.lock()
|
||||
.expect("test admission hook lock should not be poisoned")
|
||||
.remove(dst_path);
|
||||
if let Some(hook) = hook {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn run_rename_data_after_first_publication(dst_path: &str) {
|
||||
let hook = RENAME_DATA_AFTER_FIRST_PUBLICATION
|
||||
@@ -2907,23 +2929,82 @@ pub(crate) trait LocalIoBackend: Send + Sync + Debug + 'static {
|
||||
/// Default [`LocalIoBackend`]: tokio blocking-pool file I/O plus the
|
||||
/// mmap-copy / direct-read-copy positioned read, moved verbatim from the
|
||||
/// former `DiskAPI` method bodies on `LocalDisk`.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct StdBackend {
|
||||
root: PathBuf,
|
||||
#[cfg(target_os = "linux")]
|
||||
direct_io: Arc<DirectIoReadState>,
|
||||
#[cfg(target_os = "linux")]
|
||||
direct_io_write: Arc<DirectIoWriteState>,
|
||||
/// Per-disk descriptor cache for buffered reads (rustfs/backlog#1801).
|
||||
/// `None` when disabled by env, blocked by a low `RLIMIT_NOFILE`, or on
|
||||
/// non-Linux (where the cache type is unavailable). Like the io_uring
|
||||
/// cache, only the buffered read path populates it; O_DIRECT reads keep
|
||||
/// opening their own aligned descriptors.
|
||||
#[cfg(target_os = "linux")]
|
||||
fd_cache: Option<FdCache>,
|
||||
}
|
||||
|
||||
// Manual `Debug` mirrors `UringBackend`: the fd cache (and the Linux-only
|
||||
// direct-IO state) hold types that do not implement `Debug`, so a derive would
|
||||
// force `FdCache: Debug`. `finish_non_exhaustive` skips them.
|
||||
impl std::fmt::Debug for StdBackend {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("StdBackend").field("root", &self.root).finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl StdBackend {
|
||||
pub(crate) fn new(root: PathBuf) -> Self {
|
||||
Self::build(root, true)
|
||||
}
|
||||
|
||||
/// Construct without the descriptor cache.
|
||||
///
|
||||
/// `UringBackend` wraps a `StdBackend` and runs its own `FdCache` over the
|
||||
/// same positioned reads. If the inner `StdBackend` also built a cache, a
|
||||
/// fallback read (`UringBackend::pread_bytes` delegates to the inner backend
|
||||
/// on latch-off / O_DIRECT / buffered errors) would populate a *second*
|
||||
/// cache that `UringBackend`'s invalidation never touches — re-opening the
|
||||
/// stale-inode hazard `FdCache` exists to close (rustfs/backlog#1176/#1801).
|
||||
/// The wrapper therefore owns the only cache for the disk; the inner backend
|
||||
/// opens per read. This also avoids double-counting `FD_CACHE_CAPACITY`
|
||||
/// against `RLIMIT_NOFILE` (rustfs/backlog#1178).
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(crate) fn new_without_fd_cache(root: PathBuf) -> Self {
|
||||
Self::build(root, false)
|
||||
}
|
||||
|
||||
fn build(root: PathBuf, build_fd_cache: bool) -> Self {
|
||||
// Gate the fd cache on RLIMIT_NOFILE headroom (rustfs/backlog#1178):
|
||||
// 512 fds/disk with a low soft limit and several disks would hit EMFILE.
|
||||
// Fall back to open-per-read when the limit is too small.
|
||||
#[cfg(target_os = "linux")]
|
||||
let fd_cache = if build_fd_cache && is_local_fd_cache_enabled() {
|
||||
if rlimit_allows_fd_cache() {
|
||||
Some(FdCache::new())
|
||||
} else {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
"std fd cache disabled: RLIMIT_NOFILE soft limit too low for 512 fds/disk; using open-per-read"
|
||||
);
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// `build_fd_cache` is only consulted on Linux (for the fd cache); on
|
||||
// other platforms it has no effect and would trip the unused-variable lint.
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let _ = build_fd_cache;
|
||||
Self {
|
||||
root,
|
||||
#[cfg(target_os = "linux")]
|
||||
direct_io: Arc::new(DirectIoReadState::new()),
|
||||
#[cfg(target_os = "linux")]
|
||||
direct_io_write: Arc::new(DirectIoWriteState::new()),
|
||||
#[cfg(target_os = "linux")]
|
||||
fd_cache,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3001,6 +3082,9 @@ impl LocalIoBackend for StdBackend {
|
||||
direct_read_copy_fault_delta: MmapPageFaultDelta,
|
||||
blocking_task_duration: StdDuration,
|
||||
used_direct_io: bool,
|
||||
/// The descriptor opened by THIS call (None on a cache hit), handed
|
||||
/// back so the async caller can index it in the fd cache.
|
||||
opened_fd: Option<Arc<std::fs::File>>,
|
||||
}
|
||||
|
||||
enum MmapCopyReadError {
|
||||
@@ -3028,28 +3112,72 @@ impl LocalIoBackend for StdBackend {
|
||||
let direct_io_state = self.direct_io.clone();
|
||||
let offset_u64 = u64::try_from(offset).map_err(|_| DiskError::FileCorrupt)?;
|
||||
let end_offset_u64 = u64::try_from(end_offset).map_err(|_| DiskError::FileCorrupt)?;
|
||||
|
||||
// Descriptor cache (rustfs/backlog#1801): on a hit the read reuses an
|
||||
// already-open descriptor (via dup below) and skips `access` +
|
||||
// `File::open`. Linux-only — on other Unix `cached_fd` is None and the
|
||||
// read opens per call exactly as before. `fd_lookup` snapshots the
|
||||
// invalidation generation BEFORE the open so a heal/delete that lands
|
||||
// while the blocking open is in flight prevents the now-stale descriptor
|
||||
// from being inserted (rustfs/backlog#1176).
|
||||
#[cfg(target_os = "linux")]
|
||||
let fd_lookup = self.fd_cache.as_ref().map(|cache| {
|
||||
let key = FdKey {
|
||||
volume: volume.to_owned(),
|
||||
path: path.to_owned(),
|
||||
direct: false,
|
||||
};
|
||||
let gen_at_open = cache.generation();
|
||||
(cache, key, gen_at_open)
|
||||
});
|
||||
#[cfg(target_os = "linux")]
|
||||
let cached_fd: Option<Arc<std::fs::File>> = match &fd_lookup {
|
||||
Some((cache, key, _)) => cache.get(key).await,
|
||||
None => None,
|
||||
};
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let cached_fd: Option<Arc<std::fs::File>> = None;
|
||||
|
||||
let blocking_wait_start = metrics_enabled.then(std::time::Instant::now);
|
||||
let read_result = tokio::task::spawn_blocking(move || {
|
||||
let blocking_task_start = metrics_enabled.then(StdInstant::now);
|
||||
|
||||
let access_check_start = metrics_enabled.then(StdInstant::now);
|
||||
let volume_dir = local_disk_bucket_path(&root, &volume_owned)?;
|
||||
if !skip_access_checks(&volume_owned) {
|
||||
crate::disk::fs::access_std(&volume_dir)
|
||||
.map_err(|e| DiskError::from(to_access_error(e, DiskError::VolumeAccessDenied)))?;
|
||||
}
|
||||
let access_check_duration = access_check_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
|
||||
// Resolve the part path unconditionally: the O_DIRECT branch (large
|
||||
// reads) opens its own aligned descriptor by path even on a cache hit.
|
||||
let path_resolve_start = metrics_enabled.then(StdInstant::now);
|
||||
let file_path = local_disk_object_path(&root, &volume_owned, &path_owned)?;
|
||||
check_path_length(file_path.to_string_lossy().as_ref())?;
|
||||
let path_resolve_duration = path_resolve_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
|
||||
let file_open_start = metrics_enabled.then(StdInstant::now);
|
||||
let mut file = std::fs::File::open(&file_path).map_err(DiskError::from)?;
|
||||
// Acquire the read handle (rustfs/backlog#1801). On a descriptor-cache
|
||||
// hit this reuses the cached descriptor via `dup` (one syscall, no path
|
||||
// resolution or permission re-check) and skips the volume access probe;
|
||||
// on a miss it resolves the volume, access-checks, and opens the file.
|
||||
// `File::try_clone` shares the cached descriptor's open-file offset, so
|
||||
// the read below is positioned (mmap offset argument / `read_exact_at`)
|
||||
// and never depends on the descriptor's current offset. `cached_fd` being
|
||||
// None also marks this call as a miss for the cache-insert side-channel.
|
||||
let (file, access_check_duration) = if let Some(cached) = cached_fd.as_ref() {
|
||||
(cached.as_ref().try_clone().map_err(DiskError::from)?, StdDuration::ZERO)
|
||||
} else {
|
||||
// Measure the volume access probe only — the part-path resolution
|
||||
// above is accounted in `path_resolve_duration` (rustfs/backlog#1801).
|
||||
let access_check_start = metrics_enabled.then(StdInstant::now);
|
||||
let volume_dir = local_disk_bucket_path(&root, &volume_owned)?;
|
||||
if !skip_access_checks(&volume_owned) {
|
||||
crate::disk::fs::access_std(&volume_dir)
|
||||
.map_err(|e| DiskError::from(to_access_error(e, DiskError::VolumeAccessDenied)))?;
|
||||
}
|
||||
let access_check_duration = access_check_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
(std::fs::File::open(&file_path).map_err(DiskError::from)?, access_check_duration)
|
||||
};
|
||||
let file_open_duration = file_open_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
|
||||
let metadata_lookup_start = metrics_enabled.then(StdInstant::now);
|
||||
// On a cache hit this fstats the cached descriptor — the inode it was
|
||||
// opened against, which invalidation keeps current for live entries. EC
|
||||
// shards are fixed-length, so a still-cached pre-heal length is benign.
|
||||
let meta = file.metadata().map_err(DiskError::from)?;
|
||||
let metadata_lookup_duration = metadata_lookup_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
|
||||
@@ -3160,13 +3288,15 @@ impl LocalIoBackend for StdBackend {
|
||||
bytes
|
||||
}
|
||||
LocalReadCopyMethod::DirectReadCopy => {
|
||||
use std::io::{Read as _, Seek as _};
|
||||
use std::os::unix::fs::FileExt;
|
||||
|
||||
let direct_read_copy_start = metrics_enabled.then(StdInstant::now);
|
||||
let direct_read_copy_faults_before = read_mmap_page_fault_counts(metrics_enabled);
|
||||
file.seek(SeekFrom::Start(offset_u64)).map_err(DiskError::from)?;
|
||||
let mut buffer = vec![0; length];
|
||||
file.read_exact(&mut buffer).map_err(DiskError::from)?;
|
||||
// Positioned read: a cache hit reads through a `dup`'d handle
|
||||
// that shares the cached descriptor's offset, so this must not
|
||||
// touch the descriptor offset (rustfs/backlog#1801).
|
||||
file.read_exact_at(&mut buffer, offset_u64).map_err(DiskError::from)?;
|
||||
let direct_read_copy_faults_after = read_mmap_page_fault_counts(metrics_enabled);
|
||||
direct_read_copy_duration =
|
||||
direct_read_copy_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
@@ -3193,6 +3323,16 @@ impl LocalIoBackend for StdBackend {
|
||||
|
||||
let blocking_task_duration = blocking_task_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
|
||||
// Hand the freshly opened descriptor back so the async caller can index
|
||||
// the cache — None on a hit (the cache already holds it). mmap/reclaim
|
||||
// above only borrowed `file`, so it is still owned here and moves into the
|
||||
// Arc; `cached_fd.is_none()` is true exactly when this call did the open.
|
||||
// Non-Linux has no fd cache, so skip the Arc allocation there.
|
||||
#[cfg(target_os = "linux")]
|
||||
let opened_fd: Option<Arc<std::fs::File>> = cached_fd.is_none().then(|| Arc::new(file));
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let opened_fd: Option<Arc<std::fs::File>> = None;
|
||||
|
||||
Ok::<MmapCopyReadResult, MmapCopyReadError>(MmapCopyReadResult {
|
||||
bytes,
|
||||
access_check_duration,
|
||||
@@ -3208,6 +3348,7 @@ impl LocalIoBackend for StdBackend {
|
||||
direct_read_copy_fault_delta,
|
||||
blocking_task_duration,
|
||||
used_direct_io,
|
||||
opened_fd,
|
||||
})
|
||||
})
|
||||
.await
|
||||
@@ -3313,6 +3454,16 @@ impl LocalIoBackend for StdBackend {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Index the freshly opened descriptor for future cache hits
|
||||
// (rustfs/backlog#1801). `insert_if_fresh` refuses to cache if an
|
||||
// invalidation (heal/delete/rename) bumped the generation between the
|
||||
// open snapshot and now, so a stale pre-mutation inode is never served
|
||||
// (rustfs/backlog#1176). On a cache hit `opened_fd` is None; on non-Linux
|
||||
// there is no fd cache, so this is gated out entirely.
|
||||
#[cfg(target_os = "linux")]
|
||||
if let (Some((cache, key, gen_at_open)), Some(opened)) = (fd_lookup, read_result.opened_fd) {
|
||||
cache.insert_if_fresh(key, opened, gen_at_open).await;
|
||||
}
|
||||
let bytes = read_result.bytes;
|
||||
|
||||
// Log successful mmap read metrics
|
||||
@@ -3516,6 +3667,39 @@ impl LocalIoBackend for StdBackend {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Descriptor-cache invalidation for StdBackend (rustfs/backlog#1801). On
|
||||
// non-Linux `fd_cache` does not exist, so these overrides are absent and the
|
||||
// trait's default no-op impls apply. On Linux they mirror UringBackend so
|
||||
// the existing LocalDisk mutation hooks (rename_data/rename_file/delete/
|
||||
// delete_volume/close) drop stale descriptors on every inode swap.
|
||||
#[cfg(target_os = "linux")]
|
||||
async fn invalidate_cached_fd(&self, volume: &str, path: &str) {
|
||||
if let Some(cache) = self.fd_cache.as_ref() {
|
||||
cache.invalidate_exact(volume, path).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn invalidate_cached_fds_under(&self, volume: &str, path: &str) {
|
||||
if let Some(cache) = self.fd_cache.as_ref() {
|
||||
cache.invalidate_under(volume, path);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn invalidate_cached_fds_for_volume(&self, volume: &str) {
|
||||
if let Some(cache) = self.fd_cache.as_ref() {
|
||||
cache.invalidate_volume(volume);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
async fn clear_cached_fds(&self) {
|
||||
if let Some(cache) = self.fd_cache.as_ref() {
|
||||
cache.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable the per-disk descriptor cache for io_uring reads (backlog#1145).
|
||||
@@ -3541,6 +3725,20 @@ fn is_io_uring_fd_cache_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_IO_URING_FD_CACHE, DEFAULT_RUSTFS_IO_URING_FD_CACHE)
|
||||
}
|
||||
|
||||
/// Enable the per-disk descriptor cache for the default `StdBackend` reads
|
||||
/// (rustfs/backlog#1801). Independent of the io_uring switch so each backend is
|
||||
/// separately controllable; both share the same `rlimit_allows_fd_cache` guard
|
||||
/// because each may hold up to `FD_CACHE_CAPACITY` (512) descriptors per disk.
|
||||
#[cfg(target_os = "linux")]
|
||||
const ENV_RUSTFS_LOCAL_FD_CACHE: &str = "RUSTFS_LOCAL_FD_CACHE";
|
||||
#[cfg(target_os = "linux")]
|
||||
const DEFAULT_RUSTFS_LOCAL_FD_CACHE: bool = true;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn is_local_fd_cache_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_LOCAL_FD_CACHE, DEFAULT_RUSTFS_LOCAL_FD_CACHE)
|
||||
}
|
||||
|
||||
/// Whether the soft `RLIMIT_NOFILE` has enough headroom to run the fd cache
|
||||
/// safely (rustfs/backlog#1178). The cache holds up to `FD_CACHE_CAPACITY` (512)
|
||||
/// descriptors PER DISK and `try_new` cannot know the disk count, so a low limit
|
||||
@@ -3946,7 +4144,7 @@ impl UringBackend {
|
||||
// struct (rustfs/backlog#1185).
|
||||
let root_label = root.display().to_string();
|
||||
Some(Self {
|
||||
inner: StdBackend::new(root.clone()),
|
||||
inner: StdBackend::new_without_fd_cache(root.clone()),
|
||||
root,
|
||||
root_label,
|
||||
driver: std::mem::ManuallyDrop::new(driver),
|
||||
@@ -7890,6 +8088,11 @@ impl DiskAPI for LocalDisk {
|
||||
std::io::Write::write_all(&mut new_meta, &meta)?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
new_meta.sync_data()?;
|
||||
}
|
||||
// Windows rejects renaming a directory while one of its children is
|
||||
// still open, even when the child handle shares delete access.
|
||||
drop(new_meta);
|
||||
if durability.syncs_commit_metadata() {
|
||||
os::fsync_dir_std(&staging_path)?;
|
||||
}
|
||||
std::fs::rename(&staging_path, &transaction_path)?;
|
||||
@@ -8096,7 +8299,7 @@ impl DiskAPI for LocalDisk {
|
||||
let durability = effective_durability(dst_volume);
|
||||
if durability.syncs_data_shards() && !src_is_dir {
|
||||
let src = src_file_path.clone();
|
||||
tokio::task::spawn_blocking(move || std::fs::File::open(&src)?.sync_data())
|
||||
tokio::task::spawn_blocking(move || os::sync_file(&src))
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(to_file_error)?;
|
||||
@@ -9016,7 +9219,20 @@ impl DiskAPI for LocalDisk {
|
||||
#[cfg(windows)]
|
||||
let source_parent = src_file_parent.to_path_buf();
|
||||
let rename_commit_guard_for_preparation = rename_commit_guard.clone();
|
||||
let inline_preparation = os::run_blocking_namespace_operation(mutation_lease.clone(), move || {
|
||||
let sync = durability.syncs_commit_metadata();
|
||||
#[cfg(test)]
|
||||
run_inline_before_file_sync_admission(dst_path);
|
||||
let mut file_sync_admission = if sync {
|
||||
Some(
|
||||
os::acquire_file_sync_admission(self.file_sync_permits.clone())
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let prepare_inline_metadata = move || {
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?;
|
||||
#[cfg(windows)]
|
||||
@@ -9053,7 +9269,6 @@ impl DiskAPI for LocalDisk {
|
||||
None
|
||||
}
|
||||
});
|
||||
let sync = durability.syncs_commit_metadata();
|
||||
let mut staged_rollback_path = None;
|
||||
if let Some(d) = old_data_dir.as_ref() {
|
||||
let _ = xlmeta.data.remove_two(version_id, *d);
|
||||
@@ -9098,8 +9313,12 @@ impl DiskAPI for LocalDisk {
|
||||
has_dst_buf.is_none(),
|
||||
prepared_metadata_source,
|
||||
))
|
||||
})
|
||||
.await
|
||||
};
|
||||
let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() {
|
||||
os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await
|
||||
} else {
|
||||
os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await
|
||||
}
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from);
|
||||
|
||||
@@ -9149,14 +9368,26 @@ impl DiskAPI for LocalDisk {
|
||||
let backup_path = dst_parent
|
||||
.join(rollback_data_dir.to_string())
|
||||
.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
// rename_all acquires the backup path's namespace lease. Do not
|
||||
// hold a disk admission while acquiring another namespace lock.
|
||||
drop(file_sync_admission.take());
|
||||
if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await {
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Err(err);
|
||||
}
|
||||
run_rename_data_after_first_publication(dst_path);
|
||||
if durability.syncs_commit_metadata()
|
||||
if sync {
|
||||
file_sync_admission = Some(
|
||||
os::acquire_file_sync_admission(self.file_sync_permits.clone())
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)?,
|
||||
);
|
||||
}
|
||||
if let Some(admission) = file_sync_admission.as_ref()
|
||||
&& let Some(backup_parent) = backup_path.parent()
|
||||
&& let Err(err) = os::fsync_dir(backup_parent).await
|
||||
&& let Err(err) =
|
||||
os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await
|
||||
{
|
||||
return Err(DiskError::from(to_file_error(err)));
|
||||
}
|
||||
@@ -9194,9 +9425,10 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
// Persist the commit rename's directory entry across power loss.
|
||||
if durability.syncs_commit_metadata()
|
||||
if let Some(admission) = file_sync_admission.as_ref()
|
||||
&& let Some(dst_parent) = dst_file_path.parent()
|
||||
&& let Err(err) = os::fsync_dir(dst_parent).await
|
||||
&& let Err(err) =
|
||||
os::fsync_dir_with_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission).await
|
||||
{
|
||||
rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?;
|
||||
return Err(err);
|
||||
@@ -9209,13 +9441,17 @@ impl DiskAPI for LocalDisk {
|
||||
// not its own entry, so for a new inline object fsync the ancestor
|
||||
// chain up to and including the bucket. Overwrites already have a
|
||||
// durable object dir; the starts_with guard bounds the walk.
|
||||
if durability.syncs_commit_metadata() && destination_was_absent {
|
||||
if let Some(admission) = file_sync_admission.as_ref()
|
||||
&& destination_was_absent
|
||||
{
|
||||
let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent());
|
||||
while let Some(ancestor_dir) = ancestor {
|
||||
if !ancestor_dir.starts_with(&dst_volume_dir) {
|
||||
break;
|
||||
}
|
||||
if let Err(err) = os::fsync_dir(ancestor_dir).await {
|
||||
if let Err(err) =
|
||||
os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await
|
||||
{
|
||||
rollback_inline_metadata_commit_std(
|
||||
&dst_file_path,
|
||||
rollback_data_dir,
|
||||
@@ -9234,6 +9470,10 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
.await;
|
||||
|
||||
// The disk admission protects the durability chain, not staging
|
||||
// cleanup or cache invalidation after that chain has completed.
|
||||
drop(file_sync_admission.take());
|
||||
|
||||
// A post-commit rollback (for example, a commit-metadata fsync
|
||||
// failure under strict durability) restores the old metadata; drop any
|
||||
// descriptors cached during the committed window before propagating the
|
||||
@@ -11566,6 +11806,116 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn test_rename_part_commits_realistic_windows_multipart_path() {
|
||||
use crate::disk::RUSTFS_META_MULTIPART_BUCKET;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
assert_eq!(effective_durability(RUSTFS_META_MULTIPART_BUCKET), DurabilityMode::Strict);
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let root = dir.path().join("realistic-windows-multipart-root");
|
||||
fs::create_dir_all(&root).await.expect("disk root should be created");
|
||||
let endpoint = Endpoint::try_from(root.to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_MULTIPART_BUCKET).await;
|
||||
|
||||
let src_path = "upload/part.1";
|
||||
let dst_path = concat!(
|
||||
"6f897928dfe04a87a269ccd9f5a5897d9cbbdf6b55e4d903ef3cbc1125c0cb8f/",
|
||||
"8f897819-2604-4f3d-b843-c32a45d198b2x1786372838834745500/",
|
||||
"58ba822c-06e4-4332-81cc-be2c9d921900/part.1"
|
||||
);
|
||||
let transaction_path = disk
|
||||
.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &crate::disk::part_transaction_path(dst_path))
|
||||
.expect("transaction path should resolve");
|
||||
let deepest_marker = transaction_path
|
||||
.parent()
|
||||
.expect("transaction path should have a parent")
|
||||
.join(".part-txn-00000000-0000-0000-0000-000000000000")
|
||||
.join(PART_TRANSACTION_OLD_DATA_ABSENT);
|
||||
assert!(
|
||||
deepest_marker.as_os_str().len() > 260,
|
||||
"regression path must cross the traditional Windows MAX_PATH boundary: {deepest_marker:?}"
|
||||
);
|
||||
let payload = Bytes::from_static(b"part payload");
|
||||
let meta = Bytes::from_static(b"part metadata");
|
||||
disk.write_all(RUSTFS_META_TMP_BUCKET, src_path, payload.clone())
|
||||
.await
|
||||
.expect("source part should be written");
|
||||
|
||||
disk.prepare_part_transaction(RUSTFS_META_TMP_BUCKET, src_path, RUSTFS_META_MULTIPART_BUCKET, dst_path, meta.clone())
|
||||
.await
|
||||
.expect("realistic Windows part transaction should be prepared");
|
||||
disk.rename_part(RUSTFS_META_TMP_BUCKET, src_path, RUSTFS_META_MULTIPART_BUCKET, dst_path, meta.clone())
|
||||
.await
|
||||
.expect("realistic Windows part should be committed");
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, dst_path, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("realistic Windows part transaction should be settled");
|
||||
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, dst_path)
|
||||
.await
|
||||
.expect("destination part should be readable"),
|
||||
payload
|
||||
);
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &format!("{dst_path}.meta"))
|
||||
.await
|
||||
.expect("destination metadata should be readable"),
|
||||
meta
|
||||
);
|
||||
|
||||
let replacement_payload = Bytes::from_static(b"replacement part payload");
|
||||
let replacement_meta = Bytes::from_static(b"replacement part metadata");
|
||||
disk.write_all(RUSTFS_META_TMP_BUCKET, src_path, replacement_payload.clone())
|
||||
.await
|
||||
.expect("replacement source part should be written");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
src_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
dst_path,
|
||||
replacement_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("replacement Windows part transaction should be prepared");
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
src_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
dst_path,
|
||||
replacement_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("replacement Windows part should be committed");
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, dst_path, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("replacement Windows part transaction should be settled");
|
||||
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, dst_path)
|
||||
.await
|
||||
.expect("replacement destination part should be readable"),
|
||||
replacement_payload
|
||||
);
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &format!("{dst_path}.meta"))
|
||||
.await
|
||||
.expect("replacement destination metadata should be readable"),
|
||||
replacement_meta
|
||||
);
|
||||
assert!(
|
||||
matches!(disk.read_all(RUSTFS_META_TMP_BUCKET, src_path).await, Err(DiskError::FileNotFound)),
|
||||
"successful replacement must remove its source part"
|
||||
);
|
||||
assert!(!transaction_path.exists(), "settled replacement must remove its transaction directory");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_part_transaction_rolls_back_data_published_before_metadata() {
|
||||
use tempfile::tempdir;
|
||||
@@ -12290,7 +12640,7 @@ mod test {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_rename_data_new_inline_object_fsyncs_new_ancestor_dirs() {
|
||||
async fn windows_and_unix_rename_data_new_inline_object_fsyncs_new_ancestor_dirs() {
|
||||
// The inline commit path (fi.data present) has the same mkdir gap as the
|
||||
// non-inline path: a first PUT under a new prefix must fsync the newly
|
||||
// created prefix and bucket dirs.
|
||||
@@ -12319,10 +12669,122 @@ mod test {
|
||||
os::fsync_dir_recorder::was_fsynced(&prefix_dir),
|
||||
"the newly created prefix dir must be fsynced on an inline first PUT"
|
||||
);
|
||||
assert_eq!(
|
||||
os::fsync_dir_recorder::was_limited(&prefix_dir),
|
||||
cfg!(unix),
|
||||
"only Unix inline prefix fsyncs should use the disk file-sync limit"
|
||||
);
|
||||
assert!(
|
||||
os::fsync_dir_recorder::was_fsynced(&bucket_dir),
|
||||
"the bucket dir must be fsynced on an inline first PUT"
|
||||
);
|
||||
assert_eq!(
|
||||
os::fsync_dir_recorder::was_limited(&bucket_dir),
|
||||
cfg!(unix),
|
||||
"only Unix inline bucket fsyncs should use the disk file-sync limit"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn strict_inline_rename_retains_admission_until_commit_fsync() {
|
||||
use std::sync::atomic::{AtomicU8, Ordering};
|
||||
use std::sync::mpsc;
|
||||
use tempfile::tempdir;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
const FIRST_BARRIER: u8 = 1;
|
||||
const SECOND_PREPARATION: u8 = 2;
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let mut disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
disk.file_sync_permits = Arc::new(Semaphore::new(1));
|
||||
let disk = Arc::new(disk);
|
||||
let bucket = "inline-admission-order";
|
||||
let first_object = "first-object";
|
||||
let second_object = "second-object";
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
|
||||
let (first_prepared_tx, first_prepared_rx) = mpsc::channel();
|
||||
let (release_first_tx, release_first_rx) = mpsc::channel();
|
||||
set_inline_preparation_before_backup(first_object, move || {
|
||||
first_prepared_tx.send(()).expect("signal first preparation");
|
||||
release_first_rx.recv().expect("wait for queued rename");
|
||||
});
|
||||
let first_disk = disk.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
first_disk
|
||||
.rename_data(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"first-stage",
|
||||
test_file_info(first_object, Uuid::new_v4(), None, Some(Bytes::from_static(b"first"))),
|
||||
bucket,
|
||||
first_object,
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::task::spawn_blocking(move || first_prepared_rx.recv_timeout(Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("first preparation waiter should run")
|
||||
.expect("first rename should hold the only disk admission");
|
||||
|
||||
let first_event = Arc::new(AtomicU8::new(0));
|
||||
let first_barrier_event = first_event.clone();
|
||||
let first_object_dir = disk
|
||||
.get_object_path_for_io(bucket, first_object)
|
||||
.expect("first object path should resolve");
|
||||
os::fsync_dir_recorder::set_before_limited(&first_object_dir, move || {
|
||||
let _ = first_barrier_event.compare_exchange(0, FIRST_BARRIER, Ordering::SeqCst, Ordering::SeqCst);
|
||||
});
|
||||
|
||||
let second_preparation_event = first_event.clone();
|
||||
set_inline_preparation_before_backup(second_object, move || {
|
||||
second_preparation_event.fetch_or(SECOND_PREPARATION, Ordering::SeqCst);
|
||||
});
|
||||
let (second_admission_tx, second_admission_rx) = oneshot::channel();
|
||||
set_inline_before_file_sync_admission(second_object, move || {
|
||||
second_admission_tx.send(()).expect("signal second admission attempt");
|
||||
});
|
||||
let second_disk = disk.clone();
|
||||
let mut second = Box::pin(async move {
|
||||
second_disk
|
||||
.rename_data(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"second-stage",
|
||||
test_file_info(second_object, Uuid::new_v4(), None, Some(Bytes::from_static(b"second"))),
|
||||
bucket,
|
||||
second_object,
|
||||
)
|
||||
.await
|
||||
});
|
||||
let mut second_admission_rx = Box::pin(second_admission_rx);
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
tokio::select! {
|
||||
_ = &mut second => panic!("second rename must wait for disk admission"),
|
||||
signal = &mut second_admission_rx => signal.expect("second admission hook should run"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second rename should reach the admission queue");
|
||||
|
||||
release_first_tx.send(()).expect("release first preparation");
|
||||
let (first_result, second_result) = tokio::time::timeout(Duration::from_secs(30), async { tokio::join!(first, second) })
|
||||
.await
|
||||
.expect("both inline renames should complete");
|
||||
first_result
|
||||
.expect("first rename task should join")
|
||||
.expect("first inline rename should commit");
|
||||
second_result.expect("second inline rename should commit");
|
||||
|
||||
assert_eq!(
|
||||
first_event.load(Ordering::SeqCst),
|
||||
FIRST_BARRIER | SECOND_PREPARATION,
|
||||
"the queued rename must not overtake the admitted rename before its commit fsync"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
@@ -13569,14 +14031,16 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn test_rename_data_writes_old_metadata_backup_for_inline_overwrite() {
|
||||
use std::sync::mpsc;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
|
||||
let bucket = "bucket";
|
||||
let object = "inline-object";
|
||||
@@ -13602,10 +14066,32 @@ mod test {
|
||||
.await
|
||||
.expect("tmp object dir should be created");
|
||||
|
||||
let (published_tx, published_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
set_rename_data_after_first_publication(object, move || {
|
||||
published_tx.send(()).expect("signal backup publication");
|
||||
release_rx.recv().expect("wait for lock-order assertion");
|
||||
});
|
||||
let new_fi = test_file_info(object, version_id, None, Some(Bytes::from_static(b"inline-new")));
|
||||
let resp = disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, new_fi, bucket, object)
|
||||
let rename_disk = disk.clone();
|
||||
let rename = tokio::spawn(async move {
|
||||
rename_disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, new_fi, bucket, object)
|
||||
.await
|
||||
});
|
||||
tokio::task::spawn_blocking(move || published_rx.recv_timeout(Duration::from_secs(10)))
|
||||
.await
|
||||
.expect("publication waiter should run")
|
||||
.expect("rollback backup must be published");
|
||||
assert_eq!(
|
||||
disk.file_sync_permits.available_permits(),
|
||||
os::MAX_PARALLEL_FILE_SYNCS,
|
||||
"backup publication must not acquire namespace while holding disk admission"
|
||||
);
|
||||
release_tx.send(()).expect("release backup publication");
|
||||
let resp = rename
|
||||
.await
|
||||
.expect("inline rename task should join")
|
||||
.expect("inline rename_data should commit");
|
||||
|
||||
assert_eq!(resp.old_data_dir, Some(old_data_dir));
|
||||
@@ -13616,10 +14102,20 @@ mod test {
|
||||
os::fsync_dir_recorder::was_fsynced(backup_path.parent().expect("backup must have a parent")),
|
||||
"strict inline overwrite must persist the rollback backup directory entry"
|
||||
);
|
||||
assert_eq!(
|
||||
os::fsync_dir_recorder::was_limited(backup_path.parent().expect("backup must have a parent")),
|
||||
cfg!(unix),
|
||||
"only Unix rollback backup fsyncs should use the disk file-sync limit"
|
||||
);
|
||||
assert!(
|
||||
os::fsync_dir_recorder::was_fsynced(&dst_object_dir),
|
||||
"strict inline overwrite must persist the committed xl.meta directory entry"
|
||||
);
|
||||
assert_eq!(
|
||||
os::fsync_dir_recorder::was_limited(&dst_object_dir),
|
||||
cfg!(unix),
|
||||
"only Unix inline commit fsyncs should use the disk file-sync limit"
|
||||
);
|
||||
// The rollback backup must contain the previous metadata bytes verbatim so
|
||||
// that undo_write can restore the prior committed object; guards the inline
|
||||
// backup write against truncation/corruption regressions.
|
||||
@@ -14216,10 +14712,12 @@ mod test {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn windows_and_unix_cancelled_inline_preparation_serializes_newer_commit() {
|
||||
use std::sync::mpsc;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
@@ -14258,6 +14756,11 @@ mod test {
|
||||
.await
|
||||
.expect("preparation waiter should run")
|
||||
.expect("preparation must reach the backup hook");
|
||||
assert_eq!(
|
||||
disk.file_sync_permits.available_permits(),
|
||||
os::MAX_PARALLEL_FILE_SYNCS - 1,
|
||||
"strict inline preparation must hold one disk file-sync permit"
|
||||
);
|
||||
cancelled.abort();
|
||||
assert!(cancelled.await.expect_err("operation should be cancelled").is_cancelled());
|
||||
|
||||
@@ -14331,6 +14834,56 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn relaxed_inline_preparation_does_not_use_file_sync_limit() {
|
||||
use std::sync::mpsc;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Relaxed);
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
let bucket = "relaxed-inline-preparation";
|
||||
let object = "inline-object";
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
set_inline_preparation_before_backup(object, move || {
|
||||
entered_tx.send(()).expect("signal blocked preparation");
|
||||
release_rx.recv().expect("wait for permit assertion");
|
||||
});
|
||||
let rename_disk = Arc::clone(&disk);
|
||||
let rename = tokio::spawn(async move {
|
||||
rename_disk
|
||||
.rename_data(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"relaxed-inline-stage",
|
||||
test_file_info(object, Uuid::new_v4(), None, Some(Bytes::from_static(b"payload"))),
|
||||
bucket,
|
||||
object,
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(10)))
|
||||
.await
|
||||
.expect("preparation waiter should run")
|
||||
.expect("preparation must reach the hook");
|
||||
assert_eq!(
|
||||
disk.file_sync_permits.available_permits(),
|
||||
os::MAX_PARALLEL_FILE_SYNCS,
|
||||
"relaxed inline preparation must not consume strict sync capacity"
|
||||
);
|
||||
|
||||
release_tx.send(()).expect("release inline preparation");
|
||||
rename
|
||||
.await
|
||||
.expect("rename task should join")
|
||||
.expect("relaxed inline rename should commit");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_purge_pending_payload_stays_object_and_cleans_local_backup() {
|
||||
use tempfile::tempdir;
|
||||
@@ -19279,6 +19832,133 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
/// Same heal hazard as the io_uring test, but exercised through the default
|
||||
/// `StdBackend` read path (rustfs/backlog#1801): a cached descriptor keeps
|
||||
/// serving the pre-heal inode until `invalidate_cached_fds_under` drops it.
|
||||
/// `StdBackend` reads via mmap/`try_clone`, so this proves the dup-based hit
|
||||
/// path also defers to invalidation rather than masking a healed shard.
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn std_fd_cache_hides_a_healed_shard_until_invalidated() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
let root_dir = tempdir().expect("operation should succeed");
|
||||
let root = root_dir.path().to_path_buf();
|
||||
let backend = temp_env::with_vars([(ENV_RUSTFS_LOCAL_FD_CACHE, Some("true"))], || StdBackend::new(root.clone()));
|
||||
if backend.fd_cache.is_none() {
|
||||
// RLIMIT_NOFILE too low for 512 fds/disk (rustfs/backlog#1178): the
|
||||
// cache is off, so there is nothing to exercise. Do not vacuously pass.
|
||||
eprintln!(
|
||||
"std_fd_cache_hides_a_healed_shard_until_invalidated: skipped \
|
||||
(RLIMIT_NOFILE too low for the std fd cache)"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let volume = "bucket";
|
||||
let object = "obj/0d1e2f/part.1";
|
||||
let dir = root.join(volume).join("obj/0d1e2f");
|
||||
std::fs::create_dir_all(&dir).expect("operation should succeed");
|
||||
let part = root.join(volume).join(object);
|
||||
std::fs::write(&part, b"corrupt-shard").expect("operation should succeed");
|
||||
|
||||
let before = backend
|
||||
.pread_bytes(volume, object, 0, b"corrupt-shard".len(), None)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
assert_eq!(before, Bytes::from_static(b"corrupt-shard"));
|
||||
|
||||
// Heal: rename rebuilt content onto the same part path — inode swap, path
|
||||
// unchanged. A cached descriptor would keep reading the old inode.
|
||||
let rebuilt = dir.join("part.1.rebuilt");
|
||||
std::fs::write(&rebuilt, b"healed--shard").expect("operation should succeed");
|
||||
std::fs::rename(&rebuilt, &part).expect("operation should succeed");
|
||||
|
||||
let stale = backend
|
||||
.pread_bytes(volume, object, 0, b"healed--shard".len(), None)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
assert_eq!(
|
||||
stale,
|
||||
Bytes::from_static(b"corrupt-shard"),
|
||||
"a cached descriptor is expected to still see the pre-heal inode — this is the \
|
||||
hazard invalidate_cached_fds exists to close, and the assertion proves the cache is live"
|
||||
);
|
||||
|
||||
backend.invalidate_cached_fds_under(volume, "obj/0d1e2f");
|
||||
let healed = backend
|
||||
.pread_bytes(volume, object, 0, b"healed--shard".len(), None)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
assert_eq!(healed, Bytes::from_static(b"healed--shard"));
|
||||
}
|
||||
|
||||
/// A repeated read of the same shard must (a) return correct bytes both times
|
||||
/// and (b) actually populate the descriptor cache, so the second read can skip
|
||||
/// `File::open` (rustfs/backlog#1801).
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn std_fd_cache_serves_repeated_reads_and_caches_descriptor() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
let root_dir = tempdir().expect("operation should succeed");
|
||||
let root = root_dir.path().to_path_buf();
|
||||
let backend = temp_env::with_vars([(ENV_RUSTFS_LOCAL_FD_CACHE, Some("true"))], || StdBackend::new(root.clone()));
|
||||
let cache = match backend.fd_cache.as_ref() {
|
||||
Some(c) => c,
|
||||
None => {
|
||||
eprintln!(
|
||||
"std_fd_cache_serves_repeated_reads_and_caches_descriptor: skipped \
|
||||
(RLIMIT_NOFILE too low for the std fd cache)"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let volume = "bucket";
|
||||
let object = "obj/abc/part.1";
|
||||
std::fs::create_dir_all(root.join(volume).join("obj/abc")).expect("operation should succeed");
|
||||
let payload = b"hello-small-shard-payload";
|
||||
std::fs::write(root.join(volume).join(object), payload).expect("operation should succeed");
|
||||
|
||||
let first = backend
|
||||
.pread_bytes(volume, object, 0, payload.len(), None)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
assert_eq!(first, Bytes::from_static(payload));
|
||||
|
||||
// After the first miss the freshly opened descriptor is indexed; a second
|
||||
// read of the same path is a cache hit.
|
||||
assert_eq!(cache.entry_count().await, 1, "the first read should have cached exactly one descriptor");
|
||||
|
||||
let second = backend
|
||||
.pread_bytes(volume, object, 0, payload.len(), None)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
assert_eq!(second, Bytes::from_static(payload));
|
||||
|
||||
// Invalidating by the object prefix drops the cached descriptor.
|
||||
backend.invalidate_cached_fds_under(volume, "obj/abc");
|
||||
assert_eq!(cache.entry_count().await, 0, "prefix invalidation must drop the cached descriptor");
|
||||
}
|
||||
|
||||
/// `StdBackend::new_without_fd_cache` must not build a descriptor cache.
|
||||
/// `UringBackend` wraps a `StdBackend` and owns the only cache for the disk,
|
||||
/// so an inner cache would be populated by fallback reads
|
||||
/// (`UringBackend::pread_bytes` delegates inward) yet never invalidated —
|
||||
/// the stale-inode hazard `FdCache` exists to close (backlog#1176/#1801).
|
||||
/// This pins the contract so a future constructor change cannot regress it.
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn new_without_fd_cache_builds_no_descriptor_cache() {
|
||||
let root_dir = tempfile::tempdir().expect("operation should succeed");
|
||||
let backend = StdBackend::new_without_fd_cache(root_dir.path().to_path_buf());
|
||||
assert!(
|
||||
backend.fd_cache.is_none(),
|
||||
"new_without_fd_cache must not build a descriptor cache — UringBackend owns the only cache for the disk"
|
||||
);
|
||||
}
|
||||
|
||||
/// The mutation paths on `LocalDisk` must actually call
|
||||
/// `invalidate_cached_fds`, not merely have it available (backlog#1145).
|
||||
/// `rename_file` replaces the inode at a path a reader has already cached;
|
||||
|
||||
@@ -1251,7 +1251,7 @@ pub struct VolumeInfo {
|
||||
pub created: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
|
||||
#[derive(Deserialize, Serialize, Debug, Default, Clone, Copy)]
|
||||
pub struct ReadOptions {
|
||||
pub incl_free_versions: bool,
|
||||
pub read_data: bool,
|
||||
|
||||
+192
-14
@@ -78,28 +78,59 @@ pub fn check_path_length(path_name: &str) -> Result<()> {
|
||||
/// their own unique tempdir to stay robust against parallel test execution.
|
||||
#[cfg(test)]
|
||||
pub(crate) mod fsync_dir_recorder {
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
static RECORDED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
type Hook = Box<dyn FnOnce() + Send>;
|
||||
|
||||
pub(crate) fn record(dir: &Path) {
|
||||
let mut recorded = RECORDED.lock().expect("fsync dir recorder poisoned");
|
||||
recorded.push(dir.to_path_buf());
|
||||
if let Ok(canonical) = dir.canonicalize()
|
||||
&& canonical != dir
|
||||
static RECORDED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
static LIMITED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
static BEFORE_LIMITED: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
|
||||
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
fn record_path(paths: &Mutex<Vec<PathBuf>>, path: &Path, description: &str) {
|
||||
let mut paths = paths.lock().expect(description);
|
||||
paths.push(path.to_path_buf());
|
||||
if let Ok(canonical) = path.canonicalize()
|
||||
&& canonical != path
|
||||
{
|
||||
recorded.push(canonical);
|
||||
paths.push(canonical);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn was_fsynced(dir: &Path) -> bool {
|
||||
let canonical = dir.canonicalize().ok();
|
||||
RECORDED
|
||||
.lock()
|
||||
.expect("fsync dir recorder poisoned")
|
||||
fn contains_path(paths: &[PathBuf], path: &Path) -> bool {
|
||||
let canonical = path.canonicalize().ok();
|
||||
paths
|
||||
.iter()
|
||||
.any(|p| p == dir || canonical.as_ref().is_some_and(|canonical| p == canonical))
|
||||
.any(|recorded| recorded == path || canonical.as_ref().is_some_and(|canonical| recorded == canonical))
|
||||
}
|
||||
|
||||
pub(crate) fn record(dir: &Path) {
|
||||
record_path(&RECORDED, dir, "fsync dir recorder");
|
||||
}
|
||||
|
||||
pub(crate) fn was_fsynced(dir: &Path) -> bool {
|
||||
contains_path(&RECORDED.lock().expect("fsync dir recorder poisoned"), dir)
|
||||
}
|
||||
|
||||
pub(crate) fn record_limited(dir: &Path) {
|
||||
record_path(&LIMITED, dir, "limited fsync dir recorder");
|
||||
let hook = BEFORE_LIMITED.lock().expect("limited fsync hook poisoned").remove(dir);
|
||||
if let Some(hook) = hook {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn was_limited(dir: &Path) -> bool {
|
||||
contains_path(&LIMITED.lock().expect("limited fsync dir recorder poisoned"), dir)
|
||||
}
|
||||
|
||||
pub(crate) fn set_before_limited(dir: &Path, hook: impl FnOnce() + Send + 'static) {
|
||||
BEFORE_LIMITED
|
||||
.lock()
|
||||
.expect("limited fsync hook poisoned")
|
||||
.insert(dir.to_path_buf(), Box::new(hook));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -497,7 +528,7 @@ pub(crate) mod file_sync_probe {
|
||||
}
|
||||
}
|
||||
|
||||
fn sync_file(path: &Path) -> io::Result<()> {
|
||||
pub(crate) fn sync_file(path: &Path) -> io::Result<()> {
|
||||
#[cfg(test)]
|
||||
let _probe = file_sync_probe::enter(path);
|
||||
#[cfg(test)]
|
||||
@@ -1120,6 +1151,79 @@ pub(crate) async fn run_blocking_namespace_operation<T: Send + 'static>(
|
||||
.map_err(|err| io::Error::other(format!("blocking namespace operation failed: {err}")))?
|
||||
}
|
||||
|
||||
/// Admit one strict inline commit under the disk sync limit. The caller already
|
||||
/// owns the namespace lease, establishing namespace -> disk ordering. Holding
|
||||
/// admission across adjacent durability barriers prevents one transaction from
|
||||
/// repeatedly joining the disk semaphore tail.
|
||||
pub(crate) struct FileSyncAdmission {
|
||||
disk_permit: Arc<OwnedSemaphorePermit>,
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_file_sync_admission(disk_permits: Arc<Semaphore>) -> io::Result<FileSyncAdmission> {
|
||||
let disk_permit = disk_permits
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|_| io::Error::other("disk file sync concurrency limiter closed"))?;
|
||||
Ok(FileSyncAdmission {
|
||||
disk_permit: Arc::new(disk_permit),
|
||||
})
|
||||
}
|
||||
|
||||
/// Keep the disk admission and namespace lease with the blocking syscall if
|
||||
/// the async waiter is cancelled. The process-wide admission remains with the
|
||||
/// waiter so cancellation cannot starve healthy disks.
|
||||
pub(crate) async fn run_blocking_namespace_file_sync_operation<T: Send + 'static>(
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
admission: &FileSyncAdmission,
|
||||
operation: impl FnOnce() -> io::Result<T> + Send + 'static,
|
||||
) -> io::Result<T> {
|
||||
run_blocking_namespace_file_sync_operation_with_global(lease, admission, &FILE_SYNC_PERMITS, operation).await
|
||||
}
|
||||
|
||||
async fn run_blocking_namespace_file_sync_operation_with_global<T: Send + 'static>(
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
admission: &FileSyncAdmission,
|
||||
global_permits: &Semaphore,
|
||||
operation: impl FnOnce() -> io::Result<T> + Send + 'static,
|
||||
) -> io::Result<T> {
|
||||
let global_permit = global_permits
|
||||
.acquire()
|
||||
.await
|
||||
.map_err(|_| io::Error::other("global file sync concurrency limiter closed"))?;
|
||||
let disk_permit = admission.disk_permit.clone();
|
||||
let result = tokio::task::spawn_blocking(move || {
|
||||
let _lease = lease;
|
||||
let _disk_permit = disk_permit;
|
||||
operation()
|
||||
})
|
||||
.await;
|
||||
drop(global_permit);
|
||||
result.map_err(|err| io::Error::other(format!("blocking namespace file sync operation failed: {err}")))?
|
||||
}
|
||||
|
||||
pub(crate) async fn fsync_dir_with_namespace_file_sync_limit(
|
||||
dir: impl AsRef<Path>,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
admission: &FileSyncAdmission,
|
||||
) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let dir = dir.as_ref().to_path_buf();
|
||||
run_blocking_namespace_file_sync_operation(lease, admission, move || {
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::record_limited(&dir);
|
||||
fsync_dir_std(dir)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = (lease, admission);
|
||||
fsync_dir_std(dir)
|
||||
}
|
||||
}
|
||||
|
||||
struct RenamePreparation {
|
||||
parent_guard: Option<ExistingBaseDirectoryGuard>,
|
||||
#[cfg(windows)]
|
||||
@@ -2784,6 +2888,7 @@ pub fn is_dir_not_empty_error(err: &io::Error) -> bool {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
use tempfile::tempdir;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
@@ -4553,6 +4658,79 @@ mod tests {
|
||||
fsync_dir(temp_dir.path()).await.expect("fsync dir must succeed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn file_sync_admission_is_reused_across_commit_barriers() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let limiter = Arc::new(Semaphore::new(1));
|
||||
let lease = acquire_namespace_mutation_lease(temp_dir.path()).await;
|
||||
let admission = acquire_file_sync_admission(limiter.clone())
|
||||
.await
|
||||
.expect("first commit should acquire admission");
|
||||
|
||||
run_blocking_namespace_file_sync_operation(lease.clone(), &admission, || Ok(()))
|
||||
.await
|
||||
.expect("first barrier should complete under the admission");
|
||||
let mut waiting = Box::pin(acquire_file_sync_admission(limiter));
|
||||
assert!(
|
||||
futures::poll!(&mut waiting).is_pending(),
|
||||
"another commit must remain queued between durability barriers"
|
||||
);
|
||||
run_blocking_namespace_file_sync_operation(lease, &admission, || Ok(()))
|
||||
.await
|
||||
.expect("later barrier should reuse admission without requeuing");
|
||||
|
||||
drop(admission);
|
||||
tokio::time::timeout(Duration::from_secs(30), waiting)
|
||||
.await
|
||||
.expect("queued commit should acquire admission after release")
|
||||
.expect("queued commit should acquire admission");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn cancelled_file_sync_waiter_keeps_disk_admission_until_blocking_work_finishes() {
|
||||
use std::sync::mpsc;
|
||||
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let limiter = Arc::new(Semaphore::new(1));
|
||||
let global_permits = Arc::new(Semaphore::new(1));
|
||||
let lease = acquire_namespace_mutation_lease(temp_dir.path()).await;
|
||||
let admission = acquire_file_sync_admission(limiter.clone())
|
||||
.await
|
||||
.expect("file sync admission should be acquired");
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let waiter_global_permits = global_permits.clone();
|
||||
let waiter = tokio::spawn(async move {
|
||||
run_blocking_namespace_file_sync_operation_with_global(lease, &admission, waiter_global_permits.as_ref(), move || {
|
||||
entered_tx.send(()).expect("signal blocking work");
|
||||
release_rx.recv().expect("wait for blocking work release");
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("blocking work waiter should run")
|
||||
.expect("blocking work should start");
|
||||
waiter.abort();
|
||||
assert!(waiter.await.expect_err("waiter should be cancelled").is_cancelled());
|
||||
let returned_global_permit = global_permits
|
||||
.try_acquire()
|
||||
.expect("cancelled waiter must return global capacity for healthy disks");
|
||||
assert!(
|
||||
limiter.clone().try_acquire_owned().is_err(),
|
||||
"cancelled waiter must not return disk capacity while blocking work is active"
|
||||
);
|
||||
|
||||
release_tx.send(()).expect("release blocking work");
|
||||
let _returned_permit = tokio::time::timeout(Duration::from_secs(30), limiter.acquire_owned())
|
||||
.await
|
||||
.expect("disk capacity should return after blocking work finishes")
|
||||
.expect("disk limiter should remain open");
|
||||
drop(returned_global_permit);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
async fn sync_dir_files_syncs_regular_files_and_dir() {
|
||||
|
||||
@@ -18,7 +18,11 @@ use std::io::IoSlice;
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
||||
use tracing::error;
|
||||
use uuid::Uuid;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_ERASURE: &str = "erasure";
|
||||
const EVENT_BITROT_SHORT_SHARD_READ: &str = "bitrot_short_shard_read";
|
||||
const EVENT_BITROT_HASH_MISMATCH: &str = "bitrot_hash_mismatch";
|
||||
|
||||
/// A shard source that may already hold its bytes in memory.
|
||||
///
|
||||
@@ -73,7 +77,6 @@ pin_project! {
|
||||
buf: Vec<u8>,
|
||||
skip_verify: bool,
|
||||
last_verify_duration: Duration,
|
||||
id: Uuid,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,7 +93,6 @@ where
|
||||
buf: Vec::new(),
|
||||
skip_verify,
|
||||
last_verify_duration: Duration::ZERO,
|
||||
id: Uuid::new_v4(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,7 +120,7 @@ where
|
||||
|
||||
let need = self.hash_algo.size() + want;
|
||||
self.read_scratch_block(need, want).await?;
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need], &self.id)?;
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need])?;
|
||||
out.copy_from_slice(data);
|
||||
self.last_verify_duration = verify;
|
||||
Ok(want)
|
||||
@@ -157,7 +159,7 @@ where
|
||||
}
|
||||
let filled = fill(&mut self.inner, &mut self.buf[..need]).await?;
|
||||
if filled < need {
|
||||
return Err(short_shard_read(&self.id, filled.saturating_sub(self.hash_algo.size()), want));
|
||||
return Err(short_shard_read(filled.saturating_sub(self.hash_algo.size()), want));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -166,15 +168,23 @@ where
|
||||
/// buffer returns its length, a short read is UnexpectedEof (backlog#799 B2).
|
||||
fn finish_len(&self, data_len: usize, want: usize) -> std::io::Result<usize> {
|
||||
if data_len < want {
|
||||
return Err(short_shard_read(&self.id, data_len, want));
|
||||
return Err(short_shard_read(data_len, want));
|
||||
}
|
||||
Ok(data_len)
|
||||
}
|
||||
}
|
||||
|
||||
/// A truncated shard is `UnexpectedEof`, not a short success (backlog#799 B2).
|
||||
fn short_shard_read(id: &Uuid, got: usize, want: usize) -> std::io::Error {
|
||||
error!("bitrot reader short shard read: id={id} got {got} of {want} bytes");
|
||||
fn short_shard_read(got: usize, want: usize) -> std::io::Error {
|
||||
error!(
|
||||
event = EVENT_BITROT_SHORT_SHARD_READ,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE,
|
||||
state = "failed",
|
||||
got,
|
||||
want,
|
||||
"short shard read: got {got} of {want} bytes"
|
||||
);
|
||||
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, format!("short shard read: got {got} of {want} bytes"))
|
||||
}
|
||||
|
||||
@@ -184,12 +194,7 @@ fn short_shard_read(id: &Uuid, got: usize, want: usize) -> std::io::Error {
|
||||
/// hash never reaches the caller's buffer. The verify duration is returned
|
||||
/// rather than stored so this stays a free function usable while `self` is
|
||||
/// borrowed for the block.
|
||||
fn split_and_verify<'a>(
|
||||
hash_algo: &HashAlgorithm,
|
||||
skip_verify: bool,
|
||||
block: &'a [u8],
|
||||
id: &Uuid,
|
||||
) -> std::io::Result<(&'a [u8], Duration)> {
|
||||
fn split_and_verify<'a>(hash_algo: &HashAlgorithm, skip_verify: bool, block: &'a [u8]) -> std::io::Result<(&'a [u8], Duration)> {
|
||||
let (hash, data) = block.split_at(hash_algo.size());
|
||||
if skip_verify {
|
||||
return Ok((data, Duration::ZERO));
|
||||
@@ -198,7 +203,14 @@ fn split_and_verify<'a>(
|
||||
let actual_hash = hash_algo.hash_encode(data);
|
||||
let verify = verify_start.elapsed();
|
||||
if actual_hash.as_ref() != hash {
|
||||
error!("bitrot reader hash mismatch, id={id} data_len={}", data.len());
|
||||
error!(
|
||||
event = EVENT_BITROT_HASH_MISMATCH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE,
|
||||
state = "failed",
|
||||
data_len = data.len(),
|
||||
"bitrot hash mismatch"
|
||||
);
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
|
||||
}
|
||||
Ok((data, verify))
|
||||
@@ -254,7 +266,7 @@ where
|
||||
// `need` bytes returns `None` and falls through to the scratch path,
|
||||
// keeping the short-read contract.
|
||||
if let Some(block) = self.inner.try_take_block(need) {
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &block, &self.id)?;
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &block)?;
|
||||
out.extend_from_slice(data);
|
||||
self.last_verify_duration = verify;
|
||||
return Ok(want);
|
||||
@@ -264,7 +276,7 @@ where
|
||||
// the sink differs (`extend_from_slice` into `out` instead of
|
||||
// `copy_from_slice` into a pre-zeroed buffer).
|
||||
self.read_scratch_block(need, want).await?;
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need], &self.id)?;
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need])?;
|
||||
out.extend_from_slice(data);
|
||||
self.last_verify_duration = verify;
|
||||
Ok(want)
|
||||
|
||||
@@ -29,6 +29,7 @@ use crate::set_disk::shard_source::{ShardReadCost, ShardStripeSource, StripeRead
|
||||
use futures::FutureExt;
|
||||
use futures::stream::{FuturesUnordered, StreamExt};
|
||||
use pin_project_lite::pin_project;
|
||||
use smallvec::{SmallVec, smallvec};
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::io::ErrorKind;
|
||||
@@ -40,9 +41,15 @@ use tracing::{debug, error, warn};
|
||||
|
||||
type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, bool)> + Send + 'a>>;
|
||||
|
||||
const INLINE_SHARD_SLOTS: usize = 32;
|
||||
type ShardBuffers = SmallVec<[Option<Vec<u8>>; INLINE_SHARD_SLOTS]>;
|
||||
type ShardErrors = SmallVec<[Option<Error>; INLINE_SHARD_SLOTS]>;
|
||||
type ShardIndexes = SmallVec<[usize; INLINE_SHARD_SLOTS]>;
|
||||
type ActiveReaders = SmallVec<[bool; INLINE_SHARD_SLOTS]>;
|
||||
|
||||
/// One stripe's worth of shard buffers plus the per-shard read errors, as
|
||||
/// returned by `ParallelReader::read` / `read_stripe_timed`.
|
||||
type StripeReadOutput = (Vec<Option<Vec<u8>>>, Vec<Option<Error>>);
|
||||
type StripeReadOutput = (ShardBuffers, ShardErrors);
|
||||
|
||||
const ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING: &str = "RUSTFS_SHARD_LOCALITY_SCHEDULING";
|
||||
const ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE: &str = "RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE";
|
||||
@@ -390,7 +397,7 @@ pub(crate) struct ParallelReader<R> {
|
||||
// start, parity slots only once a data shard is missing/dead. Unengaged
|
||||
// parity stays an unopened deferred reader; `deferred_handles[i]` realigns
|
||||
// it to the current stripe when it is engaged mid-object (backlog#923).
|
||||
engaged: Vec<bool>,
|
||||
engaged: SmallVec<[bool; INLINE_SHARD_SLOTS]>,
|
||||
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
|
||||
stripe_index: usize,
|
||||
}
|
||||
@@ -573,7 +580,7 @@ where
|
||||
// behavior. With the gate on, only data slots start engaged; parity is
|
||||
// engaged on demand, stripe-aligned through its deferred handle.
|
||||
let data_shards_only = get_lockstep_data_shards_only_enabled();
|
||||
let engaged = (0..readers.len())
|
||||
let engaged: SmallVec<_> = (0..readers.len())
|
||||
.map(|index| !data_shards_only || index < e.data_shards)
|
||||
.collect();
|
||||
ParallelReader {
|
||||
@@ -612,7 +619,7 @@ where
|
||||
fn record_shard_read_result(
|
||||
shards: &mut [Option<Vec<u8>>],
|
||||
errs: &mut [Option<Error>],
|
||||
retire_readers: &mut Vec<usize>,
|
||||
retire_readers: &mut ShardIndexes,
|
||||
success: &mut usize,
|
||||
successful_costs: &mut ShardReadCostCounts,
|
||||
i: usize,
|
||||
@@ -637,7 +644,7 @@ fn record_shard_read_result(
|
||||
}
|
||||
}
|
||||
|
||||
fn retire_abandoned_readers(errs: &mut [Option<Error>], retire_readers: &mut Vec<usize>, active_readers: &[bool]) {
|
||||
fn retire_abandoned_readers(errs: &mut [Option<Error>], retire_readers: &mut ShardIndexes, active_readers: &[bool]) {
|
||||
for (i, active) in active_readers.iter().enumerate() {
|
||||
if !*active {
|
||||
continue;
|
||||
@@ -692,7 +699,7 @@ where
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
#[hotpath::measure(impl_type = "ParallelReader")]
|
||||
pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
|
||||
pub async fn read(&mut self) -> StripeReadOutput {
|
||||
// On the reconstruction-verifying GET path, read every live shard reader
|
||||
// in lockstep so all readers advance one block per stripe and stay
|
||||
// mutually aligned. The adaptive data-first path below only reads
|
||||
@@ -716,7 +723,7 @@ where
|
||||
};
|
||||
|
||||
if shard_size == 0 {
|
||||
return (vec![None; num_readers], vec![None; num_readers]);
|
||||
return (smallvec![None; num_readers], smallvec![None; num_readers]);
|
||||
}
|
||||
|
||||
// Advance to the next stripe so the following read() computes the correct
|
||||
@@ -727,8 +734,8 @@ where
|
||||
// is only read above to derive `shard_size`, so advancing here is safe.
|
||||
self.offset += shard_size;
|
||||
|
||||
let mut shards: Vec<Option<Vec<u8>>> = vec![None; num_readers];
|
||||
let mut errs = vec![None; num_readers];
|
||||
let mut shards: ShardBuffers = smallvec![None; num_readers];
|
||||
let mut errs: ShardErrors = smallvec![None; num_readers];
|
||||
let read_costs = self.read_costs.as_slice();
|
||||
let locality_preference_enabled = self.locality_preference_enabled;
|
||||
let low_cost_available = self
|
||||
@@ -759,11 +766,11 @@ where
|
||||
|
||||
self.buffers.ensure_slots(num_readers);
|
||||
|
||||
let mut retire_readers = Vec::new();
|
||||
let mut retire_readers = ShardIndexes::new();
|
||||
if num_readers >= self.data_shards {
|
||||
let mut reader_iter = ReaderLaunchIter::new(&mut self.readers, read_costs, locality_preference_enabled);
|
||||
let mut sets = FuturesUnordered::new();
|
||||
let mut active_readers = vec![false; num_readers];
|
||||
let mut active_readers: ActiveReaders = smallvec![false; num_readers];
|
||||
let stripe_read_start = self.metrics_path.map(|_| Instant::now());
|
||||
let mut scheduled = 0usize;
|
||||
for _ in 0..self.data_shards {
|
||||
@@ -1023,7 +1030,7 @@ where
|
||||
/// stripe would reintroduce the desync. A parity reader that cannot be
|
||||
/// realigned (no pending deferred handle) is likewise retired instead of
|
||||
/// being read out of position.
|
||||
async fn read_lockstep(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
|
||||
async fn read_lockstep(&mut self) -> StripeReadOutput {
|
||||
let num_readers = self.readers.len();
|
||||
let shard_size = if self.offset + self.shard_size > self.shard_file_size {
|
||||
self.shard_file_size - self.offset
|
||||
@@ -1031,8 +1038,8 @@ where
|
||||
self.shard_size
|
||||
};
|
||||
|
||||
let mut shards: Vec<Option<Vec<u8>>> = vec![None; num_readers];
|
||||
let mut errs: Vec<Option<Error>> = vec![None; num_readers];
|
||||
let mut shards: ShardBuffers = smallvec![None; num_readers];
|
||||
let mut errs: ShardErrors = smallvec![None; num_readers];
|
||||
if shard_size == 0 {
|
||||
return (shards, errs);
|
||||
}
|
||||
@@ -1071,7 +1078,7 @@ where
|
||||
// Pre-claim per-slot buffers so the `self.readers` borrow below stays
|
||||
// disjoint from `self.buffers`; `Some(buffer)` also records which slots
|
||||
// participate, avoiding a per-stripe sidecar allocation.
|
||||
let mut bufs: Vec<Option<Vec<u8>>> = Vec::with_capacity(num_readers);
|
||||
let mut bufs: ShardBuffers = SmallVec::with_capacity(num_readers);
|
||||
for i in 0..num_readers {
|
||||
bufs.push(if self.engaged[i] && self.readers[i].is_some() {
|
||||
Some(self.buffers.take(i, shard_size))
|
||||
@@ -1086,7 +1093,7 @@ where
|
||||
let locality_preference_enabled = self.locality_preference_enabled;
|
||||
let stripe_read_start = metrics_path.map(|_| Instant::now());
|
||||
|
||||
let mut retire_readers = Vec::new();
|
||||
let mut retire_readers = ShardIndexes::new();
|
||||
let mut scheduled = 0usize;
|
||||
let mut success = 0usize;
|
||||
let mut completed = 0usize;
|
||||
@@ -1351,10 +1358,7 @@ fn get_data_block_len(shards: &[Option<Vec<u8>>], data_blocks: usize) -> usize {
|
||||
/// stripe-read stage timer. Factored out so the depth-1 prefetch loop and the
|
||||
/// serial loop time reads identically. A free `async fn` (rather than a closure)
|
||||
/// so the returned future's borrow of `reader` is correctly tied to the call.
|
||||
async fn read_stripe_timed<R>(
|
||||
reader: &mut ParallelReader<R>,
|
||||
stage_metrics_enabled: bool,
|
||||
) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>)
|
||||
async fn read_stripe_timed<R>(reader: &mut ParallelReader<R>, stage_metrics_enabled: bool) -> StripeReadOutput
|
||||
where
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
@@ -1967,6 +1971,32 @@ mod tests {
|
||||
|
||||
type BoxedShardReader = crate::io_support::bitrot::ShardReader;
|
||||
|
||||
#[test]
|
||||
fn shard_scratch_stays_inline_through_the_common_limit_and_spills_safely() {
|
||||
let inline: ShardBuffers = smallvec![None; INLINE_SHARD_SLOTS];
|
||||
assert!(!inline.spilled(), "the common shard-count boundary must not allocate");
|
||||
|
||||
let spilled: ShardBuffers = smallvec![None; INLINE_SHARD_SLOTS + 1];
|
||||
assert!(spilled.spilled(), "larger supported shard counts must fall back to the heap");
|
||||
assert_eq!(spilled.len(), INLINE_SHARD_SLOTS + 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parallel_reader_preserves_slot_count_above_inline_capacity() {
|
||||
const DATA_SHARDS: usize = INLINE_SHARD_SLOTS;
|
||||
const TOTAL_SHARDS: usize = INLINE_SHARD_SLOTS + 1;
|
||||
let readers = std::iter::repeat_with(|| None).take(TOTAL_SHARDS).collect();
|
||||
let erasure = Erasure::new(DATA_SHARDS, 1, DATA_SHARDS);
|
||||
let mut reader: ParallelReader<Cursor<Vec<u8>>> = ParallelReader::new(readers, erasure, 0, DATA_SHARDS);
|
||||
|
||||
let (shards, errors) = reader.read().await;
|
||||
|
||||
assert!(shards.spilled());
|
||||
assert!(errors.spilled());
|
||||
assert_eq!(shards.len(), TOTAL_SHARDS);
|
||||
assert_eq!(errors.len(), TOTAL_SHARDS);
|
||||
}
|
||||
|
||||
/// Counts the raw bytes pulled from a shard stream, to prove which shards
|
||||
/// a decode path actually touches (backlog#923 call-count evidence).
|
||||
struct CountingShardReader {
|
||||
@@ -2343,6 +2373,19 @@ mod tests {
|
||||
assert_eq!(err.expect("range beyond total length should fail").kind(), ErrorKind::InvalidInput);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_erasure_decode_zero_length_does_not_read_or_emit() {
|
||||
let erasure = Erasure::new(2, 1, 64);
|
||||
let readers: Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> = vec![None, None, None];
|
||||
let mut output = Vec::new();
|
||||
|
||||
let (written, err) = erasure.decode(&mut output, readers, 0, 0, 0).await;
|
||||
|
||||
assert_eq!(written, 0);
|
||||
assert!(err.is_none());
|
||||
assert!(output.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_erasure_decode_with_read_costs_restores_missing_data_shard_range() {
|
||||
const DATA_SHARDS: usize = 2;
|
||||
|
||||
@@ -91,6 +91,11 @@ fn use_bytesmut_ingest() -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
fn small_ingest_capacity(erasure: &Erasure, size_hint: usize) -> usize {
|
||||
let data_len = size_hint.min(erasure.block_size);
|
||||
erasure.encoded_capacity_for_data_len(data_len).min(erasure.block_size)
|
||||
}
|
||||
|
||||
/// Keeps the encoder producer scoped to its parent future. Tokio detaches a
|
||||
/// task when its `JoinHandle` is dropped, so the producer must be aborted when
|
||||
/// an upload is cancelled before the encode pipeline finishes.
|
||||
@@ -540,13 +545,14 @@ impl Erasure {
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
require_single_block: bool,
|
||||
size_hint: usize,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let mut buf = Vec::with_capacity(self.block_size);
|
||||
let mut buf = Vec::with_capacity(small_ingest_capacity(&self, size_hint));
|
||||
let total = if require_single_block {
|
||||
let read_limit = self
|
||||
.block_size
|
||||
@@ -880,7 +886,24 @@ impl Erasure {
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
self.encode_small_direct(reader, writers, quorum, false).await
|
||||
let size_hint = self.block_size;
|
||||
self.encode_small_direct(reader, writers, quorum, false, size_hint).await
|
||||
}
|
||||
|
||||
/// Size-aware inline fast path. `size_hint` only controls the bounded initial
|
||||
/// allocation; reads remain authoritative.
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub async fn encode_inline_small_with_size_hint<R>(
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
size_hint: usize,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
self.encode_small_direct(reader, writers, quorum, false, size_hint).await
|
||||
}
|
||||
|
||||
/// Fast path for single-block non-inline objects: avoids the producer/consumer
|
||||
@@ -895,7 +918,24 @@ impl Erasure {
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
self.encode_small_direct(reader, writers, quorum, true).await
|
||||
let size_hint = self.block_size;
|
||||
self.encode_small_direct(reader, writers, quorum, true, size_hint).await
|
||||
}
|
||||
|
||||
/// Size-aware single-block fast path. `size_hint` only controls the bounded
|
||||
/// initial allocation; reads remain authoritative.
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub async fn encode_single_block_non_inline_with_size_hint<R>(
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
size_hint: usize,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
self.encode_small_direct(reader, writers, quorum, true, size_hint).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2293,7 +2333,10 @@ mod tests {
|
||||
|
||||
let erasure = Arc::new(Erasure::new(1, 0, 16));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(Vec::<u8>::new()));
|
||||
let (_reader, total) = erasure.encode_inline_small(reader, &mut writers, 1).await.unwrap();
|
||||
let (_reader, total) = erasure
|
||||
.encode_inline_small_with_size_hint(reader, &mut writers, 1, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(total, 0);
|
||||
// No shutdown was called, so nothing should be committed
|
||||
@@ -2325,7 +2368,10 @@ mod tests {
|
||||
let payload = b"hello inline small";
|
||||
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(payload.to_vec()));
|
||||
let (_reader, total) = erasure.encode_inline_small(reader, &mut writers, DATA_SHARDS).await.unwrap();
|
||||
let (_reader, total) = erasure
|
||||
.encode_inline_small_with_size_hint(reader, &mut writers, DATA_SHARDS, 1)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(total, payload.len());
|
||||
// All shards must have received data (shutdown flushed the bitrot header + shard bytes)
|
||||
@@ -2392,7 +2438,7 @@ mod tests {
|
||||
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(payload));
|
||||
let err = erasure
|
||||
.encode_single_block_non_inline(reader, &mut writers, DATA_SHARDS)
|
||||
.encode_single_block_non_inline_with_size_hint(reader, &mut writers, DATA_SHARDS, BLOCK_SIZE)
|
||||
.await
|
||||
.expect_err("single-block fast path must reject oversized readers");
|
||||
|
||||
@@ -2403,6 +2449,21 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn small_ingest_capacity_uses_bounded_size_hint() {
|
||||
let erasure = Erasure::new(4, 2, 1024 * 1024);
|
||||
assert_eq!(small_ingest_capacity(&erasure, 0), 0);
|
||||
assert_eq!(small_ingest_capacity(&erasure, 4 * 1024), 6 * 1024);
|
||||
assert_eq!(small_ingest_capacity(&erasure, 16 * 1024), 24 * 1024);
|
||||
assert_eq!(small_ingest_capacity(&erasure, usize::MAX), 1024 * 1024);
|
||||
|
||||
let legacy = Erasure::new_with_options(4, 2, 1024 * 1024, true);
|
||||
assert_eq!(small_ingest_capacity(&legacy, 4 * 1024), 6 * 1024);
|
||||
|
||||
let high_parity = Erasure::new(4, 12, 1024 * 1024);
|
||||
assert_eq!(small_ingest_capacity(&high_parity, usize::MAX), 1024 * 1024);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_full_buf_or_eof_returns_none_on_empty_reader() {
|
||||
let mut reader = Cursor::new(Vec::<u8>::new());
|
||||
|
||||
@@ -968,6 +968,15 @@ impl Erasure {
|
||||
self.data_shards + self.parity_shards
|
||||
}
|
||||
|
||||
pub(crate) fn encoded_capacity_for_data_len(&self, data_len: usize) -> usize {
|
||||
let shard_size_fn = if self.uses_legacy {
|
||||
calc_shard_size_legacy
|
||||
} else {
|
||||
calc_shard_size
|
||||
};
|
||||
shard_size_fn(data_len, self.data_shards).saturating_mul(self.total_shard_count())
|
||||
}
|
||||
|
||||
/// Whether the erasure dimensions are safe for the shard/offset arithmetic.
|
||||
///
|
||||
/// `block_size` and `data_shards` come straight from on-disk metadata; a
|
||||
|
||||
@@ -204,6 +204,8 @@ pub enum StorageError {
|
||||
required: usize,
|
||||
achieved: usize,
|
||||
},
|
||||
#[error("Bucket quota exceeded. Current usage: {current} bytes, limit: {limit} bytes")]
|
||||
QuotaExceeded { current: u64, limit: u64 },
|
||||
|
||||
// ── Generic ──────────────────────────────────────────────────────
|
||||
#[error("Unexpected error")]
|
||||
@@ -356,6 +358,13 @@ impl From<StorageError> for DiskError {
|
||||
StorageError::VolumeNotFound => DiskError::VolumeNotFound,
|
||||
StorageError::VolumeExists => DiskError::VolumeExists,
|
||||
StorageError::FileNameTooLong => DiskError::FileNameTooLong,
|
||||
StorageError::FaultyRemoteDisk => DiskError::FaultyRemoteDisk,
|
||||
StorageError::DiskAccessDenied => DiskError::DiskAccessDenied,
|
||||
StorageError::DriveIsRoot => DiskError::DriveIsRoot,
|
||||
StorageError::IsNotRegular => DiskError::IsNotRegular,
|
||||
StorageError::VolumeNotEmpty => DiskError::VolumeNotEmpty,
|
||||
StorageError::VolumeAccessDenied => DiskError::VolumeAccessDenied,
|
||||
StorageError::FileAccessDenied => DiskError::FileAccessDenied,
|
||||
_ => DiskError::other(val),
|
||||
}
|
||||
}
|
||||
@@ -540,6 +549,10 @@ impl Clone for StorageError {
|
||||
required: *required,
|
||||
achieved: *achieved,
|
||||
},
|
||||
StorageError::QuotaExceeded { current, limit } => StorageError::QuotaExceeded {
|
||||
current: *current,
|
||||
limit: *limit,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -627,6 +640,7 @@ impl StorageError {
|
||||
StorageError::NotModified => StorageErrorCode::NotModified,
|
||||
StorageError::InvalidPartNumber(_) => StorageErrorCode::InvalidPartNumber,
|
||||
StorageError::NamespaceLockQuorumUnavailable { .. } => StorageErrorCode::NamespaceLockQuorumUnavailable,
|
||||
StorageError::QuotaExceeded { .. } => StorageErrorCode::QuotaExceeded,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -752,6 +766,10 @@ impl StorageError {
|
||||
required: Default::default(),
|
||||
achieved: Default::default(),
|
||||
}),
|
||||
StorageErrorCode::QuotaExceeded => Some(StorageError::QuotaExceeded {
|
||||
current: Default::default(),
|
||||
limit: Default::default(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1301,6 +1319,7 @@ mod tests {
|
||||
.to_u32(),
|
||||
0x42
|
||||
);
|
||||
assert_eq!(StorageError::QuotaExceeded { current: 1, limit: 2 }.to_u32(), 0x53);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1319,6 +1338,10 @@ mod tests {
|
||||
StorageError::from_u32(0x42),
|
||||
Some(StorageError::NamespaceLockQuorumUnavailable { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
StorageError::from_u32(0x53),
|
||||
Some(StorageError::QuotaExceeded { current: 0, limit: 0 })
|
||||
));
|
||||
|
||||
// Test invalid code returns None
|
||||
assert!(StorageError::from_u32(0xFF).is_none());
|
||||
@@ -1476,6 +1499,49 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// Every DiskError variant must survive DiskError -> StorageError -> DiskError
|
||||
// unchanged. A variant that degrades to `DiskError::Io` on the way back loses
|
||||
// its identity for quorum aggregation (`reduce_errs` classifies by variant
|
||||
// equality), so ignore-list entries such as FaultyRemoteDisk and
|
||||
// DiskAccessDenied would silently stop matching.
|
||||
#[test]
|
||||
fn test_disk_error_storage_error_round_trip_identity_all_variants() {
|
||||
// DiskError codes are contiguous from 0x01, so enumerating via from_u32
|
||||
// covers every variant and picks up newly appended ones automatically.
|
||||
let all_variants: Vec<DiskError> = (1u32..).map_while(DiskError::from_u32).collect();
|
||||
assert!(
|
||||
all_variants.len() >= 42,
|
||||
"DiskError variant enumeration shrank: got {}, expected at least 42",
|
||||
all_variants.len()
|
||||
);
|
||||
|
||||
for original in all_variants {
|
||||
let storage_error: StorageError = original.clone().into();
|
||||
let round_tripped: DiskError = storage_error.into();
|
||||
|
||||
assert_eq!(
|
||||
std::mem::discriminant(&original),
|
||||
std::mem::discriminant(&round_tripped),
|
||||
"round trip changed variant: {original:?} -> {round_tripped:?}"
|
||||
);
|
||||
assert_eq!(original, round_tripped, "round trip not identical for {original:?}");
|
||||
}
|
||||
|
||||
// Io is the only payload-carrying variant: a representative kind and
|
||||
// message must both survive the round trip.
|
||||
let io_original = DiskError::Io(IoError::new(ErrorKind::PermissionDenied, "denied"));
|
||||
let storage_error: StorageError = io_original.clone().into();
|
||||
let io_round_tripped: DiskError = storage_error.into();
|
||||
assert_eq!(io_original, io_round_tripped);
|
||||
match io_round_tripped {
|
||||
DiskError::Io(inner) => {
|
||||
assert_eq!(inner.kind(), ErrorKind::PermissionDenied);
|
||||
assert_eq!(inner.to_string(), "denied");
|
||||
}
|
||||
other => panic!("expected DiskError::Io, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_storage_error_from_io_error() {
|
||||
// Test direct IO error conversion
|
||||
@@ -1549,6 +1615,7 @@ mod tests {
|
||||
StorageError::DecommissionAlreadyRunning,
|
||||
StorageError::RebalanceAlreadyRunning,
|
||||
StorageError::OperationCanceled,
|
||||
StorageError::QuotaExceeded { current: 1, limit: 2 },
|
||||
];
|
||||
|
||||
for original_error in test_errors {
|
||||
|
||||
@@ -120,26 +120,41 @@ struct BitrotReaderSource {
|
||||
|
||||
impl BitrotReaderSource {
|
||||
async fn open(self) -> disk::error::Result<Option<BoxedObjectReader>> {
|
||||
if let Some(data) = self.inline_data {
|
||||
let mut rd = Cursor::new(data);
|
||||
let offset = u64::try_from(self.offset).map_err(|_| DiskError::FileCorrupt)?;
|
||||
rd.set_position(offset);
|
||||
Ok(Some(ShardReader::InMemory(rd)))
|
||||
} else if let Some(disk) = self.disk {
|
||||
open_disk_reader(
|
||||
&disk,
|
||||
&self.bucket,
|
||||
&self.path,
|
||||
self.offset,
|
||||
self.length,
|
||||
self.use_mmap_read,
|
||||
self.stage_metrics.map(|metrics| metrics.path),
|
||||
)
|
||||
open_reader_source(
|
||||
self.inline_data,
|
||||
self.disk.as_ref(),
|
||||
&self.bucket,
|
||||
&self.path,
|
||||
self.offset,
|
||||
self.length,
|
||||
self.use_mmap_read,
|
||||
self.stage_metrics.map(|metrics| metrics.path),
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn open_reader_source(
|
||||
inline_data: Option<Bytes>,
|
||||
disk: Option<&DiskStore>,
|
||||
bucket: &str,
|
||||
path: &str,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
use_mmap_read: bool,
|
||||
metrics_path: Option<&'static str>,
|
||||
) -> disk::error::Result<Option<BoxedObjectReader>> {
|
||||
if let Some(data) = inline_data {
|
||||
let mut reader = Cursor::new(data);
|
||||
reader.set_position(u64::try_from(offset).map_err(|_| DiskError::FileCorrupt)?);
|
||||
Ok(Some(ShardReader::InMemory(reader)))
|
||||
} else if let Some(disk) = disk {
|
||||
open_disk_reader(disk, bucket, path, offset, length, use_mmap_read, metrics_path)
|
||||
.await
|
||||
.map(Some)
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -623,22 +638,22 @@ async fn create_bitrot_reader_from_bytes_with_stage_metrics(
|
||||
|
||||
let reader_construction_start = stage_metrics_enabled.then(Instant::now);
|
||||
let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone());
|
||||
let source = BitrotReaderSource {
|
||||
inline_data,
|
||||
disk: disk.cloned(),
|
||||
bucket: bucket.to_string(),
|
||||
path: path.to_string(),
|
||||
offset,
|
||||
length,
|
||||
use_mmap_read,
|
||||
stage_metrics,
|
||||
};
|
||||
if let Some(metrics) = stage_metrics {
|
||||
record_get_stage_duration_if_enabled(metrics.path, metrics.reader_construction_stage, reader_construction_start);
|
||||
}
|
||||
|
||||
let file_open_start = stage_metrics_enabled.then(Instant::now);
|
||||
let reader = source.open().await?;
|
||||
let reader = open_reader_source(
|
||||
inline_data,
|
||||
disk,
|
||||
bucket,
|
||||
path,
|
||||
offset,
|
||||
length,
|
||||
use_mmap_read,
|
||||
stage_metrics.map(|metrics| metrics.path),
|
||||
)
|
||||
.await?;
|
||||
if let Some(metrics) = stage_metrics {
|
||||
record_get_stage_duration_if_enabled(metrics.path, metrics.file_open_stage, file_open_start);
|
||||
}
|
||||
@@ -698,11 +713,12 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
|
||||
) -> (BitrotReader<ShardReader>, DeferredReaderStripeHandle) {
|
||||
let stripe_stride = shard_size + checksum_algo.size();
|
||||
let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone());
|
||||
let inline_source = inline_data.is_some();
|
||||
let source = BitrotReaderSource {
|
||||
inline_data,
|
||||
disk,
|
||||
bucket: bucket.to_string(),
|
||||
path: path.to_string(),
|
||||
bucket: if inline_source { String::new() } else { bucket.to_string() },
|
||||
path: if inline_source { String::new() } else { path.to_string() },
|
||||
offset,
|
||||
length,
|
||||
use_mmap_read,
|
||||
|
||||
@@ -234,11 +234,17 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn test_format_v1() {
|
||||
// A freshly created format must survive a serialize -> parse roundtrip
|
||||
// unchanged (identity on every on-disk field).
|
||||
let format = FormatV3::new(1, 4);
|
||||
let serialized = serde_json::to_string(&format).expect("FormatV3 must serialize to JSON");
|
||||
let reparsed = FormatV3::try_from(serialized.as_str()).expect("serialized FormatV3 must parse back");
|
||||
assert_eq!(reparsed, format);
|
||||
|
||||
let str = serde_json::to_string(&format);
|
||||
println!("{str:?}");
|
||||
|
||||
// minio-file-format-compat: this literal pins the on-disk format.json
|
||||
// shape (erasure version "1", distributionAlgo "CRCMOD"). `this` always
|
||||
// carries the disk's own UUID in real format.json files; a JSON null
|
||||
// there was never parseable and never written by MinIO or RustFS.
|
||||
let data = r#"
|
||||
{
|
||||
"version": "1",
|
||||
@@ -246,7 +252,7 @@ mod test {
|
||||
"id": "321b3874-987d-4c15-8fa5-757c956b1243",
|
||||
"xl": {
|
||||
"version": "1",
|
||||
"this": null,
|
||||
"this": "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
|
||||
"sets": [
|
||||
[
|
||||
"8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
|
||||
@@ -259,9 +265,23 @@ mod test {
|
||||
}
|
||||
}"#;
|
||||
|
||||
let p = FormatV3::try_from(data);
|
||||
let parsed = FormatV3::try_from(data).expect("pinned v1 format.json literal must keep parsing");
|
||||
|
||||
println!("{p:?}");
|
||||
assert_eq!(parsed.version, FormatMetaVersion::V1);
|
||||
assert_eq!(parsed.format, FormatBackend::Erasure);
|
||||
assert_eq!(
|
||||
parsed.id,
|
||||
Uuid::parse_str("321b3874-987d-4c15-8fa5-757c956b1243").expect("literal id is a valid UUID")
|
||||
);
|
||||
assert_eq!(parsed.erasure.version, FormatErasureVersion::V1);
|
||||
assert_eq!(
|
||||
parsed.erasure.this,
|
||||
Uuid::parse_str("8ab9a908-f869-4f1f-8e42-eb067ffa7eb5").expect("literal this is a valid UUID")
|
||||
);
|
||||
assert_eq!(parsed.erasure.sets.len(), 1);
|
||||
assert_eq!(parsed.erasure.sets[0].len(), 4);
|
||||
assert_eq!(parsed.erasure.sets[0][0], parsed.erasure.this);
|
||||
assert_eq!(parsed.erasure.distribution_algo, DistributionAlgoVersion::V1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -211,6 +211,26 @@ impl ObjectLockConfigSnapshot {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct QuotaAdmission {
|
||||
current_usage: u64,
|
||||
quota_limit: u64,
|
||||
}
|
||||
|
||||
impl QuotaAdmission {
|
||||
pub(crate) fn current_usage(self) -> u64 {
|
||||
self.current_usage
|
||||
}
|
||||
|
||||
pub(crate) fn quota_limit(self) -> u64 {
|
||||
self.quota_limit
|
||||
}
|
||||
|
||||
pub(crate) fn remaining(self) -> u64 {
|
||||
self.quota_limit - self.current_usage
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct ObjectOptions {
|
||||
// Use the maximum parity (N/2), used when saving server configuration files
|
||||
@@ -275,12 +295,22 @@ pub struct ObjectOptions {
|
||||
pub want_checksum: Option<Checksum>,
|
||||
pub skip_verify_bitrot: bool,
|
||||
pub capacity_scope_token: Option<Uuid>,
|
||||
/// Server-derived bucket-quota snapshot for commit-boundary admission.
|
||||
pub quota_admission: Option<QuotaAdmission>,
|
||||
/// Storage-owned journal writer used by the atomic delete path. This is
|
||||
/// populated only by the `ECStore` wrapper that holds the namespace locks.
|
||||
pub tier_delete_journal_api: Option<Arc<crate::store::ECStore>>,
|
||||
}
|
||||
|
||||
impl ObjectOptions {
|
||||
pub fn set_quota_admission(&mut self, current_usage: u64, quota_limit: u64) -> bool {
|
||||
self.quota_admission = (current_usage <= quota_limit).then_some(QuotaAdmission {
|
||||
current_usage,
|
||||
quota_limit,
|
||||
});
|
||||
self.quota_admission.is_some()
|
||||
}
|
||||
|
||||
pub(crate) fn overwrites_existing_version(&self) -> bool {
|
||||
self.version_id.is_some() || !self.versioned || self.version_suspended
|
||||
}
|
||||
|
||||
@@ -58,6 +58,7 @@ use crate::io_support::bitrot::{
|
||||
create_deferred_bitrot_reader_with_stripe_handle, object_mmap_read_enabled, object_mmap_read_max_length,
|
||||
};
|
||||
use crate::set_disk::shard_source::ShardReadCost;
|
||||
use futures::FutureExt as _;
|
||||
use futures::stream::{FuturesUnordered, StreamExt};
|
||||
use metrics::counter;
|
||||
use std::{
|
||||
@@ -221,7 +222,7 @@ impl MetadataFanoutDiagnostics {
|
||||
self.observations.iter().filter(|observation| observation.ignored).count()
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn error_responses(&self) -> usize {
|
||||
pub(in crate::set_disk) fn non_valid_responses(&self) -> usize {
|
||||
self.total_responses().saturating_sub(self.valid_responses())
|
||||
}
|
||||
|
||||
@@ -272,7 +273,7 @@ impl MetadataFanoutDiagnostics {
|
||||
self.total_responses(),
|
||||
self.valid_responses(),
|
||||
self.ignored_responses(),
|
||||
self.error_responses(),
|
||||
self.non_valid_responses(),
|
||||
);
|
||||
for observation in &self.observations {
|
||||
rustfs_io_metrics::record_get_object_metadata_response(path, observation.outcome);
|
||||
@@ -538,7 +539,7 @@ impl MetadataQuorumAccumulator {
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn default_write_quorum(&self) -> usize {
|
||||
if self.default_parity_count == 0 {
|
||||
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
|
||||
return self.total_disks;
|
||||
}
|
||||
let data_blocks = self.total_disks.saturating_sub(self.default_parity_count);
|
||||
@@ -550,7 +551,7 @@ impl MetadataQuorumAccumulator {
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize {
|
||||
if self.default_parity_count == 0 {
|
||||
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
|
||||
self.total_disks
|
||||
} else {
|
||||
self.total_disks / 2
|
||||
@@ -2222,18 +2223,18 @@ impl SetDisks {
|
||||
let mut ress = Vec::with_capacity(disks.len());
|
||||
let mut errors = Vec::with_capacity(disks.len());
|
||||
let mut observations = observe.then(|| Vec::with_capacity(disks.len()));
|
||||
let opts = Arc::new(ReadOptions {
|
||||
let opts = ReadOptions {
|
||||
incl_free_versions,
|
||||
read_data,
|
||||
healing,
|
||||
});
|
||||
let org_bucket = Arc::new(org_bucket.to_string());
|
||||
let bucket = Arc::new(bucket.to_string());
|
||||
let object = Arc::new(object.to_string());
|
||||
let version_id = Arc::new(version_id.to_string());
|
||||
};
|
||||
let org_bucket: Arc<str> = Arc::from(org_bucket);
|
||||
let bucket: Arc<str> = Arc::from(bucket);
|
||||
let object: Arc<str> = Arc::from(object);
|
||||
let version_id: Arc<str> = Arc::from(version_id);
|
||||
let futures = disks.iter().enumerate().map(|(disk_index, disk)| {
|
||||
let disk = disk.clone();
|
||||
let opts = opts.clone();
|
||||
let task_opts = opts;
|
||||
let org_bucket = org_bucket.clone();
|
||||
let bucket = bucket.clone();
|
||||
let object = object.clone();
|
||||
@@ -2242,7 +2243,8 @@ impl SetDisks {
|
||||
let response_start = observe.then(Instant::now);
|
||||
let result = if let Some(disk) = disk {
|
||||
Self::record_read_version_call(&object, disk_index);
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &opts).await
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
};
|
||||
@@ -2307,21 +2309,21 @@ impl SetDisks {
|
||||
let mut observations = Vec::with_capacity(disks.len());
|
||||
let mut accumulator =
|
||||
MetadataQuorumAccumulator::new(disks.len(), default_parity_count, true).with_requested_version_id(version_id);
|
||||
let opts = Arc::new(ReadOptions {
|
||||
let opts = ReadOptions {
|
||||
incl_free_versions,
|
||||
read_data,
|
||||
healing,
|
||||
});
|
||||
let org_bucket = Arc::new(org_bucket.to_string());
|
||||
let bucket = Arc::new(bucket.to_string());
|
||||
let object = Arc::new(object.to_string());
|
||||
let version_id = Arc::new(version_id.to_string());
|
||||
};
|
||||
let org_bucket: Arc<str> = Arc::from(org_bucket);
|
||||
let bucket: Arc<str> = Arc::from(bucket);
|
||||
let object: Arc<str> = Arc::from(object);
|
||||
let version_id: Arc<str> = Arc::from(version_id);
|
||||
let mut join_set = JoinSet::new();
|
||||
let bounded_fanout = is_get_metadata_early_stop_bounded_fanout_enabled();
|
||||
let mut next_disk_index = 0usize;
|
||||
let spawn_read_version =
|
||||
|join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| {
|
||||
let opts = opts.clone();
|
||||
let task_opts = opts;
|
||||
let org_bucket = org_bucket.clone();
|
||||
let bucket = bucket.clone();
|
||||
let object = object.clone();
|
||||
@@ -2330,7 +2332,10 @@ impl SetDisks {
|
||||
let response_start = Instant::now();
|
||||
let result = if let Some(disk) = disk {
|
||||
Self::record_read_version_call(&object, index);
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &opts).await
|
||||
#[cfg(test)]
|
||||
Self::read_version_fanout_barrier(&object, index).await;
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
};
|
||||
@@ -2397,9 +2402,13 @@ impl SetDisks {
|
||||
return Ok((ress, errors, diagnostics));
|
||||
}
|
||||
|
||||
let pending_responses = join_set.len();
|
||||
let should_hedge_single_pending_data_read =
|
||||
read_data && pending_responses == 1 && accumulator.can_still_reach_early_stop_with_pending(pending_responses);
|
||||
if bounded_fanout
|
||||
&& next_disk_index < disks.len()
|
||||
&& !accumulator.can_still_reach_early_stop_with_pending(join_set.len())
|
||||
&& (!accumulator.can_still_reach_early_stop_with_pending(pending_responses)
|
||||
|| should_hedge_single_pending_data_read)
|
||||
{
|
||||
if let Some(disk) = disks.get(next_disk_index).cloned() {
|
||||
spawn_read_version(&mut join_set, next_disk_index, disk);
|
||||
@@ -2848,8 +2857,6 @@ impl SetDisks {
|
||||
file_info.validate_for_erasure_write()?;
|
||||
}
|
||||
}
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
@@ -2857,48 +2864,65 @@ impl SetDisks {
|
||||
let dst_bucket = Arc::new(dst_bucket.to_string());
|
||||
let dst_object = Arc::new(dst_object.to_string());
|
||||
|
||||
for (i, (disk, file_info)) in disks.iter().zip(file_infos.iter()).enumerate() {
|
||||
let mut file_info = file_info.clone();
|
||||
let disk = disk.clone();
|
||||
let src_bucket = src_bucket.clone();
|
||||
let src_object = src_object.clone();
|
||||
let dst_object = dst_object.clone();
|
||||
let dst_bucket = dst_bucket.clone();
|
||||
let disk_count = disks.len();
|
||||
let fanout_disks = disks.to_vec();
|
||||
let fanout_file_infos = file_infos.to_vec();
|
||||
let fanout_src_bucket = src_bucket.clone();
|
||||
let fanout_src_object = src_object.clone();
|
||||
let fanout_dst_bucket = dst_bucket.clone();
|
||||
let fanout_dst_object = dst_object.clone();
|
||||
// Keep one coordinator task so a cancelled caller cannot drop partially
|
||||
// completed disk mutations. Per-disk futures stay ordered in `join_all`,
|
||||
// preserving slot-indexed quorum and convergence accounting without a
|
||||
// scheduler task for every disk.
|
||||
let fanout = tokio::spawn(async move {
|
||||
let futures = fanout_disks
|
||||
.into_iter()
|
||||
.zip(fanout_file_infos)
|
||||
.enumerate()
|
||||
.map(|(i, (disk, mut file_info))| {
|
||||
let src_bucket = fanout_src_bucket.clone();
|
||||
let src_object = fanout_src_object.clone();
|
||||
let dst_object = fanout_dst_object.clone();
|
||||
let dst_bucket = fanout_dst_bucket.clone();
|
||||
|
||||
futures.push(tokio::spawn(async move {
|
||||
// Test-only introspection guard: counts this task as in-flight for
|
||||
// the whole body. Compiles to `()` in production (no behavior).
|
||||
#[allow(clippy::let_unit_value)]
|
||||
let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object);
|
||||
std::panic::AssertUnwindSafe(async move {
|
||||
// Test-only introspection guard: counts this operation as
|
||||
// in-flight for the whole body. Compiles to `()` in production.
|
||||
#[allow(clippy::let_unit_value)]
|
||||
let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object);
|
||||
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
|
||||
let is_delete_marker = file_info.is_canonical_delete_marker();
|
||||
if file_info.erasure.index == 0 {
|
||||
file_info.erasure.index = i + 1;
|
||||
}
|
||||
let is_delete_marker = file_info.is_canonical_delete_marker();
|
||||
if file_info.erasure.index == 0 {
|
||||
file_info.erasure.index = i + 1;
|
||||
}
|
||||
|
||||
if !is_delete_marker && !file_info.has_valid_erasure_geometry() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
if !is_delete_marker && !file_info.has_valid_erasure_geometry() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
|
||||
// Test-only awaitable pause point right before the disk rename.
|
||||
// A no-op immediately-ready future in production.
|
||||
Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await;
|
||||
// Test-only awaitable pause point right before the disk rename.
|
||||
// A no-op immediately-ready future in production.
|
||||
Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await;
|
||||
|
||||
disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
|
||||
.await
|
||||
}));
|
||||
}
|
||||
disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
|
||||
.await
|
||||
})
|
||||
.catch_unwind()
|
||||
});
|
||||
join_all(futures).await
|
||||
});
|
||||
|
||||
let mut disk_versions = vec![None; disks.len()];
|
||||
let mut data_dirs = vec![None; disks.len()];
|
||||
let mut cleanup_data_dirs = vec![None; disks.len()];
|
||||
let mut old_current_sizes = vec![None; disks.len()];
|
||||
let mut disk_versions = vec![None; disk_count];
|
||||
let mut data_dirs = vec![None; disk_count];
|
||||
let mut cleanup_data_dirs = vec![None; disk_count];
|
||||
let mut old_current_sizes = vec![None; disk_count];
|
||||
|
||||
let results = join_all(futures).await;
|
||||
let results = fanout.await.map_err(|_| DiskError::Unexpected)?;
|
||||
|
||||
for (idx, result) in results.iter().enumerate() {
|
||||
match result.as_ref().map_err(|_| DiskError::Unexpected)? {
|
||||
@@ -3317,6 +3341,12 @@ impl SetDisks {
|
||||
#[inline(always)]
|
||||
fn record_read_version_call(_object: &str, _disk_index: usize) {}
|
||||
|
||||
#[cfg(test)]
|
||||
#[inline]
|
||||
async fn read_version_fanout_barrier(object: &str, disk_index: usize) {
|
||||
rename_fanout_barrier::checkpoint(object, disk_index, rename_fanout_barrier::PHASE_READ_VERSION).await;
|
||||
}
|
||||
|
||||
/// Test-only awaitable pause point for the rename/commit fan-out (backlog#1325,
|
||||
/// serving the barrier-style acceptances of #1312 / #1319 / #1313). `phase` is
|
||||
/// [`rename_fanout_barrier::PHASE_RENAME`] or `PHASE_CLEANUP`. When a test has
|
||||
@@ -4803,6 +4833,8 @@ pub(in crate::set_disk) mod rename_fanout_barrier_phase {
|
||||
pub const RENAME: &str = "rename";
|
||||
/// The per-disk old-data-dir cleanup phase of the commit fan-out.
|
||||
pub const CLEANUP: &str = "cleanup";
|
||||
/// The per-disk `read_version` phase of metadata read fan-out.
|
||||
pub const READ_VERSION: &str = "read_version";
|
||||
}
|
||||
|
||||
/// Test-only awaitable pause barrier + background-task introspection for the
|
||||
@@ -4846,7 +4878,9 @@ pub(in crate::set_disk) mod rename_fanout_barrier {
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
pub use super::rename_fanout_barrier_phase::{CLEANUP as PHASE_CLEANUP, RENAME as PHASE_RENAME};
|
||||
pub use super::rename_fanout_barrier_phase::{
|
||||
CLEANUP as PHASE_CLEANUP, READ_VERSION as PHASE_READ_VERSION, RENAME as PHASE_RENAME,
|
||||
};
|
||||
|
||||
/// One armed barrier: the fan-out task matching `(disk_index, phase)` pauses.
|
||||
struct Armed {
|
||||
@@ -5364,7 +5398,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_metadata_early_stop_ab_limits_data_get_read_version_fanout() {
|
||||
async fn bounded_metadata_early_stop_ab_hedges_data_get_read_version_fanout() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "bounded-data-get-fanout-bucket";
|
||||
let control_object = "bounded-data-get-control-object";
|
||||
@@ -5376,7 +5410,7 @@ mod tests {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None),
|
||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("false")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
@@ -5389,7 +5423,7 @@ mod tests {
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
DISKS as u64,
|
||||
"control path should keep the default data-read full fanout"
|
||||
"control path should keep full fanout when data-read early stop is explicitly disabled"
|
||||
);
|
||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||
},
|
||||
@@ -5419,13 +5453,109 @@ mod tests {
|
||||
.await
|
||||
.expect("healthy object metadata should reach early-stop quorum");
|
||||
|
||||
assert!(
|
||||
(3..=DISKS as u64).contains(&calls.total(disk_call_counters::KIND_READ_VERSION)),
|
||||
"healthy 2+2 bounded data-read fanout may finish at quorum before a spare hedge is needed"
|
||||
);
|
||||
assert!(
|
||||
(3..=DISKS).contains(&diagnostics.total_responses()),
|
||||
"treatment path should return after reaching quorum, with at most the spare hedge response observed"
|
||||
);
|
||||
assert!(parts_metadata.iter().filter(|fi| fi.name == treatment_object).count() >= 3);
|
||||
assert!(errs.iter().all(Option::is_none));
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn bounded_data_get_hedges_single_pending_read_version() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "bounded-data-get-hedge-bucket";
|
||||
let object = "bounded-data-get-hedge-object";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
let barrier = rename_fanout_barrier::arm(object, 2, rename_fanout_barrier::PHASE_READ_VERSION);
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let disks_for_read = disks.clone();
|
||||
let mut read = tokio::spawn(async move {
|
||||
SetDisks::read_all_fileinfo_observed(&disks_for_read, bucket, bucket, object, "", true, false, false, true, 2)
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("third scheduled read_version should pause at the deterministic barrier");
|
||||
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
|
||||
while calls.for_disk(disk_call_counters::KIND_READ_VERSION, 3) == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded data-read fanout should hedge by starting the spare disk");
|
||||
|
||||
let completed = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
|
||||
if completed.is_err() {
|
||||
barrier.release();
|
||||
}
|
||||
let (parts_metadata, errs, diagnostics) = completed
|
||||
.expect("spare metadata should allow early-stop without waiting for the paused disk")
|
||||
.expect("metadata read task should not panic")
|
||||
.expect("healthy spare metadata should resolve");
|
||||
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
3,
|
||||
"treatment path should stop after the 2+2 read/write quorum instead of issuing every disk read"
|
||||
DISKS as u64,
|
||||
"bounded data-read fanout should issue the paused disk plus one spare hedge"
|
||||
);
|
||||
assert_eq!(diagnostics.total_responses(), 3);
|
||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == treatment_object).count(), 3);
|
||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), 3);
|
||||
assert!(errs.iter().all(Option::is_none));
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_metadata_early_stop_defaults_keep_data_get_full_fanout() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "bounded-data-get-default-bucket";
|
||||
let object = "bounded-data-get-default-object";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", None::<&str>),
|
||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None::<&str>),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let (parts_metadata, errs, diagnostics) =
|
||||
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2)
|
||||
.await
|
||||
.expect("default data-read metadata should resolve");
|
||||
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
DISKS as u64,
|
||||
"default GET data-read metadata must keep full fanout for read-failure tolerance"
|
||||
);
|
||||
assert_eq!(diagnostics.total_responses(), DISKS);
|
||||
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||
assert!(errs.iter().all(Option::is_none));
|
||||
},
|
||||
)
|
||||
@@ -5753,6 +5883,51 @@ mod tests {
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_fanout_drains_after_caller_cancellation() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "rename-cancel-bucket";
|
||||
let object = "rename-cancel-object";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
let marker = metadata_test_delete_marker(object, Uuid::new_v4(), OffsetDateTime::now_utc());
|
||||
let file_infos = vec![marker; DISKS];
|
||||
let tracker = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
|
||||
let rename =
|
||||
tokio::spawn(
|
||||
async move { SetDisks::rename_data(&disks, bucket, object, &file_infos, bucket, object, DISKS - 1).await },
|
||||
);
|
||||
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("rename fan-out must reach the armed barrier");
|
||||
rename.abort();
|
||||
assert!(
|
||||
rename
|
||||
.await
|
||||
.expect_err("aborted caller should report cancellation")
|
||||
.is_cancelled(),
|
||||
"caller task should be cancelled, not panic"
|
||||
);
|
||||
assert!(tracker.running() >= 1, "the coordinator must retain in-flight disk mutations");
|
||||
|
||||
barrier.release();
|
||||
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
|
||||
while tracker.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cancelled caller's disk mutations must drain");
|
||||
|
||||
for (idx, dir) in dirs.iter().enumerate() {
|
||||
assert!(
|
||||
dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE).exists(),
|
||||
"disk {idx} must finish the rename after caller cancellation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Demo / regression guard for the barrier on the commit (old-data-dir)
|
||||
/// cleanup fan-out. Serves the same #1312/#1319 "no background disk write
|
||||
/// after release" shape, on the reclamation path that runs *after* a write is
|
||||
@@ -5923,7 +6098,7 @@ mod tests {
|
||||
assert_eq!(diagnostics.total_responses(), 3);
|
||||
assert_eq!(diagnostics.valid_responses(), 1);
|
||||
assert_eq!(diagnostics.ignored_responses(), 1);
|
||||
assert_eq!(diagnostics.error_responses(), 2);
|
||||
assert_eq!(diagnostics.non_valid_responses(), 2);
|
||||
assert_eq!(diagnostics.first_response_latency(), Some(Duration::from_millis(10)));
|
||||
assert_eq!(diagnostics.first_valid_response_latency(), Some(Duration::from_millis(30)));
|
||||
assert_eq!(diagnostics.slowest_response_latency(), Some(Duration::from_millis(30)));
|
||||
@@ -6009,6 +6184,16 @@ mod tests {
|
||||
assert_eq!(accumulator.candidate_latest_quorum(&impossible_parity), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_quorum_accumulator_treats_invalid_default_parity_as_full_fanout() {
|
||||
let accumulator = MetadataQuorumAccumulator::new(2, 2, true);
|
||||
|
||||
assert_eq!(accumulator.default_write_quorum(), 2);
|
||||
assert_eq!(accumulator.missing_response_quorum(), 2);
|
||||
assert!(accumulator.can_still_reach_early_stop_with_pending(2));
|
||||
assert!(!accumulator.can_still_reach_early_stop_with_pending(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn confirmed_missing_part_error_recognizes_legacy_and_s3_markers() {
|
||||
assert!(!is_confirmed_missing_part_error(None));
|
||||
|
||||
@@ -584,10 +584,14 @@ fn capacity_scope_from_disks(disks: &[Option<DiskStore>]) -> CapacityScope {
|
||||
///
|
||||
/// **Deprecated**: Use `adaptive_duplex_buffer_size()` for object-size-aware sizing.
|
||||
pub fn get_duplex_buffer_size() -> usize {
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_OBJECT_DUPLEX_BUFFER_SIZE,
|
||||
rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE,
|
||||
)
|
||||
static CACHED: OnceLock<usize> = OnceLock::new();
|
||||
*CACHED.get_or_init(|| {
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_OBJECT_DUPLEX_BUFFER_SIZE,
|
||||
rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE,
|
||||
)
|
||||
.max(1)
|
||||
})
|
||||
}
|
||||
|
||||
/// Get adaptive duplex buffer size based on object size.
|
||||
@@ -597,12 +601,15 @@ pub fn get_duplex_buffer_size() -> usize {
|
||||
fn adaptive_duplex_buffer_size(object_size: i64) -> usize {
|
||||
const KB: usize = 1024;
|
||||
const MB: usize = 1024 * 1024;
|
||||
match object_size {
|
||||
0..=1_048_576 => 64 * KB, // <= 1MB: 64KB
|
||||
let target = match object_size {
|
||||
0..=131_072 => 64 * KB, // <= 128KB: 64KB
|
||||
131_073..=1_048_576 => 512 * KB, // <= 1MB: reduce duplex backpressure without a 1MB pipe per request
|
||||
1_048_577..=16_777_216 => MB, // <= 16MB: 1MB
|
||||
16_777_217..=268_435_456 => 4 * MB, // <= 256MB: 4MB
|
||||
_ => 8 * MB, // > 256MB: 8MB
|
||||
}
|
||||
};
|
||||
let object_cap = usize::try_from(object_size).ok().filter(|size| *size > 0).unwrap_or(target);
|
||||
target.min(object_cap.max(64 * KB)).min(get_duplex_buffer_size())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -632,9 +639,11 @@ const ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE: &str = "RUSTFS_GET_CODEC_STREAMING_
|
||||
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENABLE: bool = true;
|
||||
|
||||
const ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_MIN_SIZE";
|
||||
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: usize = MI_B;
|
||||
// Meet the direct-memory path at its default ceiling. Codec streaming remains
|
||||
// rollout-gated and starts where the eager small-object path ends.
|
||||
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: usize = DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD;
|
||||
const ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE";
|
||||
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: usize = MI_B;
|
||||
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: usize = DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE;
|
||||
|
||||
const ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE: &str = "RUSTFS_GET_CODEC_STREAMING_ENGINE";
|
||||
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENGINE: &str = GET_CODEC_STREAMING_ENGINE_LEGACY;
|
||||
@@ -664,7 +673,13 @@ const ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE: &str = "RUSTFS_
|
||||
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE: usize = 512 * 1024;
|
||||
|
||||
const ENV_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY: &str = "RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY";
|
||||
const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY: bool = false;
|
||||
// On by default (rustfs/backlog#1802): a small object whose data shards are
|
||||
// inlined in xl.meta is reassembled straight from the already-resolved
|
||||
// metadata, skipping the Erasure reconstruct pipeline. The path has a complete
|
||||
// fallback — if the inline reassembly returns None, the GET proceeds through
|
||||
// the normal shard-read pipeline, so a miss is correctness-neutral. Set to
|
||||
// `false` to force the legacy path (kill switch).
|
||||
const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY: bool = true;
|
||||
const ENV_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD: &str = "RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD";
|
||||
const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD: usize = 128 * 1024;
|
||||
|
||||
@@ -672,10 +687,10 @@ const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD: usize = 128 * 102
|
||||
|
||||
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ENABLE";
|
||||
// Enabled by default (backlog#872): the early-stop path only engages for
|
||||
// requests `should_allow_metadata_early_stop` classifies as safe (metadata-only
|
||||
// reads by default, without version_id / healing / free-version needs) and
|
||||
// still requires a full read-quorum agreement before stopping. Set the env var
|
||||
// to `false` to fall back to full-wait metadata fanout.
|
||||
// requests `should_allow_metadata_early_stop` classifies as safe (latest-version
|
||||
// metadata-only reads by default, without version_id / healing / free-version
|
||||
// needs) and still requires a full read-quorum agreement before stopping. Set
|
||||
// the env var to `false` to fall back to full-wait metadata fanout.
|
||||
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: bool = true;
|
||||
|
||||
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT";
|
||||
@@ -720,10 +735,41 @@ mod transition_matrix_tests;
|
||||
|
||||
pub use ops::heal_walk::HealWalkVersion;
|
||||
|
||||
pub(in crate::set_disk) enum GetObjectMetadata<T> {
|
||||
Owned(T),
|
||||
Shared(Arc<T>),
|
||||
}
|
||||
|
||||
impl<T> std::ops::Deref for GetObjectMetadata<T> {
|
||||
type Target = T;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
match self {
|
||||
Self::Owned(value) => value,
|
||||
Self::Shared(value) => value,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Clone> GetObjectMetadata<T> {
|
||||
fn into_owned(self) -> T {
|
||||
match self {
|
||||
Self::Owned(value) => value,
|
||||
Self::Shared(value) => Arc::try_unwrap(value).unwrap_or_else(|value| (*value).clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type GetObjectFileInfo = (
|
||||
GetObjectMetadata<FileInfo>,
|
||||
GetObjectMetadata<Vec<FileInfo>>,
|
||||
GetObjectMetadata<Vec<Option<DiskStore>>>,
|
||||
);
|
||||
|
||||
pub(crate) struct PreparedGetObjectMetadata {
|
||||
fi: FileInfo,
|
||||
files: Vec<FileInfo>,
|
||||
disks: Vec<Option<DiskStore>>,
|
||||
fi: GetObjectMetadata<FileInfo>,
|
||||
files: GetObjectMetadata<Vec<FileInfo>>,
|
||||
disks: GetObjectMetadata<Vec<Option<DiskStore>>>,
|
||||
object_info: Option<ObjectInfo>,
|
||||
}
|
||||
|
||||
@@ -792,9 +838,9 @@ mod prepared_get_object_metadata_tests {
|
||||
#[tokio::test]
|
||||
async fn prepared_metadata_is_consumed_exactly_once() {
|
||||
let metadata = PreparedGetObjectMetadata {
|
||||
fi: FileInfo::default(),
|
||||
files: Vec::new(),
|
||||
disks: Vec::new(),
|
||||
fi: GetObjectMetadata::Owned(FileInfo::default()),
|
||||
files: GetObjectMetadata::Owned(Vec::new()),
|
||||
disks: GetObjectMetadata::Owned(Vec::new()),
|
||||
object_info: None,
|
||||
};
|
||||
|
||||
@@ -836,11 +882,8 @@ mod prepared_get_object_metadata_tests {
|
||||
.prepare_get_object_metadata(bucket, object, &opts)
|
||||
.await
|
||||
.expect("prepared metadata should resolve");
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
4,
|
||||
"preparation should fan out to each online disk exactly once"
|
||||
);
|
||||
let prepared_calls = calls.total(disk_call_counters::KIND_READ_VERSION);
|
||||
assert_eq!(prepared_calls, 4, "default prepared GET metadata should keep full data-read fanout");
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader_with_prepared_metadata(bucket, object, None, HeaderMap::new(), &opts, metadata)
|
||||
@@ -1610,8 +1653,6 @@ enum GetDirectMemoryFallbackReason {
|
||||
Range,
|
||||
PartNumber,
|
||||
VersionId,
|
||||
Versioned,
|
||||
VersionSuspended,
|
||||
InclFreeVersions,
|
||||
SkipFreeVersion,
|
||||
DataMovement,
|
||||
@@ -1637,8 +1678,6 @@ impl GetDirectMemoryFallbackReason {
|
||||
Self::Range => "range",
|
||||
Self::PartNumber => "part_number",
|
||||
Self::VersionId => "version_id",
|
||||
Self::Versioned => "versioned",
|
||||
Self::VersionSuspended => "version_suspended",
|
||||
Self::InclFreeVersions => "incl_free_versions",
|
||||
Self::SkipFreeVersion => "skip_free_version",
|
||||
Self::DataMovement => "data_movement",
|
||||
@@ -1762,12 +1801,11 @@ fn get_small_object_direct_memory_decision_with_threshold(
|
||||
if opts.version_id.is_some() {
|
||||
return GetDirectMemoryDecision::Fallback(GetDirectMemoryFallbackReason::VersionId);
|
||||
}
|
||||
if opts.versioned {
|
||||
return GetDirectMemoryDecision::Fallback(GetDirectMemoryFallbackReason::Versioned);
|
||||
}
|
||||
if opts.version_suspended {
|
||||
return GetDirectMemoryDecision::Fallback(GetDirectMemoryFallbackReason::VersionSuspended);
|
||||
}
|
||||
// Bucket-level versioning no longer blocks the inline path (rustfs/backlog#1802):
|
||||
// `fi` here is the already-resolved target version, so reassembling its inlined
|
||||
// data shards is correct whether the bucket is versioned or not. This direct-memory
|
||||
// decision still falls back for an explicit versionId (the `version_id` check above);
|
||||
// a delete-marker latest is rejected below.
|
||||
if opts.incl_free_versions {
|
||||
return GetDirectMemoryDecision::Fallback(GetDirectMemoryFallbackReason::InclFreeVersions);
|
||||
}
|
||||
@@ -2329,6 +2367,8 @@ pub struct SetDisks {
|
||||
pub default_parity_count: usize,
|
||||
pub set_index: usize,
|
||||
pub pool_index: usize,
|
||||
/// Stable namespace shared by every object lock created for this set.
|
||||
set_lock_namespace: Arc<str>,
|
||||
pub format: FormatV3,
|
||||
disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
|
||||
get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>,
|
||||
@@ -2458,13 +2498,13 @@ impl Hash for GetObjectMetadataCacheKey {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
#[derive(Debug)]
|
||||
struct GetObjectMetadataCacheEntry {
|
||||
#[allow(dead_code)] // Kept for debugging; moka handles TTL internally
|
||||
created_at: Instant,
|
||||
fi: FileInfo,
|
||||
parts_metadata: Vec<FileInfo>,
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
fi: Arc<FileInfo>,
|
||||
parts_metadata: Arc<Vec<FileInfo>>,
|
||||
online_disks: Arc<Vec<Option<DiskStore>>>,
|
||||
read_quorum: usize,
|
||||
}
|
||||
|
||||
@@ -2730,6 +2770,7 @@ impl SetDisks {
|
||||
instance_ctx: Arc<InstanceContext>,
|
||||
) -> Arc<Self> {
|
||||
let ctx = instance_ctx;
|
||||
let set_lock_namespace: Arc<str> = format!("set-{pool_index}-{set_index}").into();
|
||||
Arc::new(SetDisks {
|
||||
locker_owner,
|
||||
disks,
|
||||
@@ -2737,6 +2778,7 @@ impl SetDisks {
|
||||
default_parity_count,
|
||||
set_index,
|
||||
pool_index,
|
||||
set_lock_namespace,
|
||||
format,
|
||||
set_endpoints,
|
||||
disk_health_cache: Arc::new(RwLock::new(Vec::new())),
|
||||
@@ -3185,23 +3227,28 @@ async fn try_read_inline_data_shards_direct(
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut body = Vec::with_capacity(object_size);
|
||||
let mut remaining = object_size;
|
||||
for reader in readers.iter_mut().take(data_shards) {
|
||||
let shards_needed = object_size.div_ceil(read_length);
|
||||
if shards_needed > data_shards {
|
||||
return None;
|
||||
}
|
||||
let encoded_capacity = read_length.checked_mul(shards_needed)?;
|
||||
let mut body = Vec::with_capacity(encoded_capacity);
|
||||
for reader in readers.iter_mut().take(shards_needed) {
|
||||
let reader = reader.as_mut()?;
|
||||
let mut shard = vec![0u8; read_length];
|
||||
let Ok(read) = reader.read(&mut shard).await else {
|
||||
let Ok(read) = reader.read_appending(&mut body, read_length).await else {
|
||||
return None;
|
||||
};
|
||||
if read != read_length {
|
||||
return None;
|
||||
}
|
||||
|
||||
let take = remaining.min(shard.len());
|
||||
body.extend_from_slice(&shard[..take]);
|
||||
remaining -= take;
|
||||
if remaining == 0 {
|
||||
return Some(Bytes::from(body));
|
||||
if body.len() >= object_size {
|
||||
let body = Bytes::from(body);
|
||||
return Some(if body.len() == object_size {
|
||||
body
|
||||
} else {
|
||||
body.slice(..object_size)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4891,6 +4938,28 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_ns_lock_reuses_the_set_namespace_allocation() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
|
||||
assert_eq!(&*set.set_lock_namespace, "set-0-0");
|
||||
let before = Arc::strong_count(&set.set_lock_namespace);
|
||||
let lock = set
|
||||
.new_ns_lock("bucket", "object")
|
||||
.await
|
||||
.expect("namespace lock should be created");
|
||||
|
||||
assert_eq!(
|
||||
Arc::strong_count(&set.set_lock_namespace),
|
||||
before + 1,
|
||||
"each lock should share the set namespace instead of formatting a new String"
|
||||
);
|
||||
drop(lock);
|
||||
assert_eq!(Arc::strong_count(&set.set_lock_namespace), before);
|
||||
}
|
||||
|
||||
struct SetupTypeGuard {
|
||||
previous: SetupType,
|
||||
}
|
||||
@@ -8677,9 +8746,11 @@ mod tests {
|
||||
128 * 1024
|
||||
));
|
||||
|
||||
// Bucket-level versioning no longer blocks the inline path (rustfs/backlog#1802):
|
||||
// a latest-version read on a versioned bucket is eligible.
|
||||
let mut versioned_opts = opts.clone();
|
||||
versioned_opts.versioned = true;
|
||||
assert!(!is_get_small_object_direct_memory_eligible_with_threshold(
|
||||
assert!(is_get_small_object_direct_memory_eligible_with_threshold(
|
||||
&None,
|
||||
&object_info,
|
||||
&fi,
|
||||
@@ -8740,11 +8811,13 @@ mod tests {
|
||||
GetDirectMemoryDecision::Fallback(GetDirectMemoryFallbackReason::Range)
|
||||
);
|
||||
|
||||
// Bucket-level versioning no longer falls back (rustfs/backlog#1802): the
|
||||
// latest version on a versioned bucket is served inline like any other.
|
||||
let mut versioned_opts = opts.clone();
|
||||
versioned_opts.versioned = true;
|
||||
assert_eq!(
|
||||
get_small_object_direct_memory_decision_with_threshold(&None, &object_info, &fi, &versioned_opts, true, 128 * 1024),
|
||||
GetDirectMemoryDecision::Fallback(GetDirectMemoryFallbackReason::Versioned)
|
||||
GetDirectMemoryDecision::Use { object_size: 1024 }
|
||||
);
|
||||
|
||||
let mut encrypted = object_info.clone();
|
||||
@@ -8792,8 +8865,6 @@ mod tests {
|
||||
assert_eq!(GetDirectMemoryFallbackReason::Range.as_str(), "range");
|
||||
assert_eq!(GetDirectMemoryFallbackReason::PartNumber.as_str(), "part_number");
|
||||
assert_eq!(GetDirectMemoryFallbackReason::VersionId.as_str(), "version_id");
|
||||
assert_eq!(GetDirectMemoryFallbackReason::Versioned.as_str(), "versioned");
|
||||
assert_eq!(GetDirectMemoryFallbackReason::VersionSuspended.as_str(), "version_suspended");
|
||||
assert_eq!(GetDirectMemoryFallbackReason::InclFreeVersions.as_str(), "incl_free_versions");
|
||||
assert_eq!(GetDirectMemoryFallbackReason::SkipFreeVersion.as_str(), "skip_free_version");
|
||||
assert_eq!(GetDirectMemoryFallbackReason::DataMovement.as_str(), "data_movement");
|
||||
@@ -8840,10 +8911,17 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
async fn inline_bitrot_files_for_payload(payload: &[u8]) -> (coding::Erasure, Vec<FileInfo>, usize, HashAlgorithm) {
|
||||
let erasure = coding::Erasure::new(4, 2, 1024 * 1024);
|
||||
async fn inline_bitrot_files_for_payload_with_mode(
|
||||
payload: &[u8],
|
||||
uses_legacy: bool,
|
||||
) -> (coding::Erasure, Vec<FileInfo>, usize, HashAlgorithm) {
|
||||
let erasure = coding::Erasure::new_with_options(4, 2, 1024 * 1024, uses_legacy);
|
||||
let read_length = erasure.shard_file_offset(0, payload.len(), payload.len());
|
||||
let checksum_algo = HashAlgorithm::HighwayHash256S;
|
||||
let checksum_algo = if uses_legacy {
|
||||
HashAlgorithm::HighwayHash256SLegacy
|
||||
} else {
|
||||
HashAlgorithm::HighwayHash256S
|
||||
};
|
||||
let shards = erasure.encode_data(payload).expect("payload should encode");
|
||||
let mut files = Vec::with_capacity(shards.len());
|
||||
|
||||
@@ -8865,6 +8943,10 @@ mod tests {
|
||||
(erasure, files, read_length, checksum_algo)
|
||||
}
|
||||
|
||||
async fn inline_bitrot_files_for_payload(payload: &[u8]) -> (coding::Erasure, Vec<FileInfo>, usize, HashAlgorithm) {
|
||||
inline_bitrot_files_for_payload_with_mode(payload, false).await
|
||||
}
|
||||
|
||||
fn inline_data_shard_fileinfo(
|
||||
name: &str,
|
||||
data_blocks: usize,
|
||||
@@ -8944,15 +9026,41 @@ mod tests {
|
||||
assert_eq!(body.as_ref(), payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inline_data_shards_direct_read_reassembles_legacy_payload_with_padding() {
|
||||
let payload = b"legacy inline payload whose size is not divisible by the data shard count";
|
||||
let (erasure, files, read_length, checksum_algo) = inline_bitrot_files_for_payload_with_mode(payload, true).await;
|
||||
assert_ne!(payload.len() % erasure.data_shards, 0, "test payload must exercise EC padding");
|
||||
let mut readers = build_inline_bitrot_readers(
|
||||
&files,
|
||||
erasure.data_shards,
|
||||
"bucket",
|
||||
"object",
|
||||
read_length,
|
||||
erasure.shard_size(),
|
||||
&checksum_algo,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("legacy inline bitrot readers should build");
|
||||
|
||||
let body = try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, payload.len())
|
||||
.await
|
||||
.expect("legacy data shard direct read should succeed");
|
||||
|
||||
assert_eq!(body.len(), payload.len());
|
||||
assert_eq!(body.as_ref(), payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inline_data_shards_direct_read_rejects_corrupt_shard() {
|
||||
let payload = b"small inline object payload that will be corrupted";
|
||||
let (erasure, mut files, read_length, checksum_algo) = inline_bitrot_files_for_payload(payload).await;
|
||||
let first = files[0].data.as_mut().expect("first shard should exist");
|
||||
let mut corrupted = first.to_vec();
|
||||
let second = files[1].data.as_mut().expect("second shard should exist");
|
||||
let mut corrupted = second.to_vec();
|
||||
let last = corrupted.last_mut().expect("encoded shard should not be empty");
|
||||
*last ^= 0xff;
|
||||
*first = Bytes::from(corrupted);
|
||||
*second = Bytes::from(corrupted);
|
||||
|
||||
let mut readers = build_inline_bitrot_readers(
|
||||
&files,
|
||||
@@ -8969,7 +9077,7 @@ mod tests {
|
||||
|
||||
let body = try_read_inline_data_shards_direct(&mut readers, 4, read_length, payload.len()).await;
|
||||
|
||||
assert!(body.is_none());
|
||||
assert!(body.is_none(), "a later corrupt shard must discard the already-appended body prefix");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -9045,6 +9153,71 @@ mod tests {
|
||||
assert_eq!(body.as_ref(), payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_memory_versioned_bucket_uses_inline_data_shards_for_latest() {
|
||||
let tempdir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let endpoint =
|
||||
Endpoint::try_from(tempdir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("disk should be created");
|
||||
|
||||
let payload = vec![b'v'; 64 * 1024];
|
||||
let payload_size = i64::try_from(payload.len()).expect("test payload size should fit i64");
|
||||
let (erasure, files, _read_length, _checksum_algo) = inline_bitrot_files_for_payload(&payload).await;
|
||||
let mut fi = FileInfo::new("bucket/object", erasure.data_shards, erasure.parity_shards);
|
||||
fi.size = payload_size;
|
||||
fi.data = files[0].data.clone();
|
||||
fi.add_object_part(1, String::new(), payload.len(), None, payload_size, None, None);
|
||||
|
||||
let mut object_info = ObjectInfo {
|
||||
size: payload_size,
|
||||
actual_size: payload_size,
|
||||
parts: Arc::new(vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: payload.len(),
|
||||
actual_size: payload_size,
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
object_info.inlined = true;
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
let metrics_size_bucket = rustfs_io_metrics::get_object_size_bucket(fi.size);
|
||||
|
||||
assert_eq!(
|
||||
get_small_object_direct_memory_decision_with_threshold(&None, &object_info, &fi, &opts, true, 128 * 1024),
|
||||
GetDirectMemoryDecision::Use {
|
||||
object_size: payload.len()
|
||||
}
|
||||
);
|
||||
|
||||
let body = SetDisks::try_get_object_direct_data_shards_with_fileinfo(
|
||||
"bucket",
|
||||
"object",
|
||||
&fi,
|
||||
&files,
|
||||
&vec![Some(disk); erasure.total_shard_count()],
|
||||
true,
|
||||
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART,
|
||||
metrics_size_bucket,
|
||||
)
|
||||
.await
|
||||
.expect("versioned latest direct-memory read should not fail")
|
||||
.expect("versioned latest should use inline data shards");
|
||||
|
||||
assert_eq!(body.as_ref(), payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_memory_data_shards_direct_read_reassembles_single_block_payload() {
|
||||
use uuid::Uuid;
|
||||
@@ -11132,4 +11305,13 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adaptive_duplex_buffer_size_raises_mid_sized_gets_without_penalizing_tiny_objects() {
|
||||
assert_eq!(adaptive_duplex_buffer_size(64 * 1024), 64 * 1024);
|
||||
assert_eq!(adaptive_duplex_buffer_size(128 * 1024), 64 * 1024);
|
||||
assert_eq!(adaptive_duplex_buffer_size(256 * 1024), 256 * 1024);
|
||||
assert_eq!(adaptive_duplex_buffer_size(1024 * 1024), 512 * 1024);
|
||||
assert_eq!(adaptive_duplex_buffer_size(2 * 1024 * 1024), 1024 * 1024);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -362,9 +362,9 @@ impl SetDisks {
|
||||
healing: true,
|
||||
};
|
||||
let checks = target_disks.into_iter().map(|disk| {
|
||||
let read_options = read_options.clone();
|
||||
let task_read_options = read_options;
|
||||
async move {
|
||||
let file_info = match disk.read_version("", bucket, object, version_id, &read_options).await {
|
||||
let file_info = match disk.read_version("", bucket, object, version_id, &task_read_options).await {
|
||||
Ok(file_info) => file_info,
|
||||
Err(
|
||||
DiskError::DiskNotFound
|
||||
@@ -2558,6 +2558,96 @@ mod heal_result_report_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_target_readback_checks_the_requested_historical_version() {
|
||||
let (temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "replacement-target-readback-versioned";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned bucket should be created");
|
||||
|
||||
let mut old_reader = PutObjReader::from_vec(vec![0x5a; 1024 * 1024]);
|
||||
let old_info = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut old_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("old object version should be written");
|
||||
let old_version = old_info
|
||||
.version_id
|
||||
.expect("versioned put should return the old version id")
|
||||
.to_string();
|
||||
let mut latest_reader = PutObjReader::from_vec(vec![0x33; 1024 * 1024]);
|
||||
let latest_info = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut latest_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("latest object version should be written");
|
||||
let latest_version = latest_info
|
||||
.version_id
|
||||
.expect("versioned put should return the latest version id")
|
||||
.to_string();
|
||||
let old_source = disks[2]
|
||||
.read_version("", bucket, object, &old_version, &ReadOptions::default())
|
||||
.await
|
||||
.expect("old version metadata should be readable");
|
||||
let old_data_dir = old_source.data_dir.expect("old version should have a data directory");
|
||||
let targets = vec![set.set_endpoints[0].to_string(), set.set_endpoints[1].to_string()];
|
||||
|
||||
assert!(
|
||||
set.replacement_targets_have_version(bucket, object, &old_version, &targets)
|
||||
.await
|
||||
.expect("healthy historical target shards should be readable")
|
||||
);
|
||||
assert!(
|
||||
set.replacement_targets_have_version(bucket, object, &latest_version, &targets)
|
||||
.await
|
||||
.expect("healthy latest target shards should be readable")
|
||||
);
|
||||
|
||||
tokio::fs::remove_file(
|
||||
temp_dirs[1]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(old_data_dir.to_string())
|
||||
.join("part.1"),
|
||||
)
|
||||
.await
|
||||
.expect("old target shard should be removed after the initial commit");
|
||||
|
||||
assert!(
|
||||
!set.replacement_targets_have_version(bucket, object, &old_version, &targets)
|
||||
.await
|
||||
.expect("missing old target shard should be observable")
|
||||
);
|
||||
assert!(
|
||||
set.replacement_targets_have_version(bucket, object, &latest_version, &targets)
|
||||
.await
|
||||
.expect("latest target evidence should remain independent")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_heal_cached_layout_rejects_a_disk_from_another_slot() {
|
||||
let mut _temp_dirs = Vec::new();
|
||||
|
||||
@@ -39,16 +39,9 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
|
||||
// Calculate quorum based on lockers count (majority)
|
||||
let lockers_count = self.lockers.len();
|
||||
let write_quorum = if lockers_count > 1 { (lockers_count / 2) + 1 } else { 1 };
|
||||
NamespaceLock::with_clients_and_quorum(
|
||||
format!("set-{}-{}", self.pool_index, self.set_index),
|
||||
self.lockers.clone(),
|
||||
write_quorum,
|
||||
)
|
||||
NamespaceLock::with_clients_and_quorum_shared(self.set_lock_namespace.clone(), self.lockers.clone(), write_quorum)
|
||||
} else {
|
||||
NamespaceLock::Local(LocalLock::new(
|
||||
format!("set-{}-{}", self.pool_index, self.set_index),
|
||||
self.local_lock_manager.clone(),
|
||||
))
|
||||
NamespaceLock::with_local_manager_shared(self.set_lock_namespace.clone(), self.local_lock_manager.clone())
|
||||
};
|
||||
|
||||
let resource = ObjectKey {
|
||||
|
||||
@@ -162,6 +162,22 @@ fn map_upload_id_metadata_error(bucket: &str, object: &str, upload_id: &str, err
|
||||
err.into()
|
||||
}
|
||||
|
||||
/// Abort a multipart commit when the guard's refresh heartbeat has observed a
|
||||
/// refresh-quorum loss (backlog#899 Phase 2): a stale holder must not race a
|
||||
/// concurrent committer past its fenced commit point.
|
||||
fn fence_commit_on_lock_loss(guard: Option<&ObjectLockDiagGuard>, mode: &'static str, lock_path: &str) -> Result<()> {
|
||||
if guard.is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode,
|
||||
bucket: RUSTFS_META_MULTIPART_BUCKET.to_string(),
|
||||
object: lock_path.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn multipart_bucket_incarnation_id(metadata: &HashMap<String, String>) -> Result<Option<Uuid>> {
|
||||
let Some(value) = rustfs_utils::http::metadata_compat::get_consistent_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) else {
|
||||
if rustfs_utils::http::metadata_compat::contains_key_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) {
|
||||
@@ -954,12 +970,17 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
let write_path = classify_multipart_part_write_path(multipart_part_size, fi.erasure.block_size);
|
||||
rustfs_io_metrics::record_put_object_path(write_path.multipart_metric_label());
|
||||
let small_size_hint = if matches!(write_path, SmallWritePath::SingleBlockNonInline) {
|
||||
usize::try_from(multipart_part_size).map_err(Error::other)?
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let encode_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
|
||||
let (reader, w_size) = match write_path {
|
||||
SmallWritePath::SingleBlockNonInline => {
|
||||
Arc::clone(&erasure)
|
||||
.encode_single_block_non_inline(stream, &mut writers, write_quorum)
|
||||
.encode_single_block_non_inline_with_size_hint(stream, &mut writers, write_quorum, small_size_hint)
|
||||
.await?
|
||||
}
|
||||
SmallWritePath::PipelineBatchedLarge => {
|
||||
@@ -1066,29 +1087,38 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
|
||||
let part_path = format!("{}/{}/{}", upload_id_path, fi.data_dir.unwrap_or_default(), part_suffix);
|
||||
let part_lock_path = format!("{upload_id_path}/{part_suffix}");
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockAcquire).await;
|
||||
// Serialize only the commit (rename_part), not the whole upload. Each
|
||||
// concurrent stream writes to its own unique temp dir (see `tmp_part`
|
||||
// above), so the encode/stream phase never conflicts and must stay
|
||||
// lock-free — holding a lock across it would serialize slow re-transmits
|
||||
// of the same part and defeat the S3 "last finisher wins" semantics
|
||||
// (it also caused UploadPart lock-acquire timeouts). The mixed-generation
|
||||
// hazard is confined to rename_part, where two temp parts are moved
|
||||
// cross-disk onto the SAME final part_path: interleaving there can leave
|
||||
// shards from two generations, each individually bitrot-valid, that only
|
||||
// surface as silent corruption at read time (backlog#853). A write lock
|
||||
// scoped to the uploadId namespace makes each commit atomic across disks,
|
||||
// so the last committer wins consistently. A guarded completion takes
|
||||
// the object lock before this upload lock to preserve global ordering.
|
||||
let _upload_commit_guard = if opts.no_lock {
|
||||
None
|
||||
// Serialize only same-part commits (rename_part), not the whole upload.
|
||||
// Each concurrent stream writes to its own unique temp dir (see
|
||||
// `tmp_part` above), so the encode/stream phase never conflicts and must
|
||||
// stay lock-free — holding a lock across it would serialize slow
|
||||
// re-transmits of the same part and defeat the S3 "last finisher wins"
|
||||
// semantics. The mixed-generation hazard is confined to rename_part,
|
||||
// where two temp parts are moved cross-disk onto the SAME final
|
||||
// part_path: interleaving there can leave shards from two generations,
|
||||
// each individually bitrot-valid, that only surface as silent corruption
|
||||
// at read time (backlog#853). A write lock scoped to this part number
|
||||
// makes each same-part commit atomic across disks, so the last committer
|
||||
// wins consistently, while different part numbers commit onto disjoint
|
||||
// part paths and stay concurrent (issue#5961 — an uploadId-wide write
|
||||
// lock serialized them into 503 lock-acquire timeouts). The shared
|
||||
// uploadId read lock keeps completion/abort (which take the uploadId
|
||||
// write lock) from racing any in-flight part commit; a guarded
|
||||
// completion takes the object lock before the upload lock to preserve
|
||||
// global ordering.
|
||||
let (_upload_commit_guard, _part_commit_guard) = if opts.no_lock {
|
||||
(None, None)
|
||||
} else {
|
||||
Some(
|
||||
self.acquire_write_lock_diag("put_object_part_commit", RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
|
||||
.await?,
|
||||
)
|
||||
let upload_guard = self
|
||||
.acquire_read_lock_diag("put_object_part_commit", RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
|
||||
.await?;
|
||||
let part_guard = self
|
||||
.acquire_write_lock_diag("put_object_part_commit", RUSTFS_META_MULTIPART_BUCKET, &part_lock_path)
|
||||
.await?;
|
||||
(Some(upload_guard), Some(part_guard))
|
||||
};
|
||||
let (commit_fi, _) = self.check_upload_id_exists(bucket, object, upload_id, false).await?;
|
||||
ensure_multipart_bucket_incarnation(
|
||||
@@ -1102,15 +1132,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.await?;
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost).await;
|
||||
if _upload_commit_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_commit",
|
||||
bucket: RUSTFS_META_MULTIPART_BUCKET.to_string(),
|
||||
object: upload_id_path.clone(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
fence_commit_on_lock_loss(_upload_commit_guard.as_ref(), "put_object_part_commit", &upload_id_path)?;
|
||||
fence_commit_on_lock_loss(_part_commit_guard.as_ref(), "put_object_part_commit", &part_lock_path)?;
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
|
||||
let _ = self
|
||||
@@ -1134,6 +1157,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterRename).await;
|
||||
drop(_part_commit_guard);
|
||||
drop(_upload_commit_guard);
|
||||
|
||||
let ret: PartInfo = PartInfo {
|
||||
@@ -1860,7 +1884,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
|
||||
object_size += ext_part.size;
|
||||
object_actual_size += ext_part.actual_size;
|
||||
if opts.quota_admission.is_some() && ext_part.actual_size < 0 {
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
object_actual_size = object_actual_size
|
||||
.checked_add(ext_part.actual_size)
|
||||
.ok_or(Error::PartMissingOrCorrupt)?;
|
||||
|
||||
fi.parts.push(completed_multipart_object_part(p.part_num, ext_part));
|
||||
}
|
||||
@@ -1889,6 +1918,15 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(admission) = opts.quota_admission {
|
||||
let quota_operation_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
if quota_operation_size > admission.remaining() {
|
||||
return Err(Error::QuotaExceeded {
|
||||
current: admission.current_usage(),
|
||||
limit: admission.quota_limit(),
|
||||
});
|
||||
}
|
||||
}
|
||||
if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) {
|
||||
if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) {
|
||||
fi.checksum = Some(Bytes::from(rc_crc_bytes));
|
||||
@@ -2551,29 +2589,157 @@ mod tests {
|
||||
.new_multipart_upload(bucket, object, create_opts)
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let part = put_test_part(set_disks, bucket, object, &upload.upload_id, 1, content, content.len() as i64).await;
|
||||
(upload.upload_id, vec![part])
|
||||
}
|
||||
|
||||
async fn put_test_part(
|
||||
set_disks: &Arc<SetDisks>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
part_number: usize,
|
||||
content: &[u8],
|
||||
actual_size: i64,
|
||||
) -> CompletePart {
|
||||
let mut reader = PutObjReader::new(
|
||||
HashReader::from_stream(
|
||||
Cursor::new(content.to_vec()),
|
||||
content.len() as i64,
|
||||
content.len() as i64,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.expect("hash reader should be constructed"),
|
||||
HashReader::from_stream(Cursor::new(content.to_vec()), content.len() as i64, actual_size, None, None, false)
|
||||
.expect("hash reader should be constructed"),
|
||||
);
|
||||
let part = set_disks
|
||||
.put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.put_object_part(bucket, object, upload_id, part_number, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("uploading the part should succeed");
|
||||
(
|
||||
upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
)
|
||||
CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_multipart_quota_rejection_preserves_destination_and_upload() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-quota-admission-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
let existing_payload = b"existing object";
|
||||
let mut existing_reader = PutObjReader::from_vec(existing_payload.to_vec());
|
||||
let existing = set_disks
|
||||
.put_object(bucket, object, &mut existing_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("existing object should be stored");
|
||||
|
||||
let payload = vec![0x51; 4096];
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await;
|
||||
let mut denied_opts = ObjectOptions::default();
|
||||
assert!(denied_opts.set_quota_admission(100, 4195));
|
||||
|
||||
let err = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &denied_opts)
|
||||
.await
|
||||
.expect_err("completion larger than the remaining quota must be rejected");
|
||||
assert!(matches!(
|
||||
err,
|
||||
StorageError::QuotaExceeded {
|
||||
current: 100,
|
||||
limit: 4195
|
||||
}
|
||||
));
|
||||
|
||||
let current = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("quota rejection must preserve the existing destination");
|
||||
assert_eq!(current.etag, existing.etag);
|
||||
assert!(
|
||||
set_disks
|
||||
.check_upload_id_exists(bucket, object, &upload_id, false)
|
||||
.await
|
||||
.is_ok(),
|
||||
"quota rejection must leave the multipart upload retryable"
|
||||
);
|
||||
|
||||
let mut allowed_opts = ObjectOptions::default();
|
||||
assert!(allowed_opts.set_quota_admission(100, 4196));
|
||||
let completed = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &allowed_opts)
|
||||
.await
|
||||
.expect("completion at the exact remaining-quota boundary should succeed");
|
||||
assert_eq!(completed.get_actual_size().expect("completed logical size should resolve"), 4096);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_multipart_quota_uses_compressed_logical_size() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-compressed-quota-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
let mut create_opts = ObjectOptions::default();
|
||||
insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string());
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &create_opts)
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x52; 128], 8192).await;
|
||||
let mut complete_opts = ObjectOptions::default();
|
||||
assert!(complete_opts.set_quota_admission(0, 4096));
|
||||
|
||||
let err = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
|
||||
.await
|
||||
.expect_err("logical size above the remaining quota must be rejected");
|
||||
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4096 }));
|
||||
assert!(
|
||||
set_disks
|
||||
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
|
||||
.await
|
||||
.is_ok(),
|
||||
"quota rejection must leave compressed parts retryable"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_multipart_quota_rejects_invalid_logical_sizes() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-invalid-logical-size-bucket";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
let mut create_opts = ObjectOptions::default();
|
||||
insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string());
|
||||
let mut complete_opts = ObjectOptions::default();
|
||||
assert!(complete_opts.set_quota_admission(0, u64::MAX));
|
||||
|
||||
let negative_upload = set_disks
|
||||
.new_multipart_upload(bucket, "negative", &create_opts)
|
||||
.await
|
||||
.expect("negative-size upload should be created");
|
||||
let negative_part = put_test_part(&set_disks, bucket, "negative", &negative_upload.upload_id, 1, &[0x53], -1).await;
|
||||
let negative_err = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, "negative", &negative_upload.upload_id, vec![negative_part], &complete_opts)
|
||||
.await
|
||||
.expect_err("negative logical size must fail closed");
|
||||
assert!(matches!(negative_err, StorageError::PartMissingOrCorrupt));
|
||||
|
||||
let overflow_upload = set_disks
|
||||
.new_multipart_upload(bucket, "overflow", &create_opts)
|
||||
.await
|
||||
.expect("overflow upload should be created");
|
||||
let first = put_test_part(&set_disks, bucket, "overflow", &overflow_upload.upload_id, 1, &[0x54], i64::MAX).await;
|
||||
let second = put_test_part(&set_disks, bucket, "overflow", &overflow_upload.upload_id, 2, &[0x55], 1).await;
|
||||
let overflow_err = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, "overflow", &overflow_upload.upload_id, vec![first, second], &complete_opts)
|
||||
.await
|
||||
.expect_err("overflowing logical size must fail closed");
|
||||
assert!(matches!(overflow_err, StorageError::PartMissingOrCorrupt));
|
||||
}
|
||||
|
||||
async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {
|
||||
@@ -3366,6 +3532,268 @@ mod tests {
|
||||
.expect("abort should delete the upload after UploadPart releases the lock");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn put_object_part_different_part_numbers_commit_concurrently() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
const PART1_SIZE: usize = 5 * 1024 * 1024; // non-final parts must be >= 5MiB to complete
|
||||
const PART2_SIZE: usize = 4096;
|
||||
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let locker: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager));
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, vec![locker]).await;
|
||||
let bucket = "multipart-concurrent-part-numbers-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let upload_id = upload.upload_id;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
// issue#5961: the barrier releases only once BOTH commits are paused
|
||||
// inside their commit sections, so reaching wait_until_paused proves the
|
||||
// two part numbers held their commit locks concurrently. Under an
|
||||
// uploadId-wide exclusive commit lock the second put errors at the 5s
|
||||
// lock-acquire timeout instead of arriving, and wait_until_paused fails
|
||||
// deterministically. No wall-clock bound on the success path.
|
||||
let barrier = MultipartCommitBarrier::install_for_arrivals(bucket, object, MultipartCommitPause::PutPartAfterRename, 2);
|
||||
|
||||
let put1_store = set_disks.clone();
|
||||
let put1_upload_id = upload_id.clone();
|
||||
let put1 = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x51; PART1_SIZE]);
|
||||
put1_store
|
||||
.put_object_part(bucket, object, &put1_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
let put2_store = set_disks.clone();
|
||||
let put2_upload_id = upload_id.clone();
|
||||
let put2 = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x52; PART2_SIZE]);
|
||||
put2_store
|
||||
.put_object_part(bucket, object, &put2_upload_id, 2, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
barrier.release();
|
||||
let part1 = put1
|
||||
.await
|
||||
.expect("part 1 task should not panic")
|
||||
.expect("part 1 should commit after the barrier is released");
|
||||
let part2 = put2
|
||||
.await
|
||||
.expect("part 2 task should not panic")
|
||||
.expect("part 2 should commit after the barrier is released");
|
||||
assert_eq!(part1.part_num, 1);
|
||||
assert_eq!(part2.part_num, 2);
|
||||
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
vec![
|
||||
CompletePart {
|
||||
part_num: part1.part_num,
|
||||
etag: part1.etag.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
CompletePart {
|
||||
part_num: part2.part_num,
|
||||
etag: part2.etag.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("completion should succeed with both concurrently committed parts");
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed object should open");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("completed object should stream fully");
|
||||
assert_eq!(body.len(), PART1_SIZE + PART2_SIZE);
|
||||
assert!(body[..PART1_SIZE].iter().all(|b| *b == 0x51), "part 1 bytes must round-trip");
|
||||
assert!(body[PART1_SIZE..].iter().all(|b| *b == 0x52), "part 2 bytes must round-trip");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn put_object_part_same_part_retries_serialize_on_part_lock() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-same-part-retry-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let part_lock_path = format!("{upload_id_path}/part.1");
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartAfterRename);
|
||||
|
||||
let first_store = set_disks.clone();
|
||||
let first_upload_id = upload_id.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x53; 4096]);
|
||||
first_store
|
||||
.put_object_part(bucket, object, &first_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
// The paused commit must hold its part lock EXCLUSIVELY: even a shared
|
||||
// probe on the part key has to time out. This pins the write-ness of the
|
||||
// part lock — a shared part lock would let two same-part rename_part
|
||||
// calls interleave into mixed-generation shards (backlog#853).
|
||||
let probe = set_disks
|
||||
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &part_lock_path)
|
||||
.await
|
||||
.expect("part namespace lock should be created")
|
||||
.get_read_lock(Duration::from_secs(1))
|
||||
.await;
|
||||
assert!(
|
||||
probe.is_err(),
|
||||
"the in-flight part commit must hold an exclusive write lock on its part key"
|
||||
);
|
||||
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, part_lock_path));
|
||||
let retry_store = set_disks.clone();
|
||||
let retry_upload_id = upload_id.clone();
|
||||
let retry = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x54; 4096]);
|
||||
retry_store
|
||||
.put_object_part(bucket, object, &retry_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(1).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(
|
||||
!retry.is_finished(),
|
||||
"a retry of the same part number must wait for the in-flight commit (backlog#853)"
|
||||
);
|
||||
|
||||
barrier.release();
|
||||
first
|
||||
.await
|
||||
.expect("first attempt task should not panic")
|
||||
.expect("first attempt should commit after the barrier is released");
|
||||
let retry_part = retry
|
||||
.await
|
||||
.expect("retry task should not panic")
|
||||
.expect("the retry should commit after the first attempt releases the part lock");
|
||||
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: retry_part.part_num,
|
||||
etag: retry_part.etag.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("the last committed retry must win the final part generation");
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed object should open");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("completed object should stream fully");
|
||||
assert_eq!(body, vec![0x54; 4096], "the retry's generation must be the one served");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn put_object_part_fences_part_lock_loss_before_rename() {
|
||||
let target = Arc::new(std::sync::RwLock::new(None));
|
||||
let refresh_calls = Arc::new(AtomicUsize::new(0));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = (0..4)
|
||||
.map(|_| {
|
||||
Arc::new(SelectiveLockLossClient::new(Arc::clone(&target), Arc::clone(&refresh_calls))) as Arc<dyn LockClient>
|
||||
})
|
||||
.collect();
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-put-part-part-lock-loss-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let part_lock_path = format!("{upload_id_path}/part.1");
|
||||
*target.write().expect("lock-loss target should be writable") =
|
||||
Some(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, part_lock_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
let put_upload_id = upload_id.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x47; 4096]);
|
||||
put_store
|
||||
.put_object_part(bucket, object, &put_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
tokio::time::advance(Duration::from_secs(11)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(
|
||||
refresh_calls.load(Ordering::Acquire) > 0,
|
||||
"part lock heartbeat should reach the test client"
|
||||
);
|
||||
barrier.release();
|
||||
|
||||
let err = put
|
||||
.await
|
||||
.expect("UploadPart task should not panic")
|
||||
.expect_err("UploadPart must fail after losing the part lock");
|
||||
match err {
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
bucket: lock_bucket,
|
||||
object: lock_object,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(lock_bucket, RUSTFS_META_MULTIPART_BUCKET);
|
||||
assert_eq!(lock_object, part_lock_path);
|
||||
}
|
||||
other => panic!("unexpected lock-loss error: {other:?}"),
|
||||
}
|
||||
let listed = set_disks
|
||||
.list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("part lock loss before rename must leave the upload readable");
|
||||
assert!(listed.parts.is_empty(), "part lock loss before rename must not publish the part");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn put_object_part_fences_upload_lock_loss_before_rename() {
|
||||
|
||||
@@ -56,6 +56,22 @@ use http::HeaderValue;
|
||||
use rustfs_utils::path::decode_dir_object;
|
||||
use std::future::Future;
|
||||
|
||||
#[inline]
|
||||
fn duration_millis_f64(duration: std::time::Duration) -> f64 {
|
||||
duration.as_secs_f64() * 1000.0
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod duration_metrics_tests {
|
||||
use super::duration_millis_f64;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
fn duration_millis_preserves_sub_millisecond_precision() {
|
||||
assert_eq!(duration_millis_f64(Duration::from_micros(125)), 0.125);
|
||||
}
|
||||
}
|
||||
|
||||
fn is_restore_control_metadata(key: &str) -> bool {
|
||||
key.eq_ignore_ascii_case(X_AMZ_RESTORE.as_str())
|
||||
|| key.eq_ignore_ascii_case(rustfs_utils::http::headers::AMZ_RESTORE_EXPIRY_DAYS)
|
||||
@@ -734,8 +750,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
0,
|
||||
object_info.size,
|
||||
&mut output,
|
||||
fi,
|
||||
files,
|
||||
fi.into_owned(),
|
||||
files.into_owned(),
|
||||
&disks,
|
||||
self.set_index,
|
||||
self.pool_index,
|
||||
@@ -851,8 +867,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
offset,
|
||||
length,
|
||||
&mut writer,
|
||||
fi,
|
||||
files,
|
||||
fi.into_owned(),
|
||||
files.into_owned(),
|
||||
&disks,
|
||||
set_index,
|
||||
pool_index,
|
||||
@@ -1107,8 +1123,12 @@ impl SetDisks {
|
||||
writers.push(w);
|
||||
errors.push(e);
|
||||
}
|
||||
let writer_setup_ms = writer_setup_stage_start.elapsed().as_millis() as u64;
|
||||
rustfs_io_metrics::record_put_object_stage_duration("set_disk_writer_setup", writer_setup_ms as f64);
|
||||
let writer_setup_elapsed = writer_setup_stage_start.elapsed();
|
||||
let writer_setup_ms = writer_setup_elapsed.as_millis() as u64;
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
"set_disk_writer_setup",
|
||||
duration_millis_f64(writer_setup_elapsed),
|
||||
);
|
||||
|
||||
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
|
||||
if nil_count < write_quorum {
|
||||
@@ -1138,11 +1158,16 @@ impl SetDisks {
|
||||
|
||||
let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size);
|
||||
rustfs_io_metrics::record_put_object_path(write_path.metric_label());
|
||||
let small_size_hint = if matches!(write_path, SmallWritePath::Inline | SmallWritePath::SingleBlockNonInline) {
|
||||
usize::try_from(put_object_size).map_err(Error::other)?
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let encode_stage_start = Instant::now();
|
||||
let (reader, w_size) = match write_path {
|
||||
SmallWritePath::Inline => match Arc::clone(&erasure)
|
||||
.encode_inline_small(stream, &mut writers, write_quorum)
|
||||
.encode_inline_small_with_size_hint(stream, &mut writers, write_quorum, small_size_hint)
|
||||
.await
|
||||
{
|
||||
Ok((r, w)) => (r, w),
|
||||
@@ -1152,7 +1177,7 @@ impl SetDisks {
|
||||
}
|
||||
},
|
||||
SmallWritePath::SingleBlockNonInline => match Arc::clone(&erasure)
|
||||
.encode_single_block_non_inline(stream, &mut writers, write_quorum)
|
||||
.encode_single_block_non_inline_with_size_hint(stream, &mut writers, write_quorum, small_size_hint)
|
||||
.await
|
||||
{
|
||||
Ok((r, w)) => (r, w),
|
||||
@@ -1178,8 +1203,9 @@ impl SetDisks {
|
||||
}
|
||||
},
|
||||
};
|
||||
let encode_ms = encode_stage_start.elapsed().as_millis() as u64;
|
||||
rustfs_io_metrics::record_put_object_stage_duration("set_disk_encode", encode_ms as f64);
|
||||
let encode_elapsed = encode_stage_start.elapsed();
|
||||
let encode_ms = encode_elapsed.as_millis() as u64;
|
||||
rustfs_io_metrics::record_put_object_stage_duration("set_disk_encode", duration_millis_f64(encode_elapsed));
|
||||
|
||||
let _ = mem::replace(&mut data.stream, reader);
|
||||
// if let Err(err) = close_bitrot_writers(&mut writers).await {
|
||||
@@ -1497,8 +1523,18 @@ impl SetDisks {
|
||||
let _ = rustfs_common::heal_channel::send_heal_request(request).await;
|
||||
});
|
||||
}
|
||||
let rename_stage_ms = rename_stage_start.elapsed().as_millis() as u64;
|
||||
rustfs_io_metrics::record_put_object_stage_duration("set_disk_rename", rename_stage_ms as f64);
|
||||
|
||||
let rename_stage_elapsed = rename_stage_start.elapsed();
|
||||
let rename_stage_ms = rename_stage_elapsed.as_millis() as u64;
|
||||
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
|
||||
// `rename_data` has completed the authoritative quorum commit. The
|
||||
// exact old-data-dir reclamation below is best-effort space cleanup;
|
||||
// it must not serialize the next operation on this object.
|
||||
drop(object_lock_guard);
|
||||
|
||||
rustfs_io_metrics::record_put_object_stage_duration("set_disk_rename", duration_millis_f64(rename_stage_elapsed));
|
||||
if (rename_stage_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS {
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_COMMIT_TAIL_SLOW,
|
||||
@@ -1527,9 +1563,13 @@ impl SetDisks {
|
||||
let cleanup = self
|
||||
.commit_rename_data_dir(&cleanup_disks, bucket, object, &old_dir.to_string(), &committed_dir, write_quorum)
|
||||
.await;
|
||||
let cleanup_ms = cleanup_stage_start.elapsed().as_millis() as u64;
|
||||
let cleanup_elapsed = cleanup_stage_start.elapsed();
|
||||
let cleanup_ms = cleanup_elapsed.as_millis() as u64;
|
||||
cleanup_stage_ms = Some(cleanup_ms);
|
||||
rustfs_io_metrics::record_put_object_stage_duration("set_disk_old_data_cleanup", cleanup_ms as f64);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
"set_disk_old_data_cleanup",
|
||||
duration_millis_f64(cleanup_elapsed),
|
||||
);
|
||||
self.report_old_data_dir_cleanup(bucket, object, &old_dir.to_string(), &cleanup)
|
||||
.await;
|
||||
if (cleanup_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS {
|
||||
@@ -1550,8 +1590,6 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
drop(object_lock_guard); // drop object lock guard to release the lock
|
||||
|
||||
for (i, op_disk) in online_disks.iter().enumerate() {
|
||||
if let Some(disk) = op_disk
|
||||
&& disk.is_online().await
|
||||
@@ -1650,10 +1688,6 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
if result.is_ok() {
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
}
|
||||
|
||||
if issue3031_diag_enabled() {
|
||||
warn!(
|
||||
target: "rustfs_ecstore::set_disk",
|
||||
@@ -3167,7 +3201,8 @@ impl SetDisks {
|
||||
// Force the full quorum fanout (allow_early_stop=false): `disks` is the
|
||||
// write target below, and an early-stop subset would only carry read
|
||||
// quorum, failing write quorum on update_object_meta (backlog#872).
|
||||
let (mut fi, _, disks) = self.get_object_fileinfo_gated(bucket, object, opts, false, false).await?;
|
||||
let (fi, _, disks) = self.get_object_fileinfo_gated(bucket, object, opts, false, false).await?;
|
||||
let mut fi = fi.into_owned();
|
||||
|
||||
fi.metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_owned());
|
||||
if let Some(eval_metadata) = &opts.eval_metadata {
|
||||
@@ -3190,7 +3225,7 @@ impl SetDisks {
|
||||
});
|
||||
}
|
||||
|
||||
self.update_object_meta(bucket, object, fi.clone(), disks.as_slice()).await?;
|
||||
self.update_object_meta(bucket, object, fi.clone(), &disks).await?;
|
||||
|
||||
Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended))
|
||||
}
|
||||
@@ -4594,7 +4629,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
// _lock_guard = guard_opt;
|
||||
// }
|
||||
|
||||
let (mut fi, meta_arr, online_disks) = self.get_object_fileinfo(bucket, object, opts, true, false).await?;
|
||||
let (fi, meta_arr, online_disks) = self.get_object_fileinfo(bucket, object, opts, true, false).await?;
|
||||
let mut fi = fi.into_owned();
|
||||
/*if err != nil {
|
||||
return Err(to_object_err(err, vec![bucket, object]));
|
||||
}*/
|
||||
@@ -4708,7 +4744,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
cloned_fi.size,
|
||||
&mut writer,
|
||||
cloned_fi,
|
||||
meta_arr,
|
||||
meta_arr.into_owned(),
|
||||
&online_disks,
|
||||
set_index,
|
||||
pool_index,
|
||||
@@ -4832,7 +4868,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
};
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
let current = self.get_object_fileinfo(bucket, object, &commit_opts, true, false).await;
|
||||
let (mut current_fi, _, _) = match current {
|
||||
let (current_fi, _, _) = match current {
|
||||
Ok(current) => current,
|
||||
Err(err) => {
|
||||
drop(transition_lock_guard);
|
||||
@@ -4843,6 +4879,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let mut current_fi = current_fi.into_owned();
|
||||
let source_matches = current_fi.version_id == fi.version_id
|
||||
&& current_fi.data_dir == fi.data_dir
|
||||
&& current_fi.mod_time == fi.mod_time
|
||||
@@ -5553,7 +5590,7 @@ mod get_object_downstream_close_accounting_tests {
|
||||
let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
|
||||
let (decode_failures, emit_failures) = metrics::with_local_recorder(&recorder, || {
|
||||
let (decode_failures, emit_failures, legacy_fanout, internal_fanout) = metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "get-downstream-close-accounting";
|
||||
@@ -5621,6 +5658,14 @@ mod get_object_downstream_close_accounting_tests {
|
||||
("reason", GetObjectFailureReason::DownstreamClosed.as_str()),
|
||||
],
|
||||
),
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_total_responses",
|
||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)],
|
||||
),
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_total_responses",
|
||||
&[("path", GET_OBJECT_PATH_INTERNAL_META)],
|
||||
),
|
||||
)
|
||||
})
|
||||
});
|
||||
@@ -5628,6 +5673,11 @@ mod get_object_downstream_close_accounting_tests {
|
||||
|
||||
assert!(decode_failures > 0, "the producer must expose the downstream close at decode");
|
||||
assert_eq!(emit_failures, 0, "downstream closure must not be counted as an emit failure");
|
||||
assert_eq!(legacy_fanout, vec![4.0], "ordinary object fanout must retain the legacy_duplex path");
|
||||
assert!(
|
||||
internal_fanout.is_empty(),
|
||||
"ordinary object fanout must not be attributed to internal_meta"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -5641,7 +5691,7 @@ mod get_object_downstream_close_accounting_tests {
|
||||
let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
|
||||
let (internal_missing, legacy_unknown) = metrics::with_local_recorder(&recorder, || {
|
||||
let (internal_missing, legacy_unknown, internal_fanout, legacy_fanout) = metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let options = ObjectOptions {
|
||||
@@ -5677,6 +5727,14 @@ mod get_object_downstream_close_accounting_tests {
|
||||
("reason", GetObjectFailureReason::Unknown.as_str()),
|
||||
],
|
||||
),
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_error_responses",
|
||||
&[("path", GET_OBJECT_PATH_INTERNAL_META)],
|
||||
),
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_error_responses",
|
||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)],
|
||||
),
|
||||
)
|
||||
})
|
||||
});
|
||||
@@ -5687,6 +5745,8 @@ mod get_object_downstream_close_accounting_tests {
|
||||
legacy_unknown, 0,
|
||||
"internal metadata miss must not be attributed to legacy_duplex/unknown"
|
||||
);
|
||||
assert_eq!(internal_fanout, vec![4.0], "internal metadata fanout must retain its path label");
|
||||
assert!(legacy_fanout.is_empty(), "internal metadata fanout must not leak into legacy_duplex");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6178,9 +6238,9 @@ mod transition_commit_failure_tests {
|
||||
cache_key.clone(),
|
||||
Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi: fi.clone(),
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
fi: Arc::new((*fi).clone()),
|
||||
parts_metadata: Arc::new(parts_metadata.into_owned()),
|
||||
online_disks: Arc::new(online_disks.into_owned()),
|
||||
read_quorum: 2,
|
||||
}),
|
||||
)
|
||||
@@ -8611,8 +8671,10 @@ mod put_object_tmp_cleanup_tests {
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
||||
use super::*;
|
||||
use crate::disk::DiskAPI as _;
|
||||
use crate::set_disk::core::io_primitives::rename_fanout_barrier;
|
||||
use std::time::Duration;
|
||||
use tempfile::TempDir;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
/// Large enough that the erasure shards are written as real tmp files
|
||||
/// (never inlined into xl.meta), so both tests exercise actual cleanup.
|
||||
@@ -8697,6 +8759,168 @@ mod put_object_tmp_cleanup_tests {
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_put_releases_namespace_lock_before_old_data_cleanup() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-commit-lock-window";
|
||||
let object = "commit-lock-window-object";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let mut initial_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]);
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut initial_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initial object should be committed");
|
||||
let mut initial = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initial object should populate the metadata cache");
|
||||
let mut initial_body = Vec::new();
|
||||
initial
|
||||
.stream
|
||||
.read_to_end(&mut initial_body)
|
||||
.await
|
||||
.expect("initial body should drain");
|
||||
assert_eq!(initial_body, vec![b'0'; TEST_OBJECT_SIZE]);
|
||||
|
||||
let cleanup_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_CLEANUP);
|
||||
let first_store = Arc::clone(&set_disks);
|
||||
let first = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
first_store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), cleanup_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("first overwrite should reach old-data cleanup");
|
||||
|
||||
let mut committed = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("GET should not wait for old-data cleanup")
|
||||
.expect("committed overwrite should be readable during old-data cleanup");
|
||||
let mut committed_body = Vec::new();
|
||||
committed
|
||||
.stream
|
||||
.read_to_end(&mut committed_body)
|
||||
.await
|
||||
.expect("committed overwrite body should drain");
|
||||
assert_eq!(committed_body, vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
|
||||
let second_commit_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
|
||||
let second_store = Arc::clone(&set_disks);
|
||||
let second = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
second_store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), second_commit_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("second overwrite should acquire the namespace lock during cleanup");
|
||||
|
||||
cleanup_barrier.release();
|
||||
first
|
||||
.await
|
||||
.expect("first overwrite task should join")
|
||||
.expect("first overwrite should remain successful after cleanup");
|
||||
drop(cleanup_barrier);
|
||||
second_commit_barrier.release();
|
||||
second
|
||||
.await
|
||||
.expect("second overwrite task should join")
|
||||
.expect("second overwrite should commit after acquiring the released namespace lock");
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the latest overwrite should be readable");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("latest body should drain");
|
||||
assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_post_commit_cleanup_does_not_retain_namespace_lock() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-commit-lock-cancelled-cleanup";
|
||||
let object = "commit-lock-cancelled-cleanup-object";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let mut initial_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]);
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut initial_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initial object should be committed");
|
||||
|
||||
let cleanup_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let cleanup_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_CLEANUP);
|
||||
let first_store = Arc::clone(&set_disks);
|
||||
let first = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
first_store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), cleanup_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("first overwrite should reach old-data cleanup");
|
||||
|
||||
let second_commit_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
|
||||
let second_store = Arc::clone(&set_disks);
|
||||
let second = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
second_store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), second_commit_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("second overwrite should acquire the namespace lock before cancellation");
|
||||
|
||||
first.abort();
|
||||
assert!(
|
||||
first
|
||||
.await
|
||||
.expect_err("the first request should be cancelled during cleanup")
|
||||
.is_cancelled()
|
||||
);
|
||||
assert!(
|
||||
cleanup_tasks.running() >= 1,
|
||||
"cancelled cleanup must remain observable until its disk task drains"
|
||||
);
|
||||
cleanup_barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while cleanup_tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cancelled cleanup disk tasks should drain");
|
||||
drop(cleanup_barrier);
|
||||
|
||||
second_commit_barrier.release();
|
||||
second
|
||||
.await
|
||||
.expect("second overwrite task should join")
|
||||
.expect("second overwrite should survive the earlier request cancellation");
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the latest overwrite should be readable");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("latest body should drain");
|
||||
assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_object_no_lock_aborts_after_outer_namespace_lock_loss() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
|
||||
@@ -30,12 +30,12 @@ use crate::diagnostics::get::{
|
||||
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
|
||||
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
|
||||
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP,
|
||||
GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, GET_STAGE_READER_SETUP_DROP_PENDING,
|
||||
GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT,
|
||||
GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, GetObjectFailureReason, classify_disk_error,
|
||||
get_stage_timer_if_enabled, mark_get_object_downstream_closed, record_get_object_pipeline_failure,
|
||||
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
||||
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE,
|
||||
GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
|
||||
GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
|
||||
GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
|
||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, mark_get_object_downstream_closed,
|
||||
record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use crate::io_support::bitrot::{
|
||||
@@ -116,9 +116,9 @@ impl SetDisks {
|
||||
.then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE)
|
||||
}
|
||||
|
||||
async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option<GetObjectMetadataCacheEntry> {
|
||||
async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option<Arc<GetObjectMetadataCacheEntry>> {
|
||||
match self.lookup_cached_get_object_fileinfo(bucket, object).await {
|
||||
MetadataCacheLookup::Hit(entry) => Some((*entry).clone()),
|
||||
MetadataCacheLookup::Hit(entry) => Some(entry),
|
||||
MetadataCacheLookup::Miss | MetadataCacheLookup::RejectedInsufficientQuorum => None,
|
||||
}
|
||||
}
|
||||
@@ -180,9 +180,9 @@ impl SetDisks {
|
||||
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
|
||||
let entry = Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi: fi.clone(),
|
||||
parts_metadata: parts_metadata.to_vec(),
|
||||
online_disks: online_disks.to_vec(),
|
||||
fi: Arc::new(fi.clone()),
|
||||
parts_metadata: Arc::new(parts_metadata.to_vec()),
|
||||
online_disks: Arc::new(online_disks.to_vec()),
|
||||
read_quorum,
|
||||
});
|
||||
self.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {})
|
||||
@@ -221,10 +221,10 @@ impl SetDisks {
|
||||
let disks = self.disks.read().await.clone();
|
||||
let required_reads = self.default_read_quorum();
|
||||
|
||||
let bucket = bucket.to_string();
|
||||
let object = object.to_string();
|
||||
let version_id = version_id.to_string();
|
||||
let opts = opts.clone();
|
||||
let bucket: Arc<str> = Arc::from(bucket);
|
||||
let object: Arc<str> = Arc::from(object);
|
||||
let version_id: Arc<str> = Arc::from(version_id);
|
||||
let opts = *opts;
|
||||
|
||||
let processor = runtime_sources::batch_processors().read_processor();
|
||||
let tasks: Vec<_> = disks
|
||||
@@ -235,9 +235,9 @@ impl SetDisks {
|
||||
let bucket = bucket.clone();
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
let opts = opts.clone();
|
||||
let task_opts = opts;
|
||||
|
||||
async move { disk.read_version(&bucket, &bucket, &object, &version_id, &opts).await }
|
||||
async move { disk.read_version(&bucket, &bucket, &object, &version_id, &task_opts).await }
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
@@ -257,7 +257,7 @@ impl SetDisks {
|
||||
opts: &ObjectOptions,
|
||||
read_data: bool,
|
||||
caller_allows_early_stop: bool,
|
||||
) -> Result<(FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>)> {
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
self.get_object_fileinfo_gated(bucket, object, opts, read_data, caller_allows_early_stop)
|
||||
.await
|
||||
}
|
||||
@@ -274,7 +274,7 @@ impl SetDisks {
|
||||
opts: &ObjectOptions,
|
||||
read_data: bool,
|
||||
allow_early_stop: bool,
|
||||
) -> Result<(FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>)> {
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
let vid = opts.version_id.clone().unwrap_or_default();
|
||||
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
|
||||
@@ -300,7 +300,11 @@ impl SetDisks {
|
||||
GET_STAGE_METADATA_CACHE_LOOKUP,
|
||||
metadata_cache_lookup_start,
|
||||
);
|
||||
return Ok((cached.fi.clone(), cached.parts_metadata.clone(), cached.online_disks.clone()));
|
||||
return Ok((
|
||||
GetObjectMetadata::Shared(Arc::clone(&cached.fi)),
|
||||
GetObjectMetadata::Shared(Arc::clone(&cached.parts_metadata)),
|
||||
GetObjectMetadata::Shared(Arc::clone(&cached.online_disks)),
|
||||
));
|
||||
}
|
||||
MetadataCacheLookup::Miss => {
|
||||
rustfs_io_metrics::record_get_object_metadata_cache_decision(
|
||||
@@ -349,7 +353,12 @@ impl SetDisks {
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await?;
|
||||
metadata_fanout_diagnostics.record(GET_OBJECT_PATH_LEGACY_DUPLEX);
|
||||
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
GET_OBJECT_PATH_INTERNAL_META
|
||||
} else {
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX
|
||||
};
|
||||
metadata_fanout_diagnostics.record(metadata_metrics_path);
|
||||
let metadata_fanout_complete = metadata_fanout_diagnostics.total_responses() >= disks.len();
|
||||
// warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata);
|
||||
// warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs);
|
||||
@@ -387,7 +396,7 @@ impl SetDisks {
|
||||
|
||||
let (op_online_disks, fi, fileinfo_selection_quorum) =
|
||||
Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum)?;
|
||||
metadata_fanout_diagnostics.record_quorum_candidate_latency(GET_OBJECT_PATH_LEGACY_DUPLEX, fileinfo_selection_quorum);
|
||||
metadata_fanout_diagnostics.record_quorum_candidate_latency(metadata_metrics_path, fileinfo_selection_quorum);
|
||||
if errs.iter().any(|err| err.is_some()) {
|
||||
let version_id = resolved_read_repair_version_id(&fi, opts.version_id.as_deref());
|
||||
submit_read_repair_heal(
|
||||
@@ -418,7 +427,11 @@ impl SetDisks {
|
||||
|
||||
// let online_disks: Vec<Option<DiskStore>> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect();
|
||||
|
||||
Ok((fi, parts_metadata, op_online_disks))
|
||||
Ok((
|
||||
GetObjectMetadata::Owned(fi),
|
||||
GetObjectMetadata::Owned(parts_metadata),
|
||||
GetObjectMetadata::Owned(op_online_disks),
|
||||
))
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
@@ -2674,6 +2687,39 @@ mod metadata_cache_tests {
|
||||
assert_eq!(cached.read_quorum, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_fileinfo_cache_hit_shares_cached_metadata() {
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
let fi = valid_test_fileinfo("object");
|
||||
let parts_metadata = vec![fi.clone()];
|
||||
let online_disks = Vec::new();
|
||||
let generation = set.get_object_metadata_cache_generation("bucket", "object");
|
||||
set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, &parts_metadata, &online_disks, 0)
|
||||
.await;
|
||||
let cached = set
|
||||
.cached_get_object_fileinfo("bucket", "object")
|
||||
.await
|
||||
.expect("fresh cache entry should be returned");
|
||||
|
||||
let (returned_fi, returned_parts_metadata, returned_online_disks) = set
|
||||
.get_object_fileinfo("bucket", "object", &ObjectOptions::default(), true, false)
|
||||
.await
|
||||
.expect("cache-backed metadata lookup should succeed");
|
||||
|
||||
assert!(
|
||||
matches!(returned_fi, GetObjectMetadata::Shared(ref value) if Arc::ptr_eq(value, &cached.fi)),
|
||||
"cache hits must share FileInfo ownership"
|
||||
);
|
||||
assert!(
|
||||
matches!(returned_parts_metadata, GetObjectMetadata::Shared(ref value) if Arc::ptr_eq(value, &cached.parts_metadata)),
|
||||
"cache hits must share the metadata vector"
|
||||
);
|
||||
assert!(
|
||||
matches!(returned_online_disks, GetObjectMetadata::Shared(ref value) if Arc::ptr_eq(value, &cached.online_disks)),
|
||||
"cache hits must share the online-disk vector"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_metadata_cache_rejects_deleted_and_invalid_fileinfo() {
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
@@ -2713,9 +2759,9 @@ mod metadata_cache_tests {
|
||||
),
|
||||
Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi: fi.clone(),
|
||||
parts_metadata: vec![fi],
|
||||
online_disks: vec![None],
|
||||
fi: Arc::new(fi.clone()),
|
||||
parts_metadata: Arc::new(vec![fi]),
|
||||
online_disks: Arc::new(vec![None]),
|
||||
read_quorum: 1,
|
||||
}),
|
||||
)
|
||||
@@ -2729,9 +2775,6 @@ mod metadata_cache_tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_metadata_cache_rejects_stale_entries() {
|
||||
// moka handles TTL expiry automatically via time_to_live(250ms).
|
||||
// This test verifies that entries inserted with the cache API are retrievable
|
||||
// while fresh, and that the cache API works correctly.
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
let fi = valid_test_fileinfo("object");
|
||||
|
||||
@@ -2743,6 +2786,14 @@ mod metadata_cache_tests {
|
||||
set.cached_get_object_fileinfo("bucket", "object").await.is_some(),
|
||||
"freshly inserted entry should be returned"
|
||||
);
|
||||
|
||||
tokio::time::timeout(GET_OBJECT_METADATA_CACHE_TTL + Duration::from_secs(1), async {
|
||||
while set.cached_get_object_fileinfo("bucket", "object").await.is_some() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("metadata cache entry should expire after its TTL");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2804,9 +2855,13 @@ mod metadata_cache_tests {
|
||||
barrier.wait_until_paused().await;
|
||||
set.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
barrier.release();
|
||||
read.await
|
||||
let (fi, parts_metadata, online_disks) = read
|
||||
.await
|
||||
.expect("metadata read task should not panic")
|
||||
.expect("metadata fanout should still return its selected FileInfo");
|
||||
assert!(matches!(fi, GetObjectMetadata::Owned(_)));
|
||||
assert!(matches!(parts_metadata, GetObjectMetadata::Owned(_)));
|
||||
assert!(matches!(online_disks, GetObjectMetadata::Owned(_)));
|
||||
|
||||
assert!(
|
||||
set.get_object_metadata_cache
|
||||
@@ -2853,9 +2908,9 @@ mod metadata_cache_tests {
|
||||
let key = GetObjectMetadataCacheKey::new("bucket", "object", generation);
|
||||
let entry = Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi: fi.clone(),
|
||||
parts_metadata: vec![fi],
|
||||
online_disks: Vec::new(),
|
||||
fi: Arc::new(fi.clone()),
|
||||
parts_metadata: Arc::new(vec![fi]),
|
||||
online_disks: Arc::new(Vec::new()),
|
||||
read_quorum: 0,
|
||||
});
|
||||
|
||||
@@ -2963,9 +3018,9 @@ mod metadata_cache_tests {
|
||||
let entry = |fi: FileInfo| {
|
||||
Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
parts_metadata: vec![fi.clone()],
|
||||
fi,
|
||||
online_disks: Vec::new(),
|
||||
parts_metadata: Arc::new(vec![fi.clone()]),
|
||||
fi: Arc::new(fi),
|
||||
online_disks: Arc::new(Vec::new()),
|
||||
read_quorum: 0,
|
||||
})
|
||||
};
|
||||
@@ -3415,7 +3470,7 @@ mod tests {
|
||||
);
|
||||
assert_eq!(diagnostics.total_responses(), 9);
|
||||
assert_eq!(diagnostics.valid_responses(), 1);
|
||||
assert_eq!(diagnostics.error_responses(), 8);
|
||||
assert_eq!(diagnostics.non_valid_responses(), 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3430,7 +3485,7 @@ mod tests {
|
||||
);
|
||||
|
||||
assert_eq!(diagnostics.ignored_responses(), 2);
|
||||
assert_eq!(diagnostics.error_responses(), 3);
|
||||
assert_eq!(diagnostics.non_valid_responses(), 3);
|
||||
assert_eq!(diagnostics.observations[0].outcome, GET_METADATA_RESPONSE_DISK_NOT_FOUND);
|
||||
assert_eq!(diagnostics.observations[1].outcome, GET_METADATA_RESPONSE_IGNORED);
|
||||
assert_eq!(diagnostics.observations[2].outcome, GET_METADATA_RESPONSE_NOT_FOUND);
|
||||
@@ -3498,7 +3553,7 @@ mod tests {
|
||||
|
||||
assert_eq!(diagnostics.total_responses(), 3);
|
||||
assert_eq!(diagnostics.valid_responses(), 3);
|
||||
assert_eq!(diagnostics.error_responses(), 0);
|
||||
assert_eq!(diagnostics.non_valid_responses(), 0);
|
||||
assert!(
|
||||
diagnostics
|
||||
.observations
|
||||
@@ -3834,25 +3889,24 @@ mod tests {
|
||||
assert!(metadata_early_stop_permitted(true, true, false, "", false, false));
|
||||
// observe=false (non-observed fanout) also disables early-stop.
|
||||
assert!(!metadata_early_stop_permitted(true, false, false, "", false, false));
|
||||
// Data reads require their own explicit rollout gate.
|
||||
assert!(!metadata_early_stop_permitted(true, true, true, "", false, false));
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_early_stop_requires_explicit_data_read_opt_in() {
|
||||
fn metadata_early_stop_keeps_data_reads_opt_in_by_default() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE, None),
|
||||
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, None),
|
||||
],
|
||||
|| {
|
||||
assert!(!should_allow_metadata_early_stop(true, "", false, false));
|
||||
assert!(!should_allow_metadata_early_stop(true, "version-id", false, false));
|
||||
assert!(should_allow_metadata_early_stop(false, "", false, false));
|
||||
assert!(should_allow_metadata_early_stop(false, "version-id", false, false));
|
||||
assert!(!should_allow_metadata_early_stop(false, "version-id", false, false));
|
||||
},
|
||||
);
|
||||
temp_env::with_vars(
|
||||
@@ -3866,6 +3920,19 @@ mod tests {
|
||||
assert!(should_allow_metadata_early_stop(true, "version-id", false, false));
|
||||
},
|
||||
);
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("false")),
|
||||
],
|
||||
|| {
|
||||
assert!(!should_allow_metadata_early_stop(true, "", false, false));
|
||||
assert!(!should_allow_metadata_early_stop(true, "version-id", false, false));
|
||||
assert!(should_allow_metadata_early_stop(false, "", false, false));
|
||||
assert!(should_allow_metadata_early_stop(false, "version-id", false, false));
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -5462,33 +5529,36 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rustfs_codec_streaming_uses_conservative_default_min_size() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS)),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE, None::<&str>),
|
||||
],
|
||||
|| {
|
||||
let below_threshold_fi = codec_streaming_test_fileinfo(512 * 1024, 1);
|
||||
let below_threshold_object_info = codec_streaming_test_object_info(&below_threshold_fi);
|
||||
assert_eq!(
|
||||
codec_streaming_reader_gate_for_test(&None, &below_threshold_object_info, &below_threshold_fi, true).decision,
|
||||
GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::BelowMinSize)
|
||||
);
|
||||
fn codec_streaming_default_min_size_meets_direct_memory_ceiling() {
|
||||
for engine in [None, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS)] {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, engine),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE, None::<&str>),
|
||||
],
|
||||
|| {
|
||||
let below_threshold_fi = codec_streaming_test_fileinfo(128 * 1024 - 1, 1);
|
||||
let below_threshold_object_info = codec_streaming_test_object_info(&below_threshold_fi);
|
||||
assert_eq!(
|
||||
codec_streaming_reader_gate_for_test(&None, &below_threshold_object_info, &below_threshold_fi, true)
|
||||
.decision,
|
||||
GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::BelowMinSize)
|
||||
);
|
||||
|
||||
let threshold_fi = codec_streaming_test_fileinfo(1_048_576, 1);
|
||||
let threshold_object_info = codec_streaming_test_object_info(&threshold_fi);
|
||||
assert_eq!(
|
||||
codec_streaming_reader_gate_for_test(&None, &threshold_object_info, &threshold_fi, true).decision,
|
||||
GetCodecStreamingDecision::Use
|
||||
);
|
||||
},
|
||||
);
|
||||
let threshold_fi = codec_streaming_test_fileinfo(128 * 1024, 1);
|
||||
let threshold_object_info = codec_streaming_test_object_info(&threshold_fi);
|
||||
assert_eq!(
|
||||
codec_streaming_reader_gate_for_test(&None, &threshold_object_info, &threshold_fi, true).decision,
|
||||
GetCodecStreamingDecision::Use
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -5786,10 +5856,10 @@ mod tests {
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
|
||||
],
|
||||
|| {
|
||||
let fi = codec_streaming_test_fileinfo(1024, 1);
|
||||
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
|
||||
let object_info = codec_streaming_test_object_info(&fi);
|
||||
|
||||
assert_eq!(
|
||||
@@ -5810,10 +5880,10 @@ mod tests {
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("on")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
|
||||
],
|
||||
|| {
|
||||
let fi = codec_streaming_test_fileinfo(1024, 1);
|
||||
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
|
||||
let object_info = codec_streaming_test_object_info(&fi);
|
||||
|
||||
assert_eq!(
|
||||
@@ -5831,10 +5901,10 @@ mod tests {
|
||||
[
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("false")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("on")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
|
||||
],
|
||||
|| {
|
||||
let fi = codec_streaming_test_fileinfo(1024, 1);
|
||||
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
|
||||
let object_info = codec_streaming_test_object_info(&fi);
|
||||
|
||||
assert_eq!(
|
||||
@@ -5914,10 +5984,10 @@ mod tests {
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, Some("0")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
|
||||
],
|
||||
|| {
|
||||
let fi = codec_streaming_test_fileinfo(1024, 1);
|
||||
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
|
||||
let object_info = codec_streaming_test_object_info(&fi);
|
||||
|
||||
assert_eq!(
|
||||
@@ -5934,10 +6004,10 @@ mod tests {
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, Some("100")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
|
||||
],
|
||||
|| {
|
||||
let fi = codec_streaming_test_fileinfo(1024, 1);
|
||||
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
|
||||
let object_info = codec_streaming_test_object_info(&fi);
|
||||
|
||||
assert_eq!(
|
||||
|
||||
@@ -77,9 +77,10 @@ impl SetDisks {
|
||||
version_suspended: opts.version_suspended,
|
||||
..Default::default()
|
||||
};
|
||||
let (mut fi, _, disks) = self
|
||||
let (fi, _, disks) = self
|
||||
.get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
|
||||
.await?;
|
||||
let mut fi = fi.into_owned();
|
||||
if let Some(expected_operation_id) = expected_operation_id {
|
||||
require_restore_operation_id(&fi.metadata, expected_operation_id)?;
|
||||
}
|
||||
@@ -101,7 +102,7 @@ impl SetDisks {
|
||||
bucket,
|
||||
object,
|
||||
fi.clone(),
|
||||
disks.as_slice(),
|
||||
&disks,
|
||||
&UpdateMetadataOpts {
|
||||
replace_user_metadata: true,
|
||||
..Default::default()
|
||||
@@ -143,9 +144,10 @@ impl SetDisks {
|
||||
version_suspended: opts.version_suspended,
|
||||
..Default::default()
|
||||
};
|
||||
let (mut fi, _, disks) = self
|
||||
let (fi, _, disks) = self
|
||||
.get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
|
||||
.await?;
|
||||
let mut fi = fi.into_owned();
|
||||
if let Some(expected_operation_id) = expected_operation_id {
|
||||
match restore_operation_id_from_metadata(&fi.metadata)? {
|
||||
Some(actual_operation_id) if actual_operation_id == expected_operation_id => {}
|
||||
@@ -170,7 +172,7 @@ impl SetDisks {
|
||||
bucket,
|
||||
object,
|
||||
fi,
|
||||
disks.as_slice(),
|
||||
&disks,
|
||||
&UpdateMetadataOpts {
|
||||
replace_user_metadata: true,
|
||||
..Default::default()
|
||||
|
||||
@@ -117,16 +117,17 @@ impl StripeReadState {
|
||||
Self::from_parts_with_read_costs(shards, errors, &[], read_quorum)
|
||||
}
|
||||
|
||||
pub(crate) fn from_parts_with_read_costs(
|
||||
shards: Vec<Option<Vec<u8>>>,
|
||||
errors: Vec<Option<Error>>,
|
||||
read_costs: &[ShardReadCost],
|
||||
read_quorum: usize,
|
||||
) -> Self {
|
||||
let slot_count = shards.len().max(errors.len());
|
||||
let mut slots = Vec::with_capacity(slot_count);
|
||||
pub(crate) fn from_parts_with_read_costs<S, E>(shards: S, errors: E, read_costs: &[ShardReadCost], read_quorum: usize) -> Self
|
||||
where
|
||||
S: IntoIterator<Item = Option<Vec<u8>>>,
|
||||
S::IntoIter: ExactSizeIterator,
|
||||
E: IntoIterator<Item = Option<Error>>,
|
||||
E::IntoIter: ExactSizeIterator,
|
||||
{
|
||||
let mut shards = shards.into_iter();
|
||||
let mut errors = errors.into_iter();
|
||||
let slot_count = shards.len().max(errors.len());
|
||||
let mut slots = Vec::with_capacity(slot_count);
|
||||
for index in 0..slot_count {
|
||||
let read_cost = read_costs.get(index).copied().unwrap_or(ShardReadCost::Unknown);
|
||||
slots.push(ShardSlot::with_read_cost(
|
||||
|
||||
@@ -23,6 +23,7 @@ use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use std::collections::BTreeMap;
|
||||
use std::future::Future;
|
||||
|
||||
@@ -153,6 +154,31 @@ where
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
pub async fn get_bucket_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
sys.read().await.get(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
sys.read().await.get_bucket_policy(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy_raw(&self, bucket: &str) -> Result<(String, OffsetDateTime)> {
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
sys.read().await.get_bucket_policy_raw(bucket).await
|
||||
}
|
||||
|
||||
pub async fn restricts_public_bucket_access(&self, bucket: &str) -> Result<bool> {
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
let (config, _) = sys.read().await.get_public_access_block_config(bucket).await?;
|
||||
Ok(config.restrict_public_buckets.unwrap_or(false))
|
||||
}
|
||||
|
||||
pub async fn update_bucket_metadata_config(&self, bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
|
||||
metadata_sys::update_in(&self.ctx, bucket, config_file, data).await
|
||||
}
|
||||
|
||||
pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
|
||||
metadata_sys::get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
|
||||
}
|
||||
@@ -431,7 +457,13 @@ impl ECStore {
|
||||
None
|
||||
};
|
||||
|
||||
let mut meta = existing_metadata.unwrap_or_else(|| BucketMetadata::new(bucket));
|
||||
let mut meta = existing_metadata.unwrap_or_else(|| {
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
BucketMetadata::new_with_default_durability(bucket)
|
||||
} else {
|
||||
BucketMetadata::new(bucket)
|
||||
}
|
||||
});
|
||||
let existing_incarnation_is_authoritative = meta.bucket_incarnation_sidecar;
|
||||
if confirmed_missing || is_meta_bucketname(bucket) {
|
||||
meta.set_created(opts.created_at);
|
||||
@@ -1077,6 +1109,26 @@ mod tests {
|
||||
(temp_dir, ecstore)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_metadata_methods_fail_closed_before_instance_initialization() {
|
||||
let (_temp_dir, store) = setup_multi_pool_scanner_listing_test_env().await;
|
||||
|
||||
let expected = "bucket metadata sys not initialized for this instance";
|
||||
let errors = [
|
||||
store.get_bucket_metadata("bucket").await.unwrap_err(),
|
||||
store.get_bucket_policy("bucket").await.unwrap_err(),
|
||||
store.get_bucket_policy_raw("bucket").await.unwrap_err(),
|
||||
store.restricts_public_bucket_access("bucket").await.unwrap_err(),
|
||||
store
|
||||
.update_bucket_metadata_config("bucket", crate::bucket::metadata::BUCKET_POLICY_CONFIG, Vec::new())
|
||||
.await
|
||||
.unwrap_err(),
|
||||
];
|
||||
for error in errors {
|
||||
assert_eq!(error.to_string(), format!("Io error: {expected}"));
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_bucket_with_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
|
||||
let generation_before_make = ecstore.scanner_namespace_mutation_generation();
|
||||
ecstore
|
||||
@@ -1511,6 +1563,76 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn make_bucket_seeds_new_bucket_durability_override() {
|
||||
temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, None::<&str>)], async {
|
||||
let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
|
||||
let bucket = format!("bucket-default-durability-{}", Uuid::new_v4().simple());
|
||||
|
||||
ecstore
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("new bucket should be created");
|
||||
|
||||
let metadata = metadata_sys::get_in(&ecstore.ctx, &bucket)
|
||||
.await
|
||||
.expect("metadata should load for the new bucket");
|
||||
assert_eq!(
|
||||
metadata.durability_config().and_then(|cfg| cfg.normalized_mode()).as_deref(),
|
||||
Some(crate::bucket::durability::BUCKET_DURABILITY_MODE_RELAXED)
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn force_create_existing_bucket_keeps_durability_override() {
|
||||
let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
|
||||
let bucket = format!("bucket-force-durability-{}", Uuid::new_v4().simple());
|
||||
|
||||
temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, Some("inherit"))], async {
|
||||
ecstore
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("plain bucket should be created without a durability override");
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
metadata_sys::get_in(&ecstore.ctx, &bucket)
|
||||
.await
|
||||
.expect("metadata should load after initial create")
|
||||
.durability_config()
|
||||
.is_none(),
|
||||
"test setup: the existing bucket must start without an override"
|
||||
);
|
||||
|
||||
temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, None::<&str>)], async {
|
||||
ecstore
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
force_create: true,
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("force create should update existing bucket metadata");
|
||||
})
|
||||
.await;
|
||||
|
||||
let metadata = metadata_sys::get_in(&ecstore.ctx, &bucket)
|
||||
.await
|
||||
.expect("metadata should load after force create");
|
||||
assert!(metadata.lock_enabled, "force create sanity check: Object Lock should be enabled");
|
||||
assert!(
|
||||
metadata.durability_config().is_none(),
|
||||
"force create must not apply the new-bucket default to existing bucket metadata"
|
||||
);
|
||||
}
|
||||
|
||||
/// `DeleteBucket`'s emptiness check is a raw disk scan (`has_xlmeta_files`),
|
||||
/// not an S3-level listing, so "the client drained the bucket" and "the
|
||||
/// bucket is deletable" are two different contracts. Nothing pinned the
|
||||
|
||||
@@ -9529,294 +9529,6 @@ mod test {
|
||||
.expect("a partial outage with a healthy set must not fail the walk");
|
||||
}
|
||||
|
||||
// use std::sync::Arc;
|
||||
|
||||
// use crate::cache_value::metacache_set::list_path_raw;
|
||||
// use crate::cache_value::metacache_set::ListPathRawOptions;
|
||||
// use crate::disk::endpoint::Endpoint;
|
||||
// use crate::disk::error::is_err_eof;
|
||||
// use crate::disk::format::FormatV3;
|
||||
// use crate::disk::new_disk;
|
||||
// use crate::disk::DiskAPI;
|
||||
// use crate::disk::DiskOption;
|
||||
// use crate::disk::MetaCacheEntries;
|
||||
// use crate::disk::MetaCacheEntry;
|
||||
// use crate::disk::WalkDirOptions;
|
||||
// use crate::layout::endpoints::EndpointServerPools;
|
||||
// use crate::error::Error;
|
||||
// use crate::metacache::writer::MetacacheReader;
|
||||
// use crate::set_disk::SetDisks;
|
||||
// use crate::store::list_objects::ListPathOptions;
|
||||
// use crate::store::list_objects::WalkOptions;
|
||||
// use crate::store::list_objects::WalkVersionsSortOrder;
|
||||
// use futures::future::join_all;
|
||||
// use rustfs_lock::namespace_lock::NsLockMap;
|
||||
// use tokio::sync::broadcast;
|
||||
// use tokio::sync::mpsc;
|
||||
// use tokio::sync::RwLock;
|
||||
// use uuid::Uuid;
|
||||
|
||||
// #[tokio::test]
|
||||
// async fn test_walk_dir() {
|
||||
// let mut ep = Endpoint::try_from("/Users/weisd/project/weisd/s3-rustfs/target/volume/test").unwrap();
|
||||
// ep.pool_idx = 0;
|
||||
// ep.set_idx = 0;
|
||||
// ep.disk_idx = 0;
|
||||
// ep.is_local = true;
|
||||
|
||||
// let disk = new_disk(&ep, &DiskOption::default()).await.expect("init disk fail");
|
||||
|
||||
// // let disk = match LocalDisk::new(&ep, false).await {
|
||||
// // Ok(res) => res,
|
||||
// // Err(err) => {
|
||||
// // println!("LocalDisk::new err {:?}", err);
|
||||
// // return;
|
||||
// // }
|
||||
// // };
|
||||
|
||||
// let (rd, mut wr) = tokio::io::duplex(64);
|
||||
|
||||
// let job = tokio::spawn(async move {
|
||||
// let opts = WalkDirOptions {
|
||||
// bucket: "dada".to_owned(),
|
||||
// base_dir: "".to_owned(),
|
||||
// recursive: true,
|
||||
// ..Default::default()
|
||||
// };
|
||||
|
||||
// println!("walk opts {:?}", opts);
|
||||
// if let Err(err) = disk.walk_dir(opts, &mut wr).await {
|
||||
// println!("walk_dir err {:?}", err);
|
||||
// }
|
||||
// });
|
||||
|
||||
// let job2 = tokio::spawn(async move {
|
||||
// let mut mrd = MetacacheReader::new(rd);
|
||||
|
||||
// loop {
|
||||
// match mrd.peek().await {
|
||||
// Ok(res) => {
|
||||
// if let Some(info) = res {
|
||||
// println!("info {:?}", info.name)
|
||||
// } else {
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// Err(err) => {
|
||||
// if is_err_eof(&err) {
|
||||
// break;
|
||||
// }
|
||||
|
||||
// println!("get err {:?}", err);
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
// join_all(vec![job, job2]).await;
|
||||
// }
|
||||
|
||||
// #[tokio::test]
|
||||
// async fn test_list_path_raw() {
|
||||
// let mut ep = Endpoint::try_from("/Users/weisd/project/weisd/s3-rustfs/target/volume/test").unwrap();
|
||||
// ep.pool_idx = 0;
|
||||
// ep.set_idx = 0;
|
||||
// ep.disk_idx = 0;
|
||||
// ep.is_local = true;
|
||||
|
||||
// let disk = new_disk(&ep, &DiskOption::default()).await.expect("init disk fail");
|
||||
|
||||
// // let disk = match LocalDisk::new(&ep, false).await {
|
||||
// // Ok(res) => res,
|
||||
// // Err(err) => {
|
||||
// // println!("LocalDisk::new err {:?}", err);
|
||||
// // return;
|
||||
// // }
|
||||
// // };
|
||||
|
||||
// let (_, rx) = broadcast::channel(1);
|
||||
// let bucket = "dada".to_owned();
|
||||
// let forward_to = None;
|
||||
// let disks = vec![Some(disk)];
|
||||
// let fallback_disks = Vec::new();
|
||||
|
||||
// list_path_raw(
|
||||
// rx,
|
||||
// ListPathRawOptions {
|
||||
// disks,
|
||||
// fallback_disks,
|
||||
// bucket,
|
||||
// path: "".to_owned(),
|
||||
// recursice: true,
|
||||
// forward_to,
|
||||
// min_disks: 1,
|
||||
// report_not_found: false,
|
||||
// agreed: Some(Box::new(move |entry: MetaCacheEntry| {
|
||||
// Box::pin(async move { println!("get entry: {}", entry.name) })
|
||||
// })),
|
||||
// partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<Error>]| {
|
||||
// Box::pin(async move { println!("get entries: {:?}", entries) })
|
||||
// })),
|
||||
// finished: None,
|
||||
// ..Default::default()
|
||||
// },
|
||||
// )
|
||||
// .await
|
||||
// .unwrap();
|
||||
// }
|
||||
|
||||
// #[tokio::test]
|
||||
// async fn test_set_list_path() {
|
||||
// let mut ep = Endpoint::try_from("/Users/weisd/project/weisd/s3-rustfs/target/volume/test").unwrap();
|
||||
// ep.pool_idx = 0;
|
||||
// ep.set_idx = 0;
|
||||
// ep.disk_idx = 0;
|
||||
// ep.is_local = true;
|
||||
|
||||
// let disk = new_disk(&ep, &DiskOption::default()).await.expect("init disk fail");
|
||||
// let _ = disk.set_disk_id(Some(Uuid::new_v4())).await;
|
||||
|
||||
// let set = SetDisks {
|
||||
// lockers: Vec::new(),
|
||||
// locker_owner: String::new(),
|
||||
// ns_mutex: Arc::new(RwLock::new(NsLockMap::new(false))),
|
||||
// disks: RwLock::new(vec![Some(disk)]),
|
||||
// set_endpoints: Vec::new(),
|
||||
// set_drive_count: 1,
|
||||
// default_parity_count: 0,
|
||||
// set_index: 0,
|
||||
// pool_index: 0,
|
||||
// format: FormatV3::new(1, 1),
|
||||
// };
|
||||
|
||||
// let (_tx, rx) = broadcast::channel(1);
|
||||
|
||||
// let bucket = "dada".to_owned();
|
||||
|
||||
// let opts = ListPathOptions {
|
||||
// bucket,
|
||||
// recursive: true,
|
||||
// ..Default::default()
|
||||
// };
|
||||
|
||||
// let (sender, mut recv) = mpsc::channel(10);
|
||||
|
||||
// set.list_path(rx, opts, sender).await.unwrap();
|
||||
|
||||
// while let Some(entry) = recv.recv().await {
|
||||
// println!("get entry {:?}", entry.name)
|
||||
// }
|
||||
// }
|
||||
|
||||
// #[tokio::test]
|
||||
//walk() {
|
||||
// let server_address = "localhost:9000";
|
||||
|
||||
// let (endpoint_pools, _setup_type) = EndpointServerPools::from_volumes(
|
||||
// server_address,
|
||||
// vec!["/Users/weisd/project/weisd/s3-rustfs/target/volume/test".to_string()],
|
||||
// )
|
||||
// .unwrap();
|
||||
|
||||
// let store = ECStore::new(server_address.to_string(), endpoint_pools.clone())
|
||||
// .await
|
||||
// .unwrap();
|
||||
|
||||
// let (_tx, rx) = broadcast::channel(1);
|
||||
|
||||
// let bucket = "dada".to_owned();
|
||||
// let opts = ListPathOptions {
|
||||
// bucket,
|
||||
// recursive: true,
|
||||
// ..Default::default()
|
||||
// };
|
||||
|
||||
// let (sender, mut recv) = mpsc::channel(10);
|
||||
|
||||
// store.list_merged(rx, opts, sender).await.unwrap();
|
||||
|
||||
// while let Some(entry) = recv.recv().await {
|
||||
// println!("get entry {:?}", entry.name)
|
||||
// }
|
||||
// }
|
||||
|
||||
// #[tokio::test]
|
||||
// async fn test_list_path() {
|
||||
// let server_address = "localhost:9000";
|
||||
|
||||
// let (endpoint_pools, _setup_type) = EndpointServerPools::from_volumes(
|
||||
// server_address,
|
||||
// vec!["/Users/weisd/project/weisd/s3-rustfs/target/volume/test".to_string()],
|
||||
// )
|
||||
// .unwrap();
|
||||
|
||||
// let store = ECStore::new(server_address.to_string(), endpoint_pools.clone())
|
||||
// .await
|
||||
// .unwrap();
|
||||
|
||||
// let bucket = "dada".to_owned();
|
||||
// let opts = ListPathOptions {
|
||||
// bucket,
|
||||
// recursive: true,
|
||||
// limit: 100,
|
||||
|
||||
// ..Default::default()
|
||||
// };
|
||||
|
||||
// let ret = store.list_path(&opts).await.unwrap();
|
||||
// println!("ret {:?}", ret);
|
||||
// }
|
||||
|
||||
// #[tokio::test]
|
||||
// async fn test_list_objects_v2() {
|
||||
// let server_address = "localhost:9000";
|
||||
|
||||
// let (endpoint_pools, _setup_type) = EndpointServerPools::from_volumes(
|
||||
// server_address,
|
||||
// vec!["/Users/weisd/project/weisd/s3-rustfs/target/volume/test".to_string()],
|
||||
// )
|
||||
// .unwrap();
|
||||
|
||||
// let store = ECStore::new(server_address.to_string(), endpoint_pools.clone())
|
||||
// .await
|
||||
// .unwrap();
|
||||
|
||||
// let ret = store.list_objects_v2("data", "", "", "", 100, false, "").await.unwrap();
|
||||
// println!("ret {:?}", ret);
|
||||
// }
|
||||
|
||||
// #[tokio::test]
|
||||
// async fn test_walk() {
|
||||
// let server_address = "localhost:9000";
|
||||
|
||||
// let (endpoint_pools, _setup_type) = EndpointServerPools::from_volumes(
|
||||
// server_address,
|
||||
// vec!["/Users/weisd/project/weisd/s3-rustfs/target/volume/test".to_string()],
|
||||
// )
|
||||
// .unwrap();
|
||||
|
||||
// let store = ECStore::new(server_address.to_string(), endpoint_pools.clone())
|
||||
// .await
|
||||
// .unwrap();
|
||||
|
||||
// ECStore::init(store.clone()).await.unwrap();
|
||||
|
||||
// let (_tx, rx) = broadcast::channel(1);
|
||||
|
||||
// let bucket = ".rustfs.sys";
|
||||
// let prefix = "config/iam/sts/";
|
||||
|
||||
// let (sender, mut recv) = mpsc::channel(10);
|
||||
|
||||
// let opts = WalkOptions::default();
|
||||
|
||||
// store.walk(rx, bucket, prefix, sender, opts).await.unwrap();
|
||||
|
||||
// while let Some(entry) = recv.recv().await {
|
||||
// println!("get entry {:?}", entry)
|
||||
// }
|
||||
// }
|
||||
|
||||
#[tokio::test]
|
||||
async fn merge_entry_channels_produces_sorted_unique_output_from_two_channels() {
|
||||
let (tx_a, rx_a) = mpsc::channel(4);
|
||||
|
||||
@@ -277,9 +277,10 @@ pub struct FileInfo {
|
||||
/// Values of these keys must never reach logs at any level.
|
||||
fn is_sensitive_metadata_key(key: &str) -> bool {
|
||||
// `is_encryption_metadata_key` covers the x-minio-internal- SSE prefix but not
|
||||
// its x-rustfs-internal- twin, which the dual-key invariant writes alongside it.
|
||||
// its reserved x-rustfs-internal- twin, which has no writer today but must
|
||||
// stay redacted in case one appears.
|
||||
is_encryption_metadata_key(key)
|
||||
|| starts_with_ignore_ascii_case(key, "x-rustfs-internal-server-side-encryption-")
|
||||
|| starts_with_ignore_ascii_case(key, rustfs_utils::http::RUSTFS_INTERNAL_ENCRYPTION_PREFIX)
|
||||
|| rustfs_utils::http::REPLICATION_SSE_TRANSPORT_PREFIXES
|
||||
.iter()
|
||||
.any(|prefix| starts_with_ignore_ascii_case(key, prefix))
|
||||
|
||||
@@ -91,7 +91,6 @@ metrics = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
libc = { workspace = true }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
rustfs-test-utils = { workspace = true }
|
||||
serial_test = { workspace = true }
|
||||
|
||||
@@ -785,9 +785,15 @@ mod tests {
|
||||
let heal_manager = create_test_heal_manager();
|
||||
let processor = HealChannelProcessor::new(heal_manager);
|
||||
|
||||
// Verify processor is created successfully
|
||||
let _sender = processor.get_response_sender();
|
||||
// If we can get the sender, processor was created correctly
|
||||
let sender = processor.get_response_sender();
|
||||
sender
|
||||
.send(HealChannelResponse {
|
||||
request_id: "request-id".to_string(),
|
||||
success: true,
|
||||
data: None,
|
||||
error: None,
|
||||
})
|
||||
.expect("a freshly constructed processor must accept responses on its channel");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1778,9 +1784,22 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_process_cancel_request_treats_unknown_path_as_stopped() {
|
||||
async fn test_process_cancel_request_cancels_cluster_task_for_legacy_root_path() {
|
||||
let heal_manager = create_test_heal_manager();
|
||||
let processor = HealChannelProcessor::new(heal_manager);
|
||||
let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High);
|
||||
let cluster_task_id = cluster_request.id.clone();
|
||||
let bucket_request = HealRequest::bucket("bucket".to_string());
|
||||
let bucket_task_id = bucket_request.id.clone();
|
||||
heal_manager
|
||||
.submit_heal_request(cluster_request)
|
||||
.await
|
||||
.expect("cluster request should be accepted");
|
||||
heal_manager
|
||||
.submit_heal_request(bucket_request)
|
||||
.await
|
||||
.expect("bucket request should be accepted");
|
||||
|
||||
let processor = HealChannelProcessor::new(heal_manager.clone());
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
processor
|
||||
@@ -1796,6 +1815,38 @@ mod tests {
|
||||
assert_eq!(response.request_id, ".");
|
||||
assert_eq!(response.data.as_deref(), Some("stopped".as_bytes()));
|
||||
assert!(response.error.is_none());
|
||||
assert!(matches!(
|
||||
heal_manager.get_task_status(&cluster_task_id).await,
|
||||
Err(crate::Error::TaskNotFound { .. })
|
||||
));
|
||||
assert_eq!(
|
||||
heal_manager
|
||||
.get_task_status(&bucket_task_id)
|
||||
.await
|
||||
.expect("bucket request should not match the root path"),
|
||||
HealTaskStatus::Pending
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_process_cancel_request_treats_unknown_path_as_stopped() {
|
||||
let heal_manager = create_test_heal_manager();
|
||||
let processor = HealChannelProcessor::new(heal_manager);
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
processor
|
||||
.process_cancel_request("missing".to_string(), String::new(), tx)
|
||||
.await
|
||||
.expect("cancel should process");
|
||||
|
||||
let response = rx
|
||||
.await
|
||||
.expect("oneshot should resolve")
|
||||
.expect("cancel response should be returned");
|
||||
assert!(response.success);
|
||||
assert_eq!(response.request_id, "missing");
|
||||
assert_eq!(response.data.as_deref(), Some("stopped".as_bytes()));
|
||||
assert!(response.error.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1229,6 +1229,12 @@ mod resume_loop_tests {
|
||||
Timeout,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
enum ReplacementCommitEvidence {
|
||||
Confirmed(bool),
|
||||
Error(String),
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FakeStorage {
|
||||
/// page keyed by the *incoming* continuation token
|
||||
@@ -1239,7 +1245,7 @@ mod resume_loop_tests {
|
||||
results: Mutex<HashMap<String, HealResultItem>>,
|
||||
/// Target-specific physical readback evidence per `compose_key`; the
|
||||
/// fake models a healthy backend unless a test explicitly revokes it.
|
||||
replacement_commit_evidence: Mutex<HashMap<String, bool>>,
|
||||
replacement_commit_evidence: Mutex<HashMap<String, ReplacementCommitEvidence>>,
|
||||
/// every heal_object call recorded as (name, version_id)
|
||||
heal_calls: Mutex<Vec<(String, Option<String>)>>,
|
||||
replacement_target_identity_sequences: Mutex<VecDeque<Vec<ReplacementTargetIdentity>>>,
|
||||
@@ -1260,7 +1266,13 @@ mod resume_loop_tests {
|
||||
self.replacement_commit_evidence
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(compose_key(name, version), committed);
|
||||
.insert(compose_key(name, version), ReplacementCommitEvidence::Confirmed(committed));
|
||||
}
|
||||
fn set_replacement_commit_evidence_error(&self, name: &str, version: Option<&str>, message: &str) {
|
||||
self.replacement_commit_evidence
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(compose_key(name, version), ReplacementCommitEvidence::Error(message.to_string()));
|
||||
}
|
||||
fn calls(&self) -> Vec<(String, Option<String>)> {
|
||||
self.heal_calls.lock().unwrap().clone()
|
||||
@@ -1354,12 +1366,17 @@ mod resume_loop_tests {
|
||||
_opts: &HealOpts,
|
||||
_targets: &[String],
|
||||
) -> Result<bool> {
|
||||
Ok(*self
|
||||
match self
|
||||
.replacement_commit_evidence
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get(&compose_key(object, version_id))
|
||||
.unwrap_or(&true))
|
||||
.cloned()
|
||||
.unwrap_or(ReplacementCommitEvidence::Confirmed(true))
|
||||
{
|
||||
ReplacementCommitEvidence::Confirmed(committed) => Ok(committed),
|
||||
ReplacementCommitEvidence::Error(message) => Err(Error::other(message)),
|
||||
}
|
||||
}
|
||||
async fn list_objects_for_heal(&self, _b: &str, _p: &str) -> Result<Vec<HealListItem>> {
|
||||
Ok(Vec::new())
|
||||
@@ -2142,6 +2159,58 @@ mod resume_loop_tests {
|
||||
assert!(env.checkpoint.get_checkpoint().await.processed_objects.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_delete_marker_readback_error_keeps_auto_heal_resumable() {
|
||||
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
let healer = ErasureSetHealer::new(
|
||||
env.storage.clone(),
|
||||
Arc::new(RwLock::new(HealProgress::new())),
|
||||
CancellationToken::new(),
|
||||
env.healer.disk.clone(),
|
||||
HealOpts::default(),
|
||||
HealRequestSource::AutoHeal,
|
||||
)
|
||||
.with_replacement_targets(vec!["replacement-a".to_string()], Some("generation-a".to_string()));
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![item("object", Some("dm-v1"), true)],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
env.storage.set_result(
|
||||
"object",
|
||||
Some("dm-v1"),
|
||||
HealResultItem {
|
||||
after: Infos {
|
||||
drives: vec![HealDriveInfo {
|
||||
endpoint: "replacement-a".to_string(),
|
||||
state: "ok".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
env.storage
|
||||
.set_replacement_commit_evidence_error("object", Some("dm-v1"), "injected target readback failure");
|
||||
|
||||
let result = healer
|
||||
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||
.await;
|
||||
|
||||
let error = result.expect_err("a target readback error must not complete automatic replacement");
|
||||
let error = error.to_string();
|
||||
assert!(error.contains("Transient heal skip"), "unexpected error: {error}");
|
||||
assert!(error.contains("retry scheduled"), "unexpected error: {error}");
|
||||
let state = env.resume.get_state().await;
|
||||
assert!(!state.completed, "readback errors must leave the replacement task incomplete");
|
||||
assert_eq!(state.retry_count, 1, "readback errors must arm the bounded retry path");
|
||||
assert!(env.checkpoint.get_checkpoint().await.processed_objects.is_empty());
|
||||
assert_eq!(env.storage.calls(), vec![("object".to_string(), Some("dm-v1".to_string()))]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_targeted_heal_keeps_existing_best_effort_result_semantics() {
|
||||
let env = make_env_with_targets(vec!["replacement-a".to_string()]).await;
|
||||
|
||||
@@ -52,6 +52,7 @@ const EVENT_HEAL_MAINLINE_THROTTLE: &str = "heal_mainline_throttle";
|
||||
const EVENT_HEAL_SCHEDULER_STATE: &str = "heal_scheduler_state";
|
||||
const EVENT_HEAL_QUEUE_STATE: &str = "heal_queue_state";
|
||||
const EVENT_HEAL_UNCLEAN_SHUTDOWN: &str = "heal_unclean_shutdown";
|
||||
const LEGACY_ROOT_HEAL_PATH: &str = ".";
|
||||
const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3;
|
||||
const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
|
||||
@@ -65,6 +66,26 @@ fn durable_replacement_recovery_is_due(state: &ResumeState, task_id: &str) -> bo
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending)))
|
||||
}
|
||||
|
||||
fn replacement_discovery_error_is_expected_for_deferred_endpoint(
|
||||
error: &Error,
|
||||
endpoint: &str,
|
||||
deferred_replacement_endpoints: &HashSet<String>,
|
||||
) -> bool {
|
||||
matches!(error, Error::Disk(DiskError::UnformattedDisk)) && deferred_replacement_endpoints.contains(endpoint)
|
||||
}
|
||||
|
||||
fn unblock_replacement_recovery_sets_after_validation(
|
||||
blocked_sets: &mut HashSet<String>,
|
||||
retry_succeeded: HashSet<String>,
|
||||
retry_failed: &HashSet<String>,
|
||||
) {
|
||||
for set_disk_id in retry_succeeded {
|
||||
if !retry_failed.contains(&set_disk_id) {
|
||||
blocked_sets.remove(&set_disk_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Admission/scheduler outcomes for per-object requests (Object/Metadata/MRF/
|
||||
// ECDecode) log via demote_to_debug_when! — MRF, autoheal, and scanner
|
||||
// recovery loops submit those per object, so a full queue or a retry storm
|
||||
@@ -581,7 +602,7 @@ impl RetryingHeal {
|
||||
|
||||
fn heal_type_matches_path(heal_type: &HealType, heal_path: &str) -> bool {
|
||||
let heal_path = heal_path.trim_matches('/');
|
||||
if heal_path.is_empty() {
|
||||
if heal_path.is_empty() || heal_path == LEGACY_ROOT_HEAL_PATH {
|
||||
return matches!(heal_type, HealType::Cluster);
|
||||
}
|
||||
|
||||
@@ -2488,6 +2509,7 @@ impl HealManager {
|
||||
let mut endpoints = HashMap::<String, Vec<Endpoint>>::new();
|
||||
let mut durable_recoveries = HashMap::<String, (String, Vec<Endpoint>, Vec<String>, String)>::new();
|
||||
let mut conflicted_recovery_sets = HashSet::<String>::new();
|
||||
let mut deferred_replacement_endpoints = HashSet::<String>::new();
|
||||
let local_disks = {
|
||||
let local_disk_map = local_disk_map_read().await;
|
||||
local_disk_map.values().flatten().cloned().collect::<Vec<_>>()
|
||||
@@ -2532,11 +2554,7 @@ impl HealManager {
|
||||
let mut blocked = replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned");
|
||||
for set_disk_id in retry_succeeded {
|
||||
if !retry_failed.contains(&set_disk_id) {
|
||||
blocked.remove(&set_disk_id);
|
||||
}
|
||||
}
|
||||
unblock_replacement_recovery_sets_after_validation(&mut blocked, retry_succeeded, &retry_failed);
|
||||
}
|
||||
for disk in &local_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
@@ -2569,6 +2587,7 @@ impl HealManager {
|
||||
if !super::replacement_readiness::auto_replacement_target_ready(disk, &local_disks)
|
||||
.await
|
||||
{
|
||||
deferred_replacement_endpoints.insert(endpoint.to_string());
|
||||
skipped_invalid_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
@@ -2642,6 +2661,24 @@ impl HealManager {
|
||||
let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await {
|
||||
Ok(task_ids) => task_ids,
|
||||
Err(error) => {
|
||||
let endpoint_string = endpoint.to_string();
|
||||
if replacement_discovery_error_is_expected_for_deferred_endpoint(
|
||||
&error,
|
||||
&endpoint_string,
|
||||
&deferred_replacement_endpoints,
|
||||
) {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
disk_state = "replacement_path_unavailable",
|
||||
result = "recovery_records_unavailable",
|
||||
"Replacement recovery discovery skipped for deferred replacement"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if let Some(set_disk_id) = &disk_set_disk_id {
|
||||
conflicted_recovery_sets.insert(set_disk_id.clone());
|
||||
}
|
||||
@@ -3491,11 +3528,13 @@ fn can_schedule_request(request: &HealRequest, running_per_set: &HashMap<String,
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::heal::EcstoreError;
|
||||
use crate::heal::resume::{CheckpointManager, ReplacementTargetIdentity};
|
||||
use crate::heal::storage::{HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::task::{BatchHealFailure, HealOptions, HealPriority, HealRequest, HealTask, HealType};
|
||||
use rustfs_common::heal_channel::{HealOpts, HealRequestSource};
|
||||
use rustfs_concurrency::{WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use std::sync::Mutex as StdMutex;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo};
|
||||
@@ -3616,6 +3655,9 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
if let Some(hook) = manager_recovery_test_hook() {
|
||||
*hook.listed.lock().expect("manager recovery listed lock should not poison") = true;
|
||||
}
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
@@ -3638,6 +3680,12 @@ mod tests {
|
||||
_version_id: Option<&str>,
|
||||
_opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
if let Some(hook) = manager_recovery_test_hook() {
|
||||
*hook
|
||||
.heal_object_calls
|
||||
.lock()
|
||||
.expect("manager recovery object call lock should not poison") += 1;
|
||||
}
|
||||
if bucket == "retry-transition" {
|
||||
return Ok((
|
||||
HealResultItem::default(),
|
||||
@@ -3651,10 +3699,38 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> Result<HealResultItem> {
|
||||
if let Some(hook) = manager_recovery_test_hook() {
|
||||
*hook
|
||||
.bucket_heal_calls
|
||||
.lock()
|
||||
.expect("manager recovery bucket call lock should not poison") += 1;
|
||||
}
|
||||
Ok(HealResultItem::default())
|
||||
}
|
||||
|
||||
async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
|
||||
if let Some(hook) = manager_recovery_test_hook() {
|
||||
*hook
|
||||
.global_format_calls
|
||||
.lock()
|
||||
.expect("manager recovery global format call lock should not poison") += 1;
|
||||
}
|
||||
Ok((HealResultItem::default(), None))
|
||||
}
|
||||
|
||||
async fn heal_replacement_format(
|
||||
&self,
|
||||
_dry_run: bool,
|
||||
_pool_index: usize,
|
||||
_set_index: usize,
|
||||
_targets: &[String],
|
||||
) -> Result<(HealResultItem, Option<Error>)> {
|
||||
if let Some(hook) = manager_recovery_test_hook() {
|
||||
*hook
|
||||
.replacement_format_calls
|
||||
.lock()
|
||||
.expect("manager recovery replacement format call lock should not poison") += 1;
|
||||
}
|
||||
Ok((HealResultItem::default(), None))
|
||||
}
|
||||
|
||||
@@ -3674,6 +3750,89 @@ mod tests {
|
||||
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> Result<DiskStore> {
|
||||
Err(Error::other("not implemented in tests"))
|
||||
}
|
||||
|
||||
async fn get_replacement_resume_disk(
|
||||
&self,
|
||||
_set_disk_id: &str,
|
||||
_task_id: &str,
|
||||
_excluded_targets: &[String],
|
||||
) -> Result<crate::heal::storage::ReplacementResumeDisk> {
|
||||
let Some(hook) = manager_recovery_test_hook() else {
|
||||
return Err(Error::other("not implemented in tests"));
|
||||
};
|
||||
Ok(crate::heal::storage::ReplacementResumeDisk::Existing(
|
||||
hook.replacement_resume_disk.clone(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
struct ManagerRecoveryTestHook {
|
||||
replacement_resume_disk: DiskStore,
|
||||
listed: StdMutex<bool>,
|
||||
global_format_calls: StdMutex<u32>,
|
||||
replacement_format_calls: StdMutex<u32>,
|
||||
bucket_heal_calls: StdMutex<u32>,
|
||||
heal_object_calls: StdMutex<u32>,
|
||||
}
|
||||
|
||||
static MANAGER_RECOVERY_TEST_HOOK: LazyLock<StdMutex<Option<Arc<ManagerRecoveryTestHook>>>> =
|
||||
LazyLock::new(|| StdMutex::new(None));
|
||||
|
||||
struct ManagerRecoveryTestHookGuard;
|
||||
|
||||
impl ManagerRecoveryTestHook {
|
||||
fn install(replacement_resume_disk: DiskStore) -> (Arc<Self>, ManagerRecoveryTestHookGuard) {
|
||||
let hook = Arc::new(Self {
|
||||
replacement_resume_disk,
|
||||
listed: StdMutex::new(false),
|
||||
global_format_calls: StdMutex::new(0),
|
||||
replacement_format_calls: StdMutex::new(0),
|
||||
bucket_heal_calls: StdMutex::new(0),
|
||||
heal_object_calls: StdMutex::new(0),
|
||||
});
|
||||
let previous = MANAGER_RECOVERY_TEST_HOOK
|
||||
.lock()
|
||||
.expect("manager recovery hook lock should not poison")
|
||||
.replace(hook.clone());
|
||||
assert!(previous.is_none(), "manager recovery hook already installed");
|
||||
(hook, ManagerRecoveryTestHookGuard)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ManagerRecoveryTestHookGuard {
|
||||
fn drop(&mut self) {
|
||||
*MANAGER_RECOVERY_TEST_HOOK
|
||||
.lock()
|
||||
.expect("manager recovery hook lock should not poison") = None;
|
||||
}
|
||||
}
|
||||
|
||||
fn manager_recovery_test_hook() -> Option<Arc<ManagerRecoveryTestHook>> {
|
||||
MANAGER_RECOVERY_TEST_HOOK
|
||||
.lock()
|
||||
.expect("manager recovery hook lock should not poison")
|
||||
.clone()
|
||||
}
|
||||
|
||||
async fn make_manager_resume_disk(temp: &TempDir, name: &str) -> DiskStore {
|
||||
let disk_path = temp.path().join(name);
|
||||
std::fs::create_dir_all(&disk_path).expect("manager recovery disk directory should be created");
|
||||
let endpoint = Endpoint::try_from(disk_path.to_string_lossy().as_ref()).expect("manager recovery endpoint should parse");
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("manager recovery disk should initialize");
|
||||
let metadata_volume = disk.make_volume(super::super::RUSTFS_META_BUCKET).await;
|
||||
assert!(
|
||||
matches!(metadata_volume, Ok(()) | Err(DiskError::VolumeExists)),
|
||||
"manager recovery metadata volume should exist: {metadata_volume:?}"
|
||||
);
|
||||
disk
|
||||
}
|
||||
|
||||
fn bucket_request(bucket: &str, priority: HealPriority, source: HealRequestSource) -> HealRequest {
|
||||
@@ -4525,6 +4684,165 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replacement_recovery_discovery_unformatted_is_quiet_only_for_deferred_endpoint() {
|
||||
let error = Error::Disk(DiskError::UnformattedDisk);
|
||||
let deferred = HashSet::from(["endpoint-a".to_string()]);
|
||||
|
||||
assert!(replacement_discovery_error_is_expected_for_deferred_endpoint(
|
||||
&error,
|
||||
"endpoint-a",
|
||||
&deferred
|
||||
));
|
||||
assert!(!replacement_discovery_error_is_expected_for_deferred_endpoint(
|
||||
&error,
|
||||
"endpoint-b",
|
||||
&deferred
|
||||
));
|
||||
assert!(!replacement_discovery_error_is_expected_for_deferred_endpoint(
|
||||
&Error::Disk(DiskError::Timeout),
|
||||
"endpoint-a",
|
||||
&deferred
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replacement_recovery_retry_barrier_requires_all_set_records_to_validate() {
|
||||
let mut blocked = HashSet::from(["pool_0_set_0".to_string(), "pool_0_set_1".to_string()]);
|
||||
let retry_succeeded = HashSet::from(["pool_0_set_0".to_string(), "pool_0_set_1".to_string()]);
|
||||
let retry_failed = HashSet::from(["pool_0_set_0".to_string()]);
|
||||
|
||||
unblock_replacement_recovery_sets_after_validation(&mut blocked, retry_succeeded, &retry_failed);
|
||||
|
||||
assert!(
|
||||
blocked.contains("pool_0_set_0"),
|
||||
"one failed disk record must keep the whole replacement set blocked"
|
||||
);
|
||||
assert!(
|
||||
!blocked.contains("pool_0_set_1"),
|
||||
"a blocked set may resume only after every retried record validates"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduler_completes_cleanup_pending_recovery_from_manager_anchor() {
|
||||
let temp = TempDir::new().expect("temporary manager recovery directory should be created");
|
||||
let anchor = make_manager_resume_disk(&temp, "anchor").await;
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let target = "replacement-a".to_string();
|
||||
let identity = ReplacementTargetIdentity {
|
||||
endpoint: target.clone(),
|
||||
canonical_path: "/replacement/replacement-a".to_string(),
|
||||
physical_device_ids: vec!["replacement-a".to_string()],
|
||||
filesystem_identity: "identity-replacement-a".to_string(),
|
||||
};
|
||||
let resume_manager = ResumeManager::new_replacement_intent(
|
||||
anchor.clone(),
|
||||
task_id.clone(),
|
||||
"pool_0_set_0".to_string(),
|
||||
vec!["bucket-a".to_string()],
|
||||
vec![target.clone()],
|
||||
vec![identity],
|
||||
)
|
||||
.await
|
||||
.expect("cleanup-pending replacement state should persist on the survivor anchor");
|
||||
resume_manager
|
||||
.mark_replacement_completed_and_verified()
|
||||
.await
|
||||
.expect("completion proof should persist before cleanup");
|
||||
resume_manager
|
||||
.mark_replacement_cleanup_pending()
|
||||
.await
|
||||
.expect("cleanup-pending state should persist before restart");
|
||||
CheckpointManager::new(anchor.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("checkpoint fixture should persist");
|
||||
|
||||
let (hook, _hook_guard) = ManagerRecoveryTestHook::install(anchor.clone());
|
||||
let storage = Arc::new(MockStorage);
|
||||
let manager = HealManager::new(storage.clone(), None);
|
||||
let mut request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec!["bucket-a".to_string()],
|
||||
set_disk_id: "pool_0_set_0".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
..HealOptions::default()
|
||||
},
|
||||
HealPriority::Low,
|
||||
);
|
||||
request.id = task_id.clone();
|
||||
request.source = HealRequestSource::AutoHeal;
|
||||
request.heal_endpoints = vec![target];
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request(request)
|
||||
.await
|
||||
.expect("durable recovery request should be admitted"),
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
manager
|
||||
.replacement_recovery_anchors
|
||||
.lock()
|
||||
.expect("replacement recovery anchor lock should not poison")
|
||||
.insert(task_id.clone(), anchor.endpoint().to_string());
|
||||
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
loop {
|
||||
let resume_removed = !ResumeManager::has_resume_state(&anchor, &task_id).await;
|
||||
let checkpoint_removed = !CheckpointManager::has_checkpoint(&anchor, &task_id).await;
|
||||
let anchor_removed = !manager
|
||||
.replacement_recovery_anchors
|
||||
.lock()
|
||||
.expect("replacement recovery anchor lock should not poison")
|
||||
.contains_key(&task_id);
|
||||
if resume_removed && checkpoint_removed && anchor_removed {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cleanup-pending recovery should finish through the manager scheduler");
|
||||
|
||||
assert!(
|
||||
!*hook.listed.lock().expect("manager recovery listed lock should not poison"),
|
||||
"cleanup-pending recovery must not list buckets or restart object healing"
|
||||
);
|
||||
assert_eq!(
|
||||
*hook
|
||||
.global_format_calls
|
||||
.lock()
|
||||
.expect("manager recovery global format call lock should not poison"),
|
||||
0
|
||||
);
|
||||
assert_eq!(
|
||||
*hook
|
||||
.replacement_format_calls
|
||||
.lock()
|
||||
.expect("manager recovery replacement format call lock should not poison"),
|
||||
0,
|
||||
"manager-resumed terminal cleanup must not format replacement targets"
|
||||
);
|
||||
assert_eq!(
|
||||
*hook
|
||||
.bucket_heal_calls
|
||||
.lock()
|
||||
.expect("manager recovery bucket call lock should not poison"),
|
||||
0
|
||||
);
|
||||
assert_eq!(
|
||||
*hook
|
||||
.heal_object_calls
|
||||
.lock()
|
||||
.expect("manager recovery object call lock should not poison"),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_retry_request_for_scoped_slowdown_preserves_scope() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
@@ -4793,6 +5111,17 @@ mod tests {
|
||||
assert!(manager.retrying_heals.lock().await.get(&bucket_request_id).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_type_matches_path_accepts_legacy_root() {
|
||||
assert!(heal_type_matches_path(&HealType::Cluster, LEGACY_ROOT_HEAL_PATH));
|
||||
assert!(!heal_type_matches_path(
|
||||
&HealType::Bucket {
|
||||
bucket: "bucket".to_string(),
|
||||
},
|
||||
LEGACY_ROOT_HEAL_PATH,
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_retrying_duplicate_token_can_query_and_cancel_original_retry() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
|
||||
@@ -157,225 +157,4 @@ mod tests {
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux_privileged_tests {
|
||||
use super::*;
|
||||
use std::error::Error;
|
||||
use std::ffi::CString;
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
use std::path::Path;
|
||||
|
||||
const ENABLE_ENV: &str = "RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS";
|
||||
|
||||
struct MountGuard {
|
||||
mounts: Vec<std::path::PathBuf>,
|
||||
}
|
||||
|
||||
impl MountGuard {
|
||||
fn new() -> Result<Self, Box<dyn Error + Send + Sync>> {
|
||||
let rc = unsafe { libc::unshare(libc::CLONE_NEWNS) };
|
||||
if rc != 0 {
|
||||
return Err(format!("unshare(CLONE_NEWNS) failed: {}", std::io::Error::last_os_error()).into());
|
||||
}
|
||||
make_mounts_private()?;
|
||||
Ok(Self { mounts: Vec::new() })
|
||||
}
|
||||
|
||||
fn mount_tmpfs(&mut self, target: &Path, label: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
mount_tmpfs(target, label)?;
|
||||
self.mounts.push(target.to_path_buf());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn mount_bind(&mut self, source: &Path, target: &Path) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
mount_bind(source, target)?;
|
||||
self.mounts.push(target.to_path_buf());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MountGuard {
|
||||
fn drop(&mut self) {
|
||||
for mount in self.mounts.iter().rev() {
|
||||
if let Ok(target) = c_path(mount) {
|
||||
let _ = unsafe { libc::umount2(target.as_ptr(), libc::MNT_DETACH) };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn c_path(path: &Path) -> Result<CString, Box<dyn Error + Send + Sync>> {
|
||||
Ok(CString::new(path.as_os_str().as_bytes())?)
|
||||
}
|
||||
|
||||
fn make_mounts_private() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let root = CString::new("/")?;
|
||||
let rc = unsafe {
|
||||
libc::mount(
|
||||
std::ptr::null(),
|
||||
root.as_ptr(),
|
||||
std::ptr::null(),
|
||||
(libc::MS_REC | libc::MS_PRIVATE) as libc::c_ulong,
|
||||
std::ptr::null(),
|
||||
)
|
||||
};
|
||||
if rc != 0 {
|
||||
return Err(format!("making the mount namespace private failed: {}", std::io::Error::last_os_error()).into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn mount_tmpfs(target: &Path, label: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let source = CString::new(label)?;
|
||||
let target = c_path(target)?;
|
||||
let fstype = CString::new("tmpfs")?;
|
||||
let data = CString::new("size=32m,mode=0700")?;
|
||||
let rc = unsafe {
|
||||
libc::mount(
|
||||
source.as_ptr(),
|
||||
target.as_ptr(),
|
||||
fstype.as_ptr(),
|
||||
(libc::MS_NOSUID | libc::MS_NODEV) as libc::c_ulong,
|
||||
data.as_ptr().cast(),
|
||||
)
|
||||
};
|
||||
if rc != 0 {
|
||||
return Err(format!("mount(tmpfs) failed: {}", std::io::Error::last_os_error()).into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn mount_bind(source: &Path, target: &Path) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let source = c_path(source)?;
|
||||
let target = c_path(target)?;
|
||||
let rc = unsafe {
|
||||
libc::mount(
|
||||
source.as_ptr(),
|
||||
target.as_ptr(),
|
||||
std::ptr::null(),
|
||||
libc::MS_BIND as libc::c_ulong,
|
||||
std::ptr::null(),
|
||||
)
|
||||
};
|
||||
if rc != 0 {
|
||||
return Err(format!("mount(MS_BIND) failed: {}", std::io::Error::last_os_error()).into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn privileged_enabled() -> Result<bool, Box<dyn Error + Send + Sync>> {
|
||||
let enabled = std::env::var(ENABLE_ENV)
|
||||
.ok()
|
||||
.is_some_and(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES"));
|
||||
if !enabled {
|
||||
return Ok(false);
|
||||
}
|
||||
if unsafe { libc::geteuid() } != 0 {
|
||||
return Err(format!("{ENABLE_ENV}=1 requires root or CAP_SYS_ADMIN").into());
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn run_privileged_mount_test<F, Fut>(test: F) -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
where
|
||||
F: FnOnce(MountGuard) -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = Result<(), Box<dyn Error + Send + Sync>>> + 'static,
|
||||
{
|
||||
if !privileged_enabled()? {
|
||||
return Ok(());
|
||||
}
|
||||
std::thread::spawn(move || {
|
||||
let guard = MountGuard::new()?;
|
||||
let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
|
||||
runtime.block_on(test(guard))
|
||||
})
|
||||
.join()
|
||||
.map_err(|_| "privileged mount readiness test thread panicked")?
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS=1"]
|
||||
fn auto_replacement_readiness_accepts_an_independent_mount() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
run_privileged_mount_test(|mut mounts| async move {
|
||||
let temp = TempDir::new().expect("temporary replacement roots should be created");
|
||||
let target = temp.path().join("target");
|
||||
let sibling = temp.path().join("sibling");
|
||||
std::fs::create_dir(&target).expect("target mountpoint should be created");
|
||||
std::fs::create_dir(&sibling).expect("sibling mountpoint should be created");
|
||||
mounts.mount_tmpfs(&target, "rustfs-readiness-target")?;
|
||||
mounts.mount_tmpfs(&sibling, "rustfs-readiness-sibling")?;
|
||||
|
||||
let target_endpoint = Endpoint::try_from(target.to_string_lossy().as_ref())?;
|
||||
let sibling_endpoint = Endpoint::try_from(sibling.to_string_lossy().as_ref())?;
|
||||
let target_disk = new_disk(
|
||||
&target_endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let sibling_disk = new_disk(
|
||||
&sibling_endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let identity = auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await;
|
||||
assert!(
|
||||
identity.is_some(),
|
||||
"a separately mounted replacement target with no sibling device overlap must be admitted"
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS=1"]
|
||||
fn auto_replacement_readiness_rejects_a_same_device_sibling_bind_mount() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
run_privileged_mount_test(|mut mounts| async move {
|
||||
let temp = TempDir::new().expect("temporary replacement roots should be created");
|
||||
let source = temp.path().join("source");
|
||||
let target = temp.path().join("target");
|
||||
let sibling = temp.path().join("sibling");
|
||||
std::fs::create_dir(&source).expect("source mountpoint should be created");
|
||||
std::fs::create_dir(&target).expect("target mountpoint should be created");
|
||||
std::fs::create_dir(&sibling).expect("sibling mountpoint should be created");
|
||||
mounts.mount_tmpfs(&source, "rustfs-readiness-shared-source")?;
|
||||
mounts.mount_bind(&source, &target)?;
|
||||
mounts.mount_bind(&source, &sibling)?;
|
||||
|
||||
let target_endpoint = Endpoint::try_from(target.to_string_lossy().as_ref())?;
|
||||
let sibling_endpoint = Endpoint::try_from(sibling.to_string_lossy().as_ref())?;
|
||||
let target_disk = new_disk(
|
||||
&target_endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let sibling_disk = new_disk(
|
||||
&sibling_endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert!(
|
||||
auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()])
|
||||
.await
|
||||
.is_none(),
|
||||
"replacement readiness must reject a target sharing its physical device with a sibling endpoint"
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2075,6 +2075,7 @@ impl ResumeUtils {
|
||||
match disk.list_dir("", RUSTFS_META_BUCKET, recovery_dir, -1).await {
|
||||
Ok(entries) => Ok(entries),
|
||||
Err(DiskError::FileNotFound) => Ok(Vec::new()),
|
||||
Err(error @ DiskError::UnformattedDisk) => Err(error.into()),
|
||||
Err(error) => Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to list replacement recovery records: {error}"),
|
||||
}),
|
||||
|
||||
@@ -2997,6 +2997,147 @@ mod tests {
|
||||
assert!(!*storage.listed.lock().unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_pending_recovery_removes_checkpoint_without_rebuild_work() {
|
||||
let temp = TempDir::new().expect("temporary resume disk directory should be created");
|
||||
let anchor = make_resume_disk(&temp).await;
|
||||
let task_id = crate::heal::resume::ResumeUtils::generate_task_id();
|
||||
let identity = replacement_identity("replacement-a", "device-a", "filesystem-a");
|
||||
let resume_manager = ResumeManager::new_replacement_intent(
|
||||
anchor.clone(),
|
||||
task_id.clone(),
|
||||
"pool_0_set_0".to_string(),
|
||||
vec!["bucket-a".to_string()],
|
||||
vec!["replacement-a".to_string()],
|
||||
vec![identity],
|
||||
)
|
||||
.await
|
||||
.expect("terminal replacement state should persist on the survivor anchor");
|
||||
resume_manager
|
||||
.mark_replacement_completed_and_verified()
|
||||
.await
|
||||
.expect("terminal replacement proof should persist before cleanup");
|
||||
resume_manager
|
||||
.mark_replacement_cleanup_pending()
|
||||
.await
|
||||
.expect("failed cleanup must retain a cleanup-pending state");
|
||||
CheckpointManager::new(anchor.clone(), task_id.clone())
|
||||
.await
|
||||
.expect("checkpoint fixture should persist");
|
||||
assert!(
|
||||
CheckpointManager::has_checkpoint(&anchor, &task_id).await,
|
||||
"checkpoint fixture must exist before restart cleanup"
|
||||
);
|
||||
|
||||
let storage = Arc::new(MockStorage {
|
||||
replacement_resume_disk: Mutex::new(Some(anchor.clone())),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec!["bucket-a".to_string()],
|
||||
set_disk_id: "pool_0_set_0".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
..HealOptions::default()
|
||||
},
|
||||
HealPriority::Low,
|
||||
);
|
||||
request.id = task_id.clone();
|
||||
request.source = HealRequestSource::AutoHeal;
|
||||
request.heal_endpoints = vec!["replacement-a".to_string()];
|
||||
|
||||
HealTask::from_replacement_recovery_request(request, storage.clone(), Some(anchor.endpoint().to_string()))
|
||||
.execute()
|
||||
.await
|
||||
.expect("cleanup-pending recovery must finish terminal cleanup");
|
||||
|
||||
assert!(
|
||||
!CheckpointManager::has_checkpoint(&anchor, &task_id).await,
|
||||
"terminal cleanup must remove the retained checkpoint"
|
||||
);
|
||||
assert!(
|
||||
!ResumeManager::has_resume_state(&anchor, &task_id).await,
|
||||
"terminal cleanup must remove the retained resume state"
|
||||
);
|
||||
assert_eq!(*storage.global_format_calls.lock().unwrap(), 0);
|
||||
assert!(
|
||||
storage.replacement_format_calls.lock().unwrap().is_empty(),
|
||||
"terminal checkpoint cleanup must not format replacement targets"
|
||||
);
|
||||
assert!(storage.bucket_heal_calls.lock().unwrap().is_empty());
|
||||
assert!(storage.heal_object_calls.lock().unwrap().is_empty());
|
||||
assert!(!*storage.listed.lock().unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn verified_recovery_keeps_state_when_marker_clear_fails() {
|
||||
let temp = TempDir::new().expect("temporary resume disk directory should be created");
|
||||
let anchor = make_resume_disk(&temp).await;
|
||||
let task_id = crate::heal::resume::ResumeUtils::generate_task_id();
|
||||
let target = format!("replacement-marker-missing-{task_id}");
|
||||
let identity = replacement_identity(&target, &target, &format!("identity-{target}"));
|
||||
let resume_manager = ResumeManager::new_replacement_intent(
|
||||
anchor.clone(),
|
||||
task_id.clone(),
|
||||
"pool_0_set_0".to_string(),
|
||||
vec!["bucket-a".to_string()],
|
||||
vec![target.clone()],
|
||||
vec![identity],
|
||||
)
|
||||
.await
|
||||
.expect("verified replacement state should persist on the survivor anchor");
|
||||
resume_manager
|
||||
.mark_replacement_completed_and_verified()
|
||||
.await
|
||||
.expect("verified state must persist proof before marker cleanup");
|
||||
|
||||
let storage = Arc::new(MockStorage {
|
||||
replacement_resume_disk: Mutex::new(Some(anchor.clone())),
|
||||
replacement_targets_ready: Mutex::new(true),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec!["bucket-a".to_string()],
|
||||
set_disk_id: "pool_0_set_0".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
..HealOptions::default()
|
||||
},
|
||||
HealPriority::Low,
|
||||
);
|
||||
request.id = task_id.clone();
|
||||
request.source = HealRequestSource::AutoHeal;
|
||||
request.heal_endpoints = vec![target];
|
||||
|
||||
let error = HealTask::from_replacement_recovery_request(request, storage.clone(), Some(anchor.endpoint().to_string()))
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("marker clear failure must keep the durable terminal state retryable");
|
||||
|
||||
assert!(error.to_string().contains("healing marker target is unavailable"));
|
||||
let state = ResumeManager::load_replacement_intent(anchor.clone(), &task_id)
|
||||
.await
|
||||
.expect("verified state must remain for retry after marker clear failure")
|
||||
.get_state()
|
||||
.await;
|
||||
assert!(state.completed);
|
||||
assert_eq!(state.replacement_phase, ReplacementPhase::Verified);
|
||||
assert_eq!(*storage.global_format_calls.lock().unwrap(), 0);
|
||||
assert!(
|
||||
storage.replacement_format_calls.lock().unwrap().is_empty(),
|
||||
"marker cleanup retry must not format replacement targets again"
|
||||
);
|
||||
assert!(storage.bucket_heal_calls.lock().unwrap().is_empty());
|
||||
assert!(storage.heal_object_calls.lock().unwrap().is_empty());
|
||||
assert!(!*storage.listed.lock().unwrap());
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct MockStorage {
|
||||
listed: Mutex<bool>,
|
||||
|
||||
@@ -107,7 +107,10 @@ url = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
pollster.workspace = true
|
||||
rcgen.workspace = true
|
||||
rustfs-test-utils = { workspace = true }
|
||||
rustls = { workspace = true, default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
rustls-pki-types.workspace = true
|
||||
serial_test = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tempfile = { workspace = true }
|
||||
|
||||
+23
-2
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use manager::IamCache;
|
||||
use oidc::OidcSys;
|
||||
use oidc::{OidcExtraRootCaProvider, OidcSys};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use store::object::ObjectStore;
|
||||
use sys::IamSys;
|
||||
@@ -284,6 +284,23 @@ pub fn get_global_iam_sys() -> Option<Arc<IamSys<ObjectStore>>> {
|
||||
|
||||
/// Initialize the global OIDC system. Non-fatal if no OIDC providers are configured.
|
||||
pub async fn init_oidc_sys() -> Result<()> {
|
||||
init_oidc_sys_with_extra_root_ca(None).await
|
||||
}
|
||||
|
||||
/// Initialize the global OIDC system with an additional outbound root CA bundle.
|
||||
pub async fn init_oidc_sys_with_extra_root_ca(root_ca_pem: Option<&[u8]>) -> Result<()> {
|
||||
init_oidc_sys_with_extra_root_ca_provider_inner(None, root_ca_pem).await
|
||||
}
|
||||
|
||||
/// Initialize the global OIDC system with a reload-aware outbound root CA provider.
|
||||
pub async fn init_oidc_sys_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result<()> {
|
||||
init_oidc_sys_with_extra_root_ca_provider_inner(Some(extra_root_ca_provider), None).await
|
||||
}
|
||||
|
||||
async fn init_oidc_sys_with_extra_root_ca_provider_inner(
|
||||
extra_root_ca_provider: Option<OidcExtraRootCaProvider>,
|
||||
root_ca_pem: Option<&[u8]>,
|
||||
) -> Result<()> {
|
||||
if OIDC_SYS.get().is_some() {
|
||||
debug!(
|
||||
event = EVENT_OIDC_STATE,
|
||||
@@ -303,7 +320,11 @@ pub async fn init_oidc_sys() -> Result<()> {
|
||||
"OIDC runtime starting"
|
||||
);
|
||||
|
||||
let oidc_sys = match OidcSys::new().await {
|
||||
let oidc_sys_result = match extra_root_ca_provider {
|
||||
Some(provider) => OidcSys::new_with_extra_root_ca_provider(provider).await,
|
||||
None => OidcSys::new_with_extra_root_ca(root_ca_pem).await,
|
||||
};
|
||||
let oidc_sys = match oidc_sys_result {
|
||||
Ok(sys) => {
|
||||
if sys.has_providers() {
|
||||
debug!(
|
||||
|
||||
+420
-48
@@ -25,7 +25,7 @@ use openidconnect::{
|
||||
JsonWebKeySetUrl, LogoutRequest, Nonce, PkceCodeChallenge, PkceCodeVerifier, PostLogoutRedirectUrl,
|
||||
ProviderMetadataWithLogout, RedirectUrl, RequestTokenError, Scope,
|
||||
};
|
||||
use reqwest::Client;
|
||||
use reqwest::{Certificate, Client};
|
||||
use rustfs_config::oidc::*;
|
||||
use rustfs_config::server_config::{Config as ServerConfig, KVS};
|
||||
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, EnableState, MAX_OIDC_RESPONSE_SIZE};
|
||||
@@ -38,7 +38,6 @@ use std::fmt;
|
||||
use std::future::Future;
|
||||
use std::net::IpAddr;
|
||||
use std::pin::Pin;
|
||||
#[cfg(test)]
|
||||
use std::sync::Arc;
|
||||
use std::sync::{LazyLock, Mutex, MutexGuard, RwLock};
|
||||
use std::time::{Duration as StdDuration, Instant};
|
||||
@@ -258,6 +257,7 @@ fn oidc_http_error_diagnostics(error: &OidcHttpError) -> (&'static str, String)
|
||||
}
|
||||
OidcHttpError::Reqwest(_) => ("request", String::new()),
|
||||
OidcHttpError::Http(_) => ("http_build", String::new()),
|
||||
OidcHttpError::ExtraRootCa(_) => ("extra_root_ca", String::new()),
|
||||
OidcHttpError::ForbiddenOutbound(_) => ("forbidden_outbound", String::new()),
|
||||
OidcHttpError::ResponseTooLarge(limit) => ("response_too_large", limit.to_string()),
|
||||
}
|
||||
@@ -270,6 +270,7 @@ fn oidc_http_error_diagnostics(error: &OidcHttpError) -> (&'static str, String)
|
||||
pub enum OidcHttpError {
|
||||
Reqwest(reqwest::Error),
|
||||
Http(http::Error),
|
||||
ExtraRootCa(String),
|
||||
/// The outbound destination was rejected by the shared egress policy before any
|
||||
/// connection was attempted (invalid URL, loopback/link-local/metadata/private IP,
|
||||
/// or a malformed allow-origins configuration).
|
||||
@@ -284,6 +285,7 @@ impl std::fmt::Display for OidcHttpError {
|
||||
match self {
|
||||
Self::Reqwest(e) => write!(f, "{e}"),
|
||||
Self::Http(e) => write!(f, "{e}"),
|
||||
Self::ExtraRootCa(reason) => write!(f, "failed to load OIDC extra root CA bundle: {reason}"),
|
||||
Self::ForbiddenOutbound(reason) => write!(f, "outbound request rejected: {reason}"),
|
||||
Self::ResponseTooLarge(limit) => write!(f, "oidc response body exceeds {limit} bytes"),
|
||||
}
|
||||
@@ -295,11 +297,48 @@ impl std::error::Error for OidcHttpError {
|
||||
match self {
|
||||
Self::Reqwest(e) => Some(e),
|
||||
Self::Http(e) => Some(e),
|
||||
Self::ForbiddenOutbound(_) | Self::ResponseTooLarge(_) => None,
|
||||
Self::ExtraRootCa(_) | Self::ForbiddenOutbound(_) | Self::ResponseTooLarge(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct OidcExtraRootCaMaterial {
|
||||
pub generation: u64,
|
||||
pub root_ca_pem: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
type OidcExtraRootCaFuture = Pin<Box<dyn Future<Output = Result<OidcExtraRootCaMaterial, String>> + Send>>;
|
||||
type OidcExtraRootCaLoader = dyn Fn() -> OidcExtraRootCaFuture + Send + Sync;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct OidcExtraRootCaProvider {
|
||||
loader: Arc<OidcExtraRootCaLoader>,
|
||||
}
|
||||
|
||||
impl OidcExtraRootCaProvider {
|
||||
pub fn new<F, Fut>(loader: F) -> Self
|
||||
where
|
||||
F: Fn() -> Fut + Send + Sync + 'static,
|
||||
Fut: Future<Output = Result<OidcExtraRootCaMaterial, String>> + Send + 'static,
|
||||
{
|
||||
Self {
|
||||
loader: Arc::new(move || Box::pin(loader())),
|
||||
}
|
||||
}
|
||||
|
||||
async fn load(&self) -> Result<OidcExtraRootCaMaterial, String> {
|
||||
(self.loader)().await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CachedOidcExtraRootCerts {
|
||||
generation: u64,
|
||||
initialized: bool,
|
||||
certs: Vec<Certificate>,
|
||||
}
|
||||
|
||||
/// HTTP client adapter bridging reqwest 0.13 to the `openidconnect` `AsyncHttpClient` trait.
|
||||
///
|
||||
/// A fresh client is built for every request so the destination is re-validated and the
|
||||
@@ -311,10 +350,26 @@ pub(crate) struct ReqwestHttpClient {
|
||||
/// `None` in production: the process-cached outbound policy from the environment is used.
|
||||
/// `Some(..)` only in tests, to explicitly allow a loopback mock endpoint.
|
||||
policy_override: Option<OutboundPolicy>,
|
||||
extra_root_certs: Arc<RwLock<CachedOidcExtraRootCerts>>,
|
||||
extra_root_ca_provider: Option<OidcExtraRootCaProvider>,
|
||||
#[cfg(test)]
|
||||
dns_resolver_override: Option<Arc<dyn reqwest::dns::Resolve>>,
|
||||
}
|
||||
|
||||
fn parse_oidc_extra_root_certs(source: &str, pem: &[u8]) -> Result<Vec<Certificate>, String> {
|
||||
if pem.iter().all(|byte| byte.is_ascii_whitespace()) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
Certificate::from_pem_bundle(pem).map_err(|err| format!("failed to parse OIDC extra root CA bundle from {source}: {err}"))
|
||||
}
|
||||
|
||||
fn oidc_extra_root_certs(root_ca_pem: Option<&[u8]>) -> Result<Vec<Certificate>, String> {
|
||||
match root_ca_pem {
|
||||
Some(pem) => parse_oidc_extra_root_certs("RustFS outbound TLS material", pem),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a reqwest client pinned to the shared outbound egress policy for a single request.
|
||||
///
|
||||
/// [`OutboundPolicy::resolver_for`] validates the URL shape and rejects loopback,
|
||||
@@ -326,6 +381,7 @@ pub(crate) struct ReqwestHttpClient {
|
||||
fn build_oidc_http_client(
|
||||
uri: &str,
|
||||
policy_override: Option<&OutboundPolicy>,
|
||||
extra_root_certs: &[Certificate],
|
||||
#[cfg(test)] dns_resolver_override: Option<Arc<dyn reqwest::dns::Resolve>>,
|
||||
) -> Result<(Client, Url), OidcHttpError> {
|
||||
let url = Url::parse(uri).map_err(|_| OidcHttpError::ForbiddenOutbound("invalid outbound OIDC URL".to_string()))?;
|
||||
@@ -356,6 +412,9 @@ fn build_oidc_http_client(
|
||||
if bypass_proxy {
|
||||
builder = builder.no_proxy();
|
||||
}
|
||||
if !extra_root_certs.is_empty() {
|
||||
builder = builder.tls_certs_merge(extra_root_certs.iter().cloned());
|
||||
}
|
||||
builder.build().map(|client| (client, url)).map_err(OidcHttpError::Reqwest)
|
||||
}
|
||||
|
||||
@@ -410,19 +469,93 @@ fn should_bypass_proxy_for_oidc_uri(uri: &str) -> bool {
|
||||
|
||||
impl ReqwestHttpClient {
|
||||
fn new() -> Result<Self, String> {
|
||||
Self::new_with_extra_root_certs(Vec::new())
|
||||
}
|
||||
|
||||
fn extra_root_cert_cache(certs: Vec<Certificate>) -> Arc<RwLock<CachedOidcExtraRootCerts>> {
|
||||
Arc::new(RwLock::new(CachedOidcExtraRootCerts {
|
||||
generation: 0,
|
||||
initialized: true,
|
||||
certs,
|
||||
}))
|
||||
}
|
||||
|
||||
fn new_with_extra_root_certs(extra_root_certs: Vec<Certificate>) -> Result<Self, String> {
|
||||
Ok(Self {
|
||||
policy_override: None,
|
||||
extra_root_certs: Self::extra_root_cert_cache(extra_root_certs),
|
||||
extra_root_ca_provider: None,
|
||||
#[cfg(test)]
|
||||
dns_resolver_override: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn new_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result<Self, String> {
|
||||
Ok(Self {
|
||||
policy_override: None,
|
||||
extra_root_certs: Arc::new(RwLock::new(CachedOidcExtraRootCerts::default())),
|
||||
extra_root_ca_provider: Some(extra_root_ca_provider),
|
||||
#[cfg(test)]
|
||||
dns_resolver_override: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn current_extra_root_certs(&self) -> Result<Vec<Certificate>, OidcHttpError> {
|
||||
let Some(provider) = self.extra_root_ca_provider.as_ref() else {
|
||||
return self
|
||||
.extra_root_certs
|
||||
.read()
|
||||
.map(|cache| cache.certs.clone())
|
||||
.map_err(|e| OidcHttpError::ExtraRootCa(format!("extra root certificate cache lock poisoned: {e}")));
|
||||
};
|
||||
|
||||
let material = provider.load().await.map_err(OidcHttpError::ExtraRootCa)?;
|
||||
if let Ok(cache) = self.extra_root_certs.read()
|
||||
&& cache.initialized
|
||||
&& cache.generation == material.generation
|
||||
{
|
||||
return Ok(cache.certs.clone());
|
||||
}
|
||||
|
||||
let certs = oidc_extra_root_certs(material.root_ca_pem.as_deref()).map_err(OidcHttpError::ExtraRootCa)?;
|
||||
let mut cache = self
|
||||
.extra_root_certs
|
||||
.write()
|
||||
.map_err(|e| OidcHttpError::ExtraRootCa(format!("extra root certificate cache lock poisoned: {e}")))?;
|
||||
cache.generation = material.generation;
|
||||
cache.initialized = true;
|
||||
cache.certs = certs.clone();
|
||||
Ok(certs)
|
||||
}
|
||||
|
||||
/// Test-only constructor that pins outbound requests to an explicit policy, so a
|
||||
/// loopback mock server can be reached without depending on process-wide environment.
|
||||
#[cfg(test)]
|
||||
fn with_policy(policy: OutboundPolicy) -> Self {
|
||||
Self {
|
||||
policy_override: Some(policy),
|
||||
extra_root_certs: Self::extra_root_cert_cache(Vec::new()),
|
||||
extra_root_ca_provider: None,
|
||||
dns_resolver_override: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn with_policy_and_extra_root_certs(policy: OutboundPolicy, extra_root_certs: Vec<Certificate>) -> Self {
|
||||
Self {
|
||||
policy_override: Some(policy),
|
||||
extra_root_certs: Self::extra_root_cert_cache(extra_root_certs),
|
||||
extra_root_ca_provider: None,
|
||||
dns_resolver_override: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn with_policy_and_extra_root_ca_provider(policy: OutboundPolicy, extra_root_ca_provider: OidcExtraRootCaProvider) -> Self {
|
||||
Self {
|
||||
policy_override: Some(policy),
|
||||
extra_root_certs: Arc::new(RwLock::new(CachedOidcExtraRootCerts::default())),
|
||||
extra_root_ca_provider: Some(extra_root_ca_provider),
|
||||
dns_resolver_override: None,
|
||||
}
|
||||
}
|
||||
@@ -431,6 +564,8 @@ impl ReqwestHttpClient {
|
||||
fn with_policy_and_dns_resolver(policy: OutboundPolicy, resolver: Arc<dyn reqwest::dns::Resolve>) -> Self {
|
||||
Self {
|
||||
policy_override: Some(policy),
|
||||
extra_root_certs: Self::extra_root_cert_cache(Vec::new()),
|
||||
extra_root_ca_provider: None,
|
||||
dns_resolver_override: Some(resolver),
|
||||
}
|
||||
}
|
||||
@@ -461,9 +596,11 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
|
||||
);
|
||||
}
|
||||
|
||||
let extra_root_certs = self.current_extra_root_certs().await?;
|
||||
let (client, url) = build_oidc_http_client(
|
||||
&uri,
|
||||
self.policy_override.as_ref(),
|
||||
&extra_root_certs,
|
||||
#[cfg(test)]
|
||||
self.dns_resolver_override.clone(),
|
||||
)?;
|
||||
@@ -678,7 +815,22 @@ fn trusted_aud(other_audiences: &[String], audience: &Audience) -> bool {
|
||||
impl OidcSys {
|
||||
/// Parse environment variables and discover all configured OIDC providers.
|
||||
pub async fn new() -> Result<Self, String> {
|
||||
let http_client = ReqwestHttpClient::new()?;
|
||||
Self::new_with_extra_root_ca(None).await
|
||||
}
|
||||
|
||||
/// Parse environment variables and discover providers with an additional outbound root CA bundle.
|
||||
pub(crate) async fn new_with_extra_root_ca(root_ca_pem: Option<&[u8]>) -> Result<Self, String> {
|
||||
let http_client = ReqwestHttpClient::new_with_extra_root_certs(oidc_extra_root_certs(root_ca_pem)?)?;
|
||||
Self::new_with_http_client(http_client).await
|
||||
}
|
||||
|
||||
pub(crate) async fn new_with_extra_root_ca_provider(extra_root_ca_provider: OidcExtraRootCaProvider) -> Result<Self, String> {
|
||||
let http_client = ReqwestHttpClient::new_with_extra_root_ca_provider(extra_root_ca_provider)?;
|
||||
http_client.current_extra_root_certs().await.map_err(|err| err.to_string())?;
|
||||
Self::new_with_http_client(http_client).await
|
||||
}
|
||||
|
||||
async fn new_with_http_client(http_client: ReqwestHttpClient) -> Result<Self, String> {
|
||||
let server_config = crate::server_config::current_server_config();
|
||||
let parsed_configs = load_effective_oidc_provider_configs(server_config.as_ref());
|
||||
let mut configs = HashMap::new();
|
||||
@@ -1874,7 +2026,14 @@ pub fn load_effective_oidc_provider_configs(server_config: Option<&ServerConfig>
|
||||
}
|
||||
|
||||
pub async fn validate_oidc_provider_config(config: &OidcProviderConfig) -> Result<OidcProviderValidationResult, String> {
|
||||
let http_client = ReqwestHttpClient::new()?;
|
||||
validate_oidc_provider_config_with_extra_root_ca(config, None).await
|
||||
}
|
||||
|
||||
pub async fn validate_oidc_provider_config_with_extra_root_ca(
|
||||
config: &OidcProviderConfig,
|
||||
root_ca_pem: Option<&[u8]>,
|
||||
) -> Result<OidcProviderValidationResult, String> {
|
||||
let http_client = ReqwestHttpClient::new_with_extra_root_certs(oidc_extra_root_certs(root_ca_pem)?)?;
|
||||
let state = OidcSys::discover_provider(config, &http_client).await?;
|
||||
|
||||
Ok(OidcProviderValidationResult {
|
||||
@@ -2438,6 +2597,50 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn read_mock_oidc_request_path(stream: &mut impl std::io::Read) -> String {
|
||||
let mut request_bytes = Vec::new();
|
||||
let mut buffer = [0u8; 4096];
|
||||
loop {
|
||||
match stream.read(&mut buffer) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => request_bytes.extend_from_slice(&buffer[..n]),
|
||||
Err(e) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => break,
|
||||
Err(_) => break,
|
||||
}
|
||||
if request_bytes.windows(4).any(|w| w == b"\r\n\r\n") {
|
||||
break;
|
||||
}
|
||||
if request_bytes.len() >= 8192 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let request = String::from_utf8_lossy(&request_bytes);
|
||||
request
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.split_whitespace()
|
||||
.nth(1)
|
||||
.unwrap_or("")
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn mock_oidc_response(path: &str, discovery_body: &str, expected_jwks_path: &str, jwks_body: &str) -> String {
|
||||
let (status, body) = if path.contains("/.well-known/openid-configuration") {
|
||||
(200, discovery_body)
|
||||
} else if path == expected_jwks_path {
|
||||
(200, jwks_body)
|
||||
} else {
|
||||
(404, r#"{"error":"not found"}"#)
|
||||
};
|
||||
|
||||
format!(
|
||||
"HTTP/1.1 {status} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
if status == 200 { "OK" } else { "Not Found" },
|
||||
body.len()
|
||||
)
|
||||
}
|
||||
|
||||
fn start_mock_oidc_discovery_server<F>(
|
||||
build_discovery_issuer: F,
|
||||
max_requests: usize,
|
||||
@@ -2445,7 +2648,6 @@ mod tests {
|
||||
where
|
||||
F: Fn(&str) -> (String, String, String) + Send + 'static,
|
||||
{
|
||||
use std::io::Read;
|
||||
use std::io::Write;
|
||||
use std::net::{Shutdown, TcpListener};
|
||||
use std::sync::mpsc;
|
||||
@@ -2516,41 +2718,8 @@ mod tests {
|
||||
.set_read_timeout(Some(Duration::from_secs(1)))
|
||||
.expect("failed to set discovery mock read timeout");
|
||||
|
||||
let mut request_bytes = Vec::new();
|
||||
let mut buffer = [0u8; 4096];
|
||||
loop {
|
||||
match stream.read(&mut buffer) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => request_bytes.extend_from_slice(&buffer[..n]),
|
||||
Err(e) if matches!(e.kind(), std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut) => {
|
||||
break;
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
if request_bytes.windows(4).any(|w| w == b"\r\n\r\n") {
|
||||
break;
|
||||
}
|
||||
if request_bytes.len() >= 8192 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let request = String::from_utf8_lossy(&request_bytes);
|
||||
let path = request.lines().next().unwrap_or("").split_whitespace().nth(1).unwrap_or("");
|
||||
|
||||
let (status, body) = if path.contains("/.well-known/openid-configuration") {
|
||||
(200, discovery_body.as_str())
|
||||
} else if path == expected_jwks_path {
|
||||
(200, jwks_body)
|
||||
} else {
|
||||
(404, r#"{"error":"not found"}"#)
|
||||
};
|
||||
|
||||
let response = format!(
|
||||
"HTTP/1.1 {status} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
if status == 200 { "OK" } else { "Not Found" },
|
||||
body.len()
|
||||
);
|
||||
|
||||
let path = read_mock_oidc_request_path(&mut stream);
|
||||
let response = mock_oidc_response(&path, &discovery_body, &expected_jwks_path, jwks_body);
|
||||
let _ = stream.write_all(response.as_bytes());
|
||||
let _ = stream.flush();
|
||||
let _ = stream.shutdown(Shutdown::Both);
|
||||
@@ -2568,6 +2737,114 @@ mod tests {
|
||||
Some((base, handle))
|
||||
}
|
||||
|
||||
fn start_mock_oidc_tls_discovery_server<F>(
|
||||
build_discovery_issuer: F,
|
||||
max_requests: usize,
|
||||
) -> Option<(String, String, std::thread::JoinHandle<()>)>
|
||||
where
|
||||
F: Fn(&str) -> (String, String, String) + Send + 'static,
|
||||
{
|
||||
use std::io::Write;
|
||||
use std::net::{Shutdown, TcpListener};
|
||||
use std::sync::mpsc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const IDLE_SHUTDOWN: Duration = Duration::from_secs(1);
|
||||
const ABSOLUTE_CAP: Duration = Duration::from_secs(5);
|
||||
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
let certified =
|
||||
rcgen::generate_simple_self_signed(vec!["127.0.0.1".to_string()]).expect("generate OIDC TLS test certificate");
|
||||
let cert_pem = certified.cert.pem();
|
||||
let server_config = rustls::ServerConfig::builder()
|
||||
.with_no_client_auth()
|
||||
.with_single_cert(
|
||||
vec![certified.cert.der().clone()],
|
||||
rustls_pki_types::PrivateKeyDer::try_from(certified.signing_key.serialize_der())
|
||||
.expect("convert OIDC TLS test private key"),
|
||||
)
|
||||
.expect("build OIDC TLS mock server config");
|
||||
|
||||
let listener = match TcpListener::bind("127.0.0.1:0") {
|
||||
Ok(listener) => listener,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
|
||||
Err(err) => panic!("test TLS listener should bind: {err}"),
|
||||
};
|
||||
let base = format!("https://{}", listener.local_addr().expect("listener local address should be available"));
|
||||
let (discovery_issuer, discovery_jwks_uri, expected_jwks_path) = build_discovery_issuer(&base);
|
||||
let discovery_body = serde_json::json!({
|
||||
"issuer": discovery_issuer,
|
||||
"authorization_endpoint": format!("{base}/authorize"),
|
||||
"token_endpoint": format!("{base}/token"),
|
||||
"jwks_uri": discovery_jwks_uri,
|
||||
"response_types_supported": ["code"],
|
||||
"response_modes_supported": ["query"],
|
||||
"subject_types_supported": ["public"],
|
||||
"id_token_signing_alg_values_supported": ["RS256"],
|
||||
})
|
||||
.to_string();
|
||||
let jwks_body = r#"{"keys":[]}"#;
|
||||
let (ready_tx, ready_rx) = mpsc::channel();
|
||||
|
||||
let handle = std::thread::spawn(move || {
|
||||
let server_config = Arc::new(server_config);
|
||||
listener
|
||||
.set_nonblocking(true)
|
||||
.expect("failed to set TLS discovery mock listener non-blocking");
|
||||
let _ = ready_tx.send(());
|
||||
|
||||
let mut seen = 0usize;
|
||||
let start = Instant::now();
|
||||
let mut last_completed = Instant::now();
|
||||
|
||||
loop {
|
||||
if seen > 0 && last_completed.elapsed() >= IDLE_SHUTDOWN {
|
||||
break;
|
||||
}
|
||||
if start.elapsed() >= ABSOLUTE_CAP {
|
||||
break;
|
||||
}
|
||||
|
||||
let tcp_stream = match listener.accept() {
|
||||
Ok((stream, _)) => stream,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
|
||||
std::thread::sleep(Duration::from_millis(5));
|
||||
continue;
|
||||
}
|
||||
Err(_) => break,
|
||||
};
|
||||
tcp_stream
|
||||
.set_nonblocking(false)
|
||||
.expect("failed to set TLS discovery mock stream blocking");
|
||||
tcp_stream
|
||||
.set_read_timeout(Some(Duration::from_secs(1)))
|
||||
.expect("failed to set TLS discovery mock read timeout");
|
||||
|
||||
seen += 1;
|
||||
let connection = match rustls::ServerConnection::new(server_config.clone()) {
|
||||
Ok(connection) => connection,
|
||||
Err(_) => break,
|
||||
};
|
||||
let mut stream = rustls::StreamOwned::new(connection, tcp_stream);
|
||||
let path = read_mock_oidc_request_path(&mut stream);
|
||||
let response = mock_oidc_response(&path, &discovery_body, &expected_jwks_path, jwks_body);
|
||||
let _ = stream.write_all(response.as_bytes());
|
||||
let _ = stream.flush();
|
||||
let _ = stream.sock.shutdown(Shutdown::Both);
|
||||
last_completed = Instant::now();
|
||||
|
||||
if seen >= max_requests {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
ready_rx
|
||||
.recv_timeout(Duration::from_millis(100))
|
||||
.expect("mock TLS OIDC discovery server should become ready");
|
||||
|
||||
Some((base, cert_pem, handle))
|
||||
}
|
||||
|
||||
fn discovery_error_contains_all_variants(err: &str, base: &str) -> bool {
|
||||
err.contains(base) && err.contains(&format!("{base}/")) && err.contains("discovery failed for all issuer variants")
|
||||
}
|
||||
@@ -2590,6 +2867,100 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oidc_discovery_accepts_extra_root_ca_for_https_provider() {
|
||||
let Some((base, ca_pem, handle)) = start_mock_oidc_tls_discovery_server(
|
||||
|base| (format!("{base}/application/o/rustfs"), format!("{base}/jwks"), "/jwks".to_string()),
|
||||
4,
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
let config_url = format!("{base}/application/o/rustfs");
|
||||
let config = build_mocked_oidc_provider_config("default", &config_url);
|
||||
let origin = Url::parse(&config.config_url)
|
||||
.expect("mock config_url should parse")
|
||||
.origin()
|
||||
.ascii_serialization();
|
||||
let policy = OutboundPolicy::from_allowed_origins(&origin).expect("loopback TLS origin should be allowed");
|
||||
let extra_root_certs =
|
||||
parse_oidc_extra_root_certs("test OIDC TLS CA", ca_pem.as_bytes()).expect("test CA bundle should parse");
|
||||
let http_client = ReqwestHttpClient::with_policy_and_extra_root_certs(policy, extra_root_certs);
|
||||
|
||||
let state = OidcSys::discover_provider(&config, &http_client)
|
||||
.await
|
||||
.expect("OIDC discovery should trust the extra root CA");
|
||||
|
||||
assert_eq!(state.metadata.issuer().to_string(), format!("{base}/application/o/rustfs"));
|
||||
assert!(handle.join().is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oidc_discovery_refreshes_extra_root_ca_when_generation_changes() {
|
||||
let Some((base_a, ca_pem_a, handle_a)) = start_mock_oidc_tls_discovery_server(
|
||||
|base| (format!("{base}/application/o/rustfs-a"), format!("{base}/jwks"), "/jwks".to_string()),
|
||||
4,
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
let Some((base_b, ca_pem_b, handle_b)) = start_mock_oidc_tls_discovery_server(
|
||||
|base| (format!("{base}/application/o/rustfs-b"), format!("{base}/jwks"), "/jwks".to_string()),
|
||||
4,
|
||||
) else {
|
||||
assert!(handle_a.join().is_ok());
|
||||
return;
|
||||
};
|
||||
|
||||
let origin_a = Url::parse(&base_a)
|
||||
.expect("mock base A should parse")
|
||||
.origin()
|
||||
.ascii_serialization();
|
||||
let origin_b = Url::parse(&base_b)
|
||||
.expect("mock base B should parse")
|
||||
.origin()
|
||||
.ascii_serialization();
|
||||
let allowed_origins = format!("{origin_a},{origin_b}");
|
||||
let policy = OutboundPolicy::from_allowed_origins(&allowed_origins).expect("loopback TLS origins should be allowed");
|
||||
let material = Arc::new(Mutex::new(OidcExtraRootCaMaterial {
|
||||
generation: 1,
|
||||
root_ca_pem: Some(ca_pem_a.into_bytes()),
|
||||
}));
|
||||
let provider = OidcExtraRootCaProvider::new({
|
||||
let material = material.clone();
|
||||
move || {
|
||||
let material = material.clone();
|
||||
async move {
|
||||
material
|
||||
.lock()
|
||||
.map(|material| material.clone())
|
||||
.map_err(|e| format!("test OIDC extra CA material lock poisoned: {e}"))
|
||||
}
|
||||
}
|
||||
});
|
||||
let http_client = ReqwestHttpClient::with_policy_and_extra_root_ca_provider(policy, provider);
|
||||
|
||||
let config_a = build_mocked_oidc_provider_config("a", &format!("{base_a}/application/o/rustfs-a"));
|
||||
let state_a = OidcSys::discover_provider(&config_a, &http_client)
|
||||
.await
|
||||
.expect("OIDC discovery should trust initial extra root CA");
|
||||
assert_eq!(state_a.metadata.issuer().to_string(), format!("{base_a}/application/o/rustfs-a"));
|
||||
|
||||
{
|
||||
let mut material = material
|
||||
.lock()
|
||||
.expect("test OIDC extra CA material lock should not be poisoned");
|
||||
material.generation = 2;
|
||||
material.root_ca_pem = Some(ca_pem_b.into_bytes());
|
||||
}
|
||||
let config_b = build_mocked_oidc_provider_config("b", &format!("{base_b}/application/o/rustfs-b"));
|
||||
let state_b = OidcSys::discover_provider(&config_b, &http_client)
|
||||
.await
|
||||
.expect("OIDC discovery should refresh extra root CA after generation change");
|
||||
|
||||
assert_eq!(state_b.metadata.issuer().to_string(), format!("{base_b}/application/o/rustfs-b"));
|
||||
assert!(handle_a.join().is_ok());
|
||||
assert!(handle_b.join().is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_validate_oidc_provider_config_retries_with_issuer_candidates() {
|
||||
// Discovery document must advertise the canonical issuer path. The first candidate has no
|
||||
@@ -2945,7 +3316,7 @@ mod tests {
|
||||
// Cloud metadata endpoint is never allowed.
|
||||
assert!(
|
||||
matches!(
|
||||
build_oidc_http_client("http://169.254.169.254/latest/meta-data/", None, None),
|
||||
build_oidc_http_client("http://169.254.169.254/latest/meta-data/", None, &[], None),
|
||||
Err(OidcHttpError::ForbiddenOutbound(_))
|
||||
),
|
||||
"metadata endpoint must be rejected"
|
||||
@@ -2953,7 +3324,7 @@ mod tests {
|
||||
// Loopback is rejected by default (no allow-origins configured).
|
||||
assert!(
|
||||
matches!(
|
||||
build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", None, None),
|
||||
build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", None, &[], None),
|
||||
Err(OidcHttpError::ForbiddenOutbound(_))
|
||||
),
|
||||
"loopback must be rejected by default"
|
||||
@@ -2961,7 +3332,7 @@ mod tests {
|
||||
// A public hostname passes the up-front shape/host check; the resolved IP is still
|
||||
// re-classified at connection time by the pinned resolver.
|
||||
assert!(
|
||||
build_oidc_http_client("https://accounts.example.com/.well-known/openid-configuration", None, None).is_ok(),
|
||||
build_oidc_http_client("https://accounts.example.com/.well-known/openid-configuration", None, &[], None).is_ok(),
|
||||
"public https endpoint should build"
|
||||
);
|
||||
}
|
||||
@@ -2981,13 +3352,13 @@ mod tests {
|
||||
fn build_oidc_http_client_honors_explicit_allowlist_for_loopback() {
|
||||
let policy = OutboundPolicy::from_allowed_origins("http://127.0.0.1:8080").expect("origin should parse");
|
||||
assert!(
|
||||
build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", Some(&policy), None).is_ok(),
|
||||
build_oidc_http_client("http://127.0.0.1:8080/.well-known/openid-configuration", Some(&policy), &[], None).is_ok(),
|
||||
"explicitly allow-listed loopback origin should build"
|
||||
);
|
||||
// A metadata endpoint stays forbidden even when a loopback origin is allow-listed.
|
||||
assert!(
|
||||
matches!(
|
||||
build_oidc_http_client("http://169.254.169.254/", Some(&policy), None),
|
||||
build_oidc_http_client("http://169.254.169.254/", Some(&policy), &[], None),
|
||||
Err(OidcHttpError::ForbiddenOutbound(_))
|
||||
),
|
||||
"metadata endpoint stays forbidden despite an unrelated allow-list entry"
|
||||
@@ -3190,8 +3561,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn oidc_metadata_endpoint_rejection_does_not_offer_allowlist_bypass() {
|
||||
let error = build_oidc_http_client("http://169.254.169.254/latest/meta-data/", Some(&OutboundPolicy::default()), None)
|
||||
.expect_err("metadata endpoint must remain forbidden");
|
||||
let error =
|
||||
build_oidc_http_client("http://169.254.169.254/latest/meta-data/", Some(&OutboundPolicy::default()), &[], None)
|
||||
.expect_err("metadata endpoint must remain forbidden");
|
||||
let message = error.to_string();
|
||||
|
||||
assert!(message.contains("metadata endpoint"));
|
||||
|
||||
@@ -28,6 +28,14 @@ pub const INTERNODE_OPERATION_NS_SCANNER: &str = "ns_scanner";
|
||||
pub const INTERNODE_OPERATION_GRPC_READ_ALL: &str = "grpc_read_all";
|
||||
pub const INTERNODE_OPERATION_GRPC_WRITE_ALL: &str = "grpc_write_all";
|
||||
pub const INTERNODE_OPERATION_GRPC_READ_MULTIPLE: &str = "grpc_read_multiple";
|
||||
pub const INTERNODE_OPERATION_GRPC_READ_VERSION: &str = "grpc_read_version";
|
||||
pub const INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION: &str = "grpc_batch_read_version";
|
||||
pub const INTERNODE_OPERATION_GRPC_LOCK: &str = "grpc_lock";
|
||||
pub const INTERNODE_OPERATION_GRPC_UNLOCK: &str = "grpc_unlock";
|
||||
pub const INTERNODE_OPERATION_GRPC_LOCK_BATCH: &str = "grpc_lock_batch";
|
||||
pub const INTERNODE_OPERATION_GRPC_UNLOCK_BATCH: &str = "grpc_unlock_batch";
|
||||
pub const INTERNODE_OPERATION_GRPC_REFRESH: &str = "grpc_refresh";
|
||||
pub const INTERNODE_OPERATION_GRPC_FORCE_UNLOCK: &str = "grpc_force_unlock";
|
||||
pub const INTERNODE_OPERATION_GRPC_OTHER: &str = "grpc_other";
|
||||
pub const INTERNODE_TRANSPORT_BACKEND_TCP_HTTP: &str = "tcp-http";
|
||||
pub const INTERNODE_TRANSPORT_BACKEND_GRPC: &str = "grpc";
|
||||
@@ -78,6 +86,7 @@ const INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL: &str = "rustfs_system_network_inter
|
||||
const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
|
||||
const INTERNODE_REPLAY_CACHE_OVERFLOW_BY_OPERATION_TOTAL: &str =
|
||||
"rustfs_system_network_internode_replay_cache_overflow_by_operation_total";
|
||||
const INTERNODE_REPLAY_CACHE_RECORDS_TOTAL: &str = "rustfs_system_network_internode_replay_cache_records_total";
|
||||
const INTERNODE_REPLAY_CACHE_ENTRIES: &str = "rustfs_system_network_internode_replay_cache_entries";
|
||||
const INTERNODE_REPLAY_CACHE_CAPACITY: &str = "rustfs_system_network_internode_replay_cache_capacity";
|
||||
const INTERNODE_REPLAY_CACHE_EVICTIONS_TOTAL: &str = "rustfs_system_network_internode_replay_cache_evictions_total";
|
||||
@@ -157,6 +166,10 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
|
||||
name: INTERNODE_REPLAY_CACHE_OVERFLOW_BY_OPERATION_TOTAL,
|
||||
labels: SERVER_OPERATION_BACKEND_RPC_PATH_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_REPLAY_CACHE_RECORDS_TOTAL,
|
||||
labels: SERVER_OPERATION_BACKEND_RPC_PATH_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_REPLAY_CACHE_ENTRIES,
|
||||
labels: SERVER_LABELS,
|
||||
@@ -619,6 +632,22 @@ impl InternodeMetrics {
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
pub fn record_replay_cache_record_for_operation_and_backend_path(
|
||||
&self,
|
||||
operation: &'static str,
|
||||
backend: &'static str,
|
||||
rpc_path: &str,
|
||||
) {
|
||||
counter!(
|
||||
INTERNODE_REPLAY_CACHE_RECORDS_TOTAL,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
OPERATION_LABEL => operation,
|
||||
BACKEND_LABEL => backend,
|
||||
RPC_PATH_LABEL => rpc_path.to_owned()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
pub fn record_replay_cache_state(&self, entries: usize, capacity: usize) {
|
||||
let entries = usize_to_u64_saturating(entries);
|
||||
let capacity = usize_to_u64_saturating(capacity);
|
||||
@@ -964,7 +993,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn operation_metric_descriptors_include_backend_and_operation_labels() {
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 20);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 21);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[..6] {
|
||||
assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
}
|
||||
@@ -986,14 +1015,18 @@ mod tests {
|
||||
INTERNODE_OPERATION_METRICS[13].labels,
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL]
|
||||
);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[14..16] {
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[14].labels,
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL]
|
||||
);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[15..17] {
|
||||
assert_eq!(metric.labels, &[SERVER_LABEL]);
|
||||
}
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[16].labels, &[SERVER_LABEL, REASON_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[17].labels, &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[17].labels, &[SERVER_LABEL, REASON_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[18].labels, &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]);
|
||||
// Payload histogram + large-payload counter carry operation+backend labels.
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[18].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[19].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[20].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1004,6 +1037,14 @@ mod tests {
|
||||
assert_eq!(INTERNODE_OPERATION_WALK_DIR, "walk_dir");
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_READ_ALL, "grpc_read_all");
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_WRITE_ALL, "grpc_write_all");
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_READ_VERSION, "grpc_read_version");
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, "grpc_batch_read_version");
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_LOCK, "grpc_lock");
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_UNLOCK, "grpc_unlock");
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_LOCK_BATCH, "grpc_lock_batch");
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_UNLOCK_BATCH, "grpc_unlock_batch");
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_REFRESH, "grpc_refresh");
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_FORCE_UNLOCK, "grpc_force_unlock");
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_OTHER, "grpc_other");
|
||||
|
||||
assert_eq!(INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, "tcp-http");
|
||||
@@ -1048,26 +1089,30 @@ mod tests {
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[14].name,
|
||||
"rustfs_system_network_internode_replay_cache_entries"
|
||||
"rustfs_system_network_internode_replay_cache_records_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[15].name,
|
||||
"rustfs_system_network_internode_replay_cache_capacity"
|
||||
"rustfs_system_network_internode_replay_cache_entries"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[16].name,
|
||||
"rustfs_system_network_internode_replay_cache_evictions_total"
|
||||
"rustfs_system_network_internode_replay_cache_capacity"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[17].name,
|
||||
"rustfs_system_storage_erasure_write_quorum_failures_total"
|
||||
"rustfs_system_network_internode_replay_cache_evictions_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[18].name,
|
||||
"rustfs_system_network_internode_operation_payload_bytes"
|
||||
"rustfs_system_storage_erasure_write_quorum_failures_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[19].name,
|
||||
"rustfs_system_network_internode_operation_payload_bytes"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[20].name,
|
||||
"rustfs_system_network_internode_operation_large_payloads_total"
|
||||
);
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, "grpc_read_multiple");
|
||||
@@ -1091,6 +1136,10 @@ mod tests {
|
||||
INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL,
|
||||
"rustfs_system_network_internode_signature_v1_fallback_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_REPLAY_CACHE_RECORDS_TOTAL,
|
||||
"rustfs_system_network_internode_replay_cache_records_total"
|
||||
);
|
||||
assert_eq!(FAILURE_REASON_LABEL, "failure_reason");
|
||||
assert_eq!(RPC_PATH_LABEL, "rpc_path");
|
||||
assert_eq!(REASON_LABEL, "reason");
|
||||
@@ -1144,6 +1193,11 @@ mod tests {
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
"/node_service.NodeService/ReadAll",
|
||||
);
|
||||
metrics.record_replay_cache_record_for_operation_and_backend_path(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
"/node_service.NodeService/ReadVersion",
|
||||
);
|
||||
});
|
||||
|
||||
let snapshot = metrics.snapshot();
|
||||
@@ -1179,6 +1233,28 @@ mod tests {
|
||||
assert_eq!(labels.get(BACKEND_LABEL).map(String::as_str), Some(INTERNODE_TRANSPORT_BACKEND_GRPC));
|
||||
assert_eq!(labels.get(RPC_PATH_LABEL).map(String::as_str), Some("/node_service.NodeService/ReadAll"));
|
||||
assert!(labels.get(SERVER_LABEL).is_some_and(|value| !value.is_empty()));
|
||||
|
||||
let records: Vec<_> = entries
|
||||
.iter()
|
||||
.filter(|(composite, _, _, _)| composite.key().name() == INTERNODE_REPLAY_CACHE_RECORDS_TOTAL)
|
||||
.collect();
|
||||
assert_eq!(records.len(), 1);
|
||||
let labels: HashMap<_, _> = records[0]
|
||||
.0
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key().to_string(), label.value().to_string()))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
labels.get(OPERATION_LABEL).map(String::as_str),
|
||||
Some(INTERNODE_OPERATION_GRPC_READ_VERSION)
|
||||
);
|
||||
assert_eq!(labels.get(BACKEND_LABEL).map(String::as_str), Some(INTERNODE_TRANSPORT_BACKEND_GRPC));
|
||||
assert_eq!(
|
||||
labels.get(RPC_PATH_LABEL).map(String::as_str),
|
||||
Some("/node_service.NodeService/ReadVersion")
|
||||
);
|
||||
assert!(labels.get(SERVER_LABEL).is_some_and(|value| !value.is_empty()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -294,6 +294,18 @@ pub const GET_OBJECT_SIZE_BUCKET_LE_256_KIB: &str = "le_256kib";
|
||||
pub const GET_OBJECT_SIZE_BUCKET_LE_512_KIB: &str = "le_512kib";
|
||||
pub const GET_OBJECT_SIZE_BUCKET_LE_1_MIB: &str = "le_1mib";
|
||||
pub const GET_OBJECT_SIZE_BUCKET_GT_1_MIB: &str = "gt_1mib";
|
||||
pub const GET_OBJECT_SIZE_BUCKET_UNKNOWN: &str = "unknown";
|
||||
|
||||
pub struct GetObjectStreamingBodyFailure {
|
||||
pub stage: &'static str,
|
||||
pub reason: &'static str,
|
||||
pub error_class: &'static str,
|
||||
pub strategy: &'static str,
|
||||
pub buffer_source: &'static str,
|
||||
pub size_bucket: &'static str,
|
||||
pub emitted_bytes: usize,
|
||||
pub remaining_bytes: usize,
|
||||
}
|
||||
|
||||
/// Return the bounded size bucket used by small-object GET diagnostics.
|
||||
#[inline(always)]
|
||||
@@ -592,6 +604,44 @@ pub fn record_get_object_reader_stream_poll(
|
||||
.record(duration_secs);
|
||||
}
|
||||
|
||||
/// Record a GET response body failure with bounded attribution labels.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_streaming_body_failure(failure: GetObjectStreamingBodyFailure) {
|
||||
if !metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
counter!(
|
||||
"rustfs_io_get_object_streaming_body_failure_total",
|
||||
"stage" => failure.stage,
|
||||
"reason" => failure.reason,
|
||||
"error_class" => failure.error_class,
|
||||
"strategy" => failure.strategy,
|
||||
"buffer_source" => failure.buffer_source,
|
||||
"size_bucket" => failure.size_bucket
|
||||
)
|
||||
.increment(1);
|
||||
histogram!(
|
||||
"rustfs_io_get_object_streaming_body_failure_emitted_bytes",
|
||||
"stage" => failure.stage,
|
||||
"reason" => failure.reason,
|
||||
"error_class" => failure.error_class,
|
||||
"strategy" => failure.strategy,
|
||||
"buffer_source" => failure.buffer_source,
|
||||
"size_bucket" => failure.size_bucket
|
||||
)
|
||||
.record(usize_to_f64(failure.emitted_bytes));
|
||||
histogram!(
|
||||
"rustfs_io_get_object_streaming_body_failure_remaining_bytes",
|
||||
"stage" => failure.stage,
|
||||
"reason" => failure.reason,
|
||||
"error_class" => failure.error_class,
|
||||
"strategy" => failure.strategy,
|
||||
"buffer_source" => failure.buffer_source,
|
||||
"size_bucket" => failure.size_bucket
|
||||
)
|
||||
.record(usize_to_f64(failure.remaining_bytes));
|
||||
}
|
||||
|
||||
/// Record a poll of the single-chunk in-memory GetObject handoff stream.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_memory_body_stream_poll(source: &'static str, outcome: &'static str, bytes: usize, duration_secs: f64) {
|
||||
@@ -745,8 +795,12 @@ pub fn record_get_object_metadata_cache_decision(path: &'static str, decision: &
|
||||
}
|
||||
|
||||
/// Record aggregate metadata fanout shape for one GetObject metadata read.
|
||||
///
|
||||
/// The legacy `metadata_fanout_error_responses` series records every non-valid
|
||||
/// response, including not-found and ignored outcomes. Use
|
||||
/// `metadata_response_total` outcome labels for failure attribution.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_metadata_fanout_shape(path: &'static str, total: usize, valid: usize, ignored: usize, errors: usize) {
|
||||
pub fn record_get_object_metadata_fanout_shape(path: &'static str, total: usize, valid: usize, ignored: usize, non_valid: usize) {
|
||||
if !get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
@@ -757,7 +811,7 @@ pub fn record_get_object_metadata_fanout_shape(path: &'static str, total: usize,
|
||||
histogram!("rustfs_io_get_object_metadata_fanout_ignored_responses", "path" => path)
|
||||
.record(metadata_fanout_count_to_f64(ignored));
|
||||
histogram!("rustfs_io_get_object_metadata_fanout_error_responses", "path" => path)
|
||||
.record(metadata_fanout_count_to_f64(errors));
|
||||
.record(metadata_fanout_count_to_f64(non_valid));
|
||||
}
|
||||
|
||||
/// Record a guarded metadata early-stop hit for GetObject.
|
||||
@@ -2914,6 +2968,16 @@ mod tests {
|
||||
record_list_objects(50.0, 100, false);
|
||||
record_error("get_object", "timeout");
|
||||
record_cpu_usage(25.5);
|
||||
record_get_object_streaming_body_failure(GetObjectStreamingBodyFailure {
|
||||
stage: "reader_stream",
|
||||
reason: "short_eof",
|
||||
error_class: "short_eof",
|
||||
strategy: "standard",
|
||||
buffer_source: "selected",
|
||||
size_bucket: GET_OBJECT_SIZE_BUCKET_GT_1_MIB,
|
||||
emitted_bytes: 1024,
|
||||
remaining_bytes: 512,
|
||||
});
|
||||
|
||||
// Enabled: the same recorders run their emission bodies without panicking.
|
||||
set_metrics_enabled(true);
|
||||
@@ -2922,6 +2986,16 @@ mod tests {
|
||||
record_list_objects(50.0, 100, false);
|
||||
record_error("get_object", "timeout");
|
||||
record_cpu_usage(25.5);
|
||||
record_get_object_streaming_body_failure(GetObjectStreamingBodyFailure {
|
||||
stage: "reader_stream",
|
||||
reason: "reader_error",
|
||||
error_class: "timeout",
|
||||
strategy: "standard",
|
||||
buffer_source: "selected",
|
||||
size_bucket: GET_OBJECT_SIZE_BUCKET_GT_1_MIB,
|
||||
emitted_bytes: 2048,
|
||||
remaining_bytes: 256,
|
||||
});
|
||||
|
||||
set_metrics_enabled(false);
|
||||
}
|
||||
|
||||
@@ -62,7 +62,7 @@ moka = { workspace = true, features = ["future"] }
|
||||
# Additional dependencies
|
||||
md-5 = { workspace = true }
|
||||
arc-swap = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["http"] }
|
||||
rustfs-security-governance = { workspace = true }
|
||||
# `EventName` for KMS audit records. A leaf crate with no rustfs dependencies,
|
||||
# so the audit sink can live outside this crate without a second, drifting
|
||||
|
||||
@@ -2758,10 +2758,14 @@ mod tests {
|
||||
use crate::config::{BackendConfig, KmsConfig};
|
||||
use crate::types::{CancelKeyDeletionRequest, CreateKeyRequest, DeleteKeyRequest, KeyStatus, KeyUsage};
|
||||
|
||||
// A dev Vault speaks plain HTTP, which validate() refuses unless
|
||||
// development mode is declared on the config itself — the env override
|
||||
// is applied by the config loaders, not by Default::default().
|
||||
let kms_config = KmsConfig {
|
||||
backend_config: BackendConfig::VaultKv2(Box::new(integration_vault_config())),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
.with_insecure_development_defaults();
|
||||
let backend = VaultKmsBackend::new(kms_config).await.expect("backend");
|
||||
|
||||
let key_id = format!("cancel-persist-{}", uuid::Uuid::new_v4());
|
||||
|
||||
+12
-19
@@ -27,6 +27,10 @@ use base64::Engine;
|
||||
use jiff::Zoned;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rand::random;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_CONTEXT_HEADER, INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, INTERNAL_ENCRYPTION_TAG_HEADER,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
||||
@@ -81,17 +85,6 @@ fn request_encryption_context(context: &ObjectEncryptionContext) -> HashMap<Stri
|
||||
enc_context
|
||||
}
|
||||
|
||||
const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id";
|
||||
|
||||
/// Carries the AEAD algorithm the object was sealed with.
|
||||
///
|
||||
/// The S3 `x-amz-server-side-encryption` header records the *SSE mode*
|
||||
/// (`AES256` / `aws:kms`), not the cipher, so it cannot round-trip
|
||||
/// `ChaCha20Poly1305`. Without this header a ChaCha-sealed object comes back
|
||||
/// from the projection claiming `aws:kms` and is then opened with the wrong
|
||||
/// cipher.
|
||||
const INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "x-rustfs-encryption-algorithm";
|
||||
|
||||
/// Result of object encryption
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EncryptionResult {
|
||||
@@ -807,19 +800,19 @@ impl ObjectEncryptionService {
|
||||
|
||||
// Internal headers for decryption
|
||||
headers.insert(
|
||||
"x-rustfs-encryption-iv".to_string(),
|
||||
INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
|
||||
base64::engine::general_purpose::STANDARD.encode(&metadata.iv),
|
||||
);
|
||||
|
||||
if let Some(ref tag) = metadata.tag {
|
||||
headers.insert(
|
||||
"x-rustfs-encryption-tag".to_string(),
|
||||
INTERNAL_ENCRYPTION_TAG_HEADER.to_string(),
|
||||
base64::engine::general_purpose::STANDARD.encode(tag),
|
||||
);
|
||||
}
|
||||
|
||||
headers.insert(
|
||||
"x-rustfs-encryption-key".to_string(),
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
|
||||
base64::engine::general_purpose::STANDARD.encode(&metadata.encrypted_data_key),
|
||||
);
|
||||
|
||||
@@ -831,7 +824,7 @@ impl ObjectEncryptionService {
|
||||
None => context_aad(&metadata.encryption_context).unwrap_or_default(),
|
||||
};
|
||||
headers.insert(
|
||||
"x-rustfs-encryption-context".to_string(),
|
||||
INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(),
|
||||
String::from_utf8_lossy(&context_bytes).into_owned(),
|
||||
);
|
||||
|
||||
@@ -876,13 +869,13 @@ impl ObjectEncryptionService {
|
||||
};
|
||||
|
||||
let iv = headers
|
||||
.get("x-rustfs-encryption-iv")
|
||||
.get(INTERNAL_ENCRYPTION_IV_HEADER)
|
||||
.ok_or_else(|| KmsError::validation_error("Missing IV header"))?;
|
||||
let iv = base64::engine::general_purpose::STANDARD
|
||||
.decode(iv)
|
||||
.map_err(|e| KmsError::validation_error(format!("Invalid IV: {e}")))?;
|
||||
|
||||
let tag = if let Some(tag_str) = headers.get("x-rustfs-encryption-tag") {
|
||||
let tag = if let Some(tag_str) = headers.get(INTERNAL_ENCRYPTION_TAG_HEADER) {
|
||||
Some(
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(tag_str)
|
||||
@@ -892,7 +885,7 @@ impl ObjectEncryptionService {
|
||||
None
|
||||
};
|
||||
|
||||
let encrypted_data_key = if let Some(key_str) = headers.get("x-rustfs-encryption-key") {
|
||||
let encrypted_data_key = if let Some(key_str) = headers.get(INTERNAL_ENCRYPTION_KEY_HEADER) {
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(key_str)
|
||||
.map_err(|e| KmsError::validation_error(format!("Invalid encrypted key: {e}")))?
|
||||
@@ -904,7 +897,7 @@ impl ObjectEncryptionService {
|
||||
// callers that inspect the context, but the bytes are carried through
|
||||
// untouched: re-serializing the parsed map is exactly how the original
|
||||
// ordering — and with it the ability to open the object — was lost.
|
||||
let (encryption_context, context_aad) = match headers.get("x-rustfs-encryption-context") {
|
||||
let (encryption_context, context_aad) = match headers.get(INTERNAL_ENCRYPTION_CONTEXT_HEADER) {
|
||||
Some(context_str) => (
|
||||
serde_json::from_str(context_str)
|
||||
.map_err(|e| KmsError::validation_error(format!("Invalid encryption context: {e}")))?,
|
||||
|
||||
@@ -32,14 +32,14 @@ use std::collections::HashMap;
|
||||
use std::fmt::Debug;
|
||||
use std::future::Future;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use rustfs_kms::backends::BackendCapabilities;
|
||||
use rustfs_kms::{
|
||||
CreateKeyRequest, KeyUsage, KmsConfig, KmsError, KmsManager, KmsServiceManager, KmsServiceStatus, ObjectEncryptionService,
|
||||
Result,
|
||||
CreateKeyRequest, DeleteKeyRequest, KeyUsage, KmsConfig, KmsError, KmsManager, KmsServiceManager, KmsServiceStatus,
|
||||
ObjectEncryptionService, Result,
|
||||
};
|
||||
use tempfile::TempDir;
|
||||
|
||||
@@ -126,6 +126,12 @@ pub struct TestKms {
|
||||
manager: Arc<KmsServiceManager>,
|
||||
kind: BackendKind,
|
||||
config: KmsConfig,
|
||||
/// Ids of the keys [`TestKms::create_key`] created, so a run against a
|
||||
/// persistent Vault can remove them afterwards instead of accumulating
|
||||
/// `behavior-*` keys forever (rustfs/backlog#1774). Shared through an Arc
|
||||
/// because the harness instance is consumed by the spec while the cleanup
|
||||
/// runs after it.
|
||||
created_keys: Arc<Mutex<Vec<String>>>,
|
||||
/// Held for the harness lifetime so the local key directory outlives a
|
||||
/// simulated process restart.
|
||||
_dir: Option<TempDir>,
|
||||
@@ -147,6 +153,7 @@ impl TestKms {
|
||||
manager,
|
||||
kind: BackendKind::Local,
|
||||
config,
|
||||
created_keys: Arc::new(Mutex::new(Vec::new())),
|
||||
_dir: Some(dir),
|
||||
}
|
||||
}
|
||||
@@ -166,6 +173,7 @@ impl TestKms {
|
||||
manager,
|
||||
kind: BackendKind::VaultKv2,
|
||||
config,
|
||||
created_keys: Arc::new(Mutex::new(Vec::new())),
|
||||
_dir: None,
|
||||
}
|
||||
}
|
||||
@@ -180,6 +188,7 @@ impl TestKms {
|
||||
manager,
|
||||
kind: BackendKind::VaultTransit,
|
||||
config,
|
||||
created_keys: Arc::new(Mutex::new(Vec::new())),
|
||||
_dir: None,
|
||||
}
|
||||
}
|
||||
@@ -192,6 +201,7 @@ impl TestKms {
|
||||
manager,
|
||||
kind: BackendKind::Static,
|
||||
config,
|
||||
created_keys: Arc::new(Mutex::new(Vec::new())),
|
||||
_dir: None,
|
||||
}
|
||||
}
|
||||
@@ -253,8 +263,75 @@ impl TestKms {
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_key({name}) should succeed on {}: {error:?}", self.kind.name()));
|
||||
assert_eq!(response.key_id, name, "created key id must be the requested name");
|
||||
self.created_keys
|
||||
.lock()
|
||||
.expect("created-keys lock")
|
||||
.push(response.key_id.clone());
|
||||
response.key_id
|
||||
}
|
||||
|
||||
/// Handle to the ids [`Self::create_key`] recorded, for cleanup that runs
|
||||
/// after a spec consumed the harness instance.
|
||||
pub fn created_keys_handle(&self) -> Arc<Mutex<Vec<String>>> {
|
||||
Arc::clone(&self.created_keys)
|
||||
}
|
||||
|
||||
/// Remove this instance's recorded Vault keys; see [`cleanup_vault_keys`].
|
||||
pub async fn cleanup(&self) {
|
||||
cleanup_vault_keys(self.kind, &self.config, self.created_keys_handle()).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Best-effort removal of the Vault keys a harness instance created, so a
|
||||
/// persistent dev Vault does not accumulate `behavior-*` keys across runs
|
||||
/// (rustfs/backlog#1774). A no-op for the Local and Static backends, whose
|
||||
/// state dies with the per-test temp directory.
|
||||
///
|
||||
/// The deletion runs on a fresh manager over the same configuration — the
|
||||
/// case's own manager is consumed by the spec and may have been stopped by a
|
||||
/// restart scenario — with the immediate-deletion gate enabled on the cleanup
|
||||
/// configuration only, so the configuration under test keeps the gate at its
|
||||
/// production default and specs asserting the gate's refusal stay honest.
|
||||
///
|
||||
/// Failures are reported but never panic: cleanup runs after the spec's own
|
||||
/// assertions, and a Vault hiccup here must not turn a green behavior run red.
|
||||
pub async fn cleanup_vault_keys(kind: BackendKind, config: &KmsConfig, created_keys: Arc<Mutex<Vec<String>>>) {
|
||||
if !kind.is_vault() {
|
||||
return;
|
||||
}
|
||||
let key_ids: Vec<String> = created_keys.lock().expect("created-keys lock").drain(..).collect();
|
||||
if key_ids.is_empty() {
|
||||
return;
|
||||
}
|
||||
let config = config.clone().with_immediate_deletion_allowed();
|
||||
let manager = start_manager(&config).await;
|
||||
let kms = manager.get_manager().await.expect("KMS manager should be running");
|
||||
for key_id in key_ids {
|
||||
// The Transit backend deletes in two steps (first call parks the key in
|
||||
// PendingDeletion, the next call destroys it); KV2 destroys on the
|
||||
// first call and reports KeyNotFound on the second.
|
||||
for _ in 0..2 {
|
||||
match kms
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
pending_window_in_days: None,
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: Some(key_id.clone()),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => continue,
|
||||
Err(KmsError::KeyNotFound { .. }) => break,
|
||||
Err(error) => {
|
||||
eprintln!("vault key cleanup: could not delete {key_id}: {error:?}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Err(error) = manager.stop().await {
|
||||
eprintln!("vault key cleanup: could not stop the cleanup manager: {error:?}");
|
||||
}
|
||||
}
|
||||
|
||||
async fn start_manager(config: &KmsConfig) -> Arc<KmsServiceManager> {
|
||||
@@ -332,7 +409,12 @@ where
|
||||
.chain(live_vault_backends());
|
||||
for kind in kinds {
|
||||
let case = BackendCase::new(kind).await;
|
||||
// Captured before the spec consumes the case; keys the spec creates
|
||||
// through the harness afterwards still land in the shared list.
|
||||
let config = case.kms.config().clone();
|
||||
let created_keys = case.kms.created_keys_handle();
|
||||
spec(case).await;
|
||||
cleanup_vault_keys(kind, &config, created_keys).await;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ thiserror.workspace = true
|
||||
parking_lot.workspace = true
|
||||
rand.workspace = true
|
||||
smallvec = { workspace = true, features = ["serde"] }
|
||||
smartstring.workspace = true
|
||||
compact_str.workspace = true
|
||||
crossbeam-queue = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -40,6 +40,7 @@ const UNLOCK_RETRY_ATTEMPTS: usize = 3;
|
||||
const UNLOCK_RETRY_BACKOFF: Duration = Duration::from_millis(100);
|
||||
const LOCK_ACQUIRE_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
|
||||
const LOCK_ACQUIRE_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(1);
|
||||
const LOCK_ACQUIRE_SPARE_HEDGES: usize = 1;
|
||||
const REMOTE_LOCK_RPC_FAILED_PREFIX: &str = "remote lock rpc failed:";
|
||||
const REMOTE_LOCK_RPC_TIMED_OUT_PREFIX: &str = "remote lock rpc timed out:";
|
||||
const UNRECOVERABLE_QUORUM_FAILURE_PREFIX: &str = "unrecoverable quorum failure";
|
||||
@@ -478,7 +479,7 @@ pub struct DistributedLock {
|
||||
/// Lock clients for this namespace
|
||||
clients: Vec<Arc<dyn LockClient>>,
|
||||
/// Namespace identifier
|
||||
namespace: String,
|
||||
namespace: Arc<str>,
|
||||
/// Quorum size for exclusive/write operations
|
||||
quorum: usize,
|
||||
}
|
||||
@@ -495,6 +496,11 @@ struct LockAcquireQuorumResult {
|
||||
impl DistributedLock {
|
||||
/// Create new distributed lock
|
||||
pub fn new(namespace: String, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
|
||||
Self::new_shared(namespace.into(), clients, quorum)
|
||||
}
|
||||
|
||||
/// Create a distributed lock that shares an existing namespace allocation.
|
||||
pub(crate) fn new_shared(namespace: Arc<str>, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
|
||||
let q = if clients.len() <= 1 {
|
||||
1
|
||||
} else {
|
||||
@@ -672,13 +678,22 @@ impl DistributedLock {
|
||||
self.acquire_guard(&req).await
|
||||
}
|
||||
|
||||
fn spawn_lock_requests(&self, request: &LockRequest) -> JoinSet<LockAcquireTaskResult> {
|
||||
let mut pending = JoinSet::new();
|
||||
for (idx, client) in self.clients.iter().cloned().enumerate() {
|
||||
fn spawn_lock_requests_until(
|
||||
&self,
|
||||
pending: &mut JoinSet<LockAcquireTaskResult>,
|
||||
request: &LockRequest,
|
||||
next_client_index: &mut usize,
|
||||
target_pending: usize,
|
||||
) {
|
||||
while pending.len() < target_pending {
|
||||
let Some(client) = self.clients.get(*next_client_index).cloned() else {
|
||||
break;
|
||||
};
|
||||
let idx = *next_client_index;
|
||||
*next_client_index = (*next_client_index).saturating_add(1);
|
||||
let request = request.clone();
|
||||
pending.spawn(async move { (idx, client.acquire_lock(&request).await) });
|
||||
}
|
||||
pending
|
||||
}
|
||||
|
||||
fn lock_acquire_retry_backoff(attempt: usize, rng: &mut impl rand::Rng) -> Duration {
|
||||
@@ -911,7 +926,16 @@ impl DistributedLock {
|
||||
async fn acquire_lock_quorum_once(&self, request: &LockRequest) -> Result<LockAcquireQuorumResult> {
|
||||
let required_quorum = self.required_quorum(request.lock_type);
|
||||
let attempt_started = Instant::now();
|
||||
let mut pending = self.spawn_lock_requests(request);
|
||||
let mut pending = JoinSet::new();
|
||||
let mut next_client_index = 0usize;
|
||||
self.spawn_lock_requests_until(
|
||||
&mut pending,
|
||||
request,
|
||||
&mut next_client_index,
|
||||
required_quorum
|
||||
.saturating_add(LOCK_ACQUIRE_SPARE_HEDGES)
|
||||
.min(self.clients.len()),
|
||||
);
|
||||
let mut individual_locks: Vec<HeldLockEntry> = Vec::new();
|
||||
let fallback_lock_id = request.lock_id.clone();
|
||||
let mut last_failure = None;
|
||||
@@ -1085,7 +1109,15 @@ impl DistributedLock {
|
||||
});
|
||||
}
|
||||
|
||||
if individual_locks.len() + pending.len() < required_quorum {
|
||||
let needed = required_quorum.saturating_sub(individual_locks.len());
|
||||
self.spawn_lock_requests_until(
|
||||
&mut pending,
|
||||
request,
|
||||
&mut next_client_index,
|
||||
needed.saturating_add(LOCK_ACQUIRE_SPARE_HEDGES),
|
||||
);
|
||||
let unstarted_clients = self.clients.len().saturating_sub(next_client_index);
|
||||
if individual_locks.len() + pending.len() + unstarted_clients < required_quorum {
|
||||
let rollback_count = individual_locks.len();
|
||||
Self::spawn_release_cleanup(
|
||||
individual_locks.iter().map(HeldLockEntry::release_entry).collect(),
|
||||
@@ -1907,6 +1939,99 @@ mod tests {
|
||||
assert!(!should_warn_lock_failure("Lock acquisition timeout"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shared_lock_acquire_starts_quorum_plus_one_clients() {
|
||||
let seen_ids = (0..4).map(|_| Arc::new(Mutex::new(Vec::new()))).collect::<Vec<_>>();
|
||||
let clients: Vec<Arc<dyn LockClient>> = vec![
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Success {
|
||||
delay: Duration::from_millis(20),
|
||||
}],
|
||||
seen_ids[0].clone(),
|
||||
)),
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Success {
|
||||
delay: Duration::from_millis(20),
|
||||
}],
|
||||
seen_ids[1].clone(),
|
||||
)),
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Success {
|
||||
delay: Duration::from_millis(200),
|
||||
}],
|
||||
seen_ids[2].clone(),
|
||||
)),
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Success { delay: Duration::ZERO }],
|
||||
seen_ids[3].clone(),
|
||||
)),
|
||||
];
|
||||
let lock = DistributedLock::new("test".to_string(), clients, 3);
|
||||
let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Shared, "owner")
|
||||
.with_acquire_timeout(Duration::from_secs(1));
|
||||
|
||||
let guard = lock
|
||||
.acquire_guard(&request)
|
||||
.await
|
||||
.expect("shared lock acquisition should not error")
|
||||
.expect("read quorum should be reached");
|
||||
|
||||
assert_eq!(guard.entries.len(), 2, "read quorum should track exactly two acquired entries");
|
||||
assert_eq!(
|
||||
seen_ids[3].lock().unwrap().len(),
|
||||
0usize,
|
||||
"fourth client should stay unstarted while one hedge is pending"
|
||||
);
|
||||
drop(guard);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shared_lock_acquire_replenishes_after_early_failure() {
|
||||
let seen_ids = (0..4).map(|_| Arc::new(Mutex::new(Vec::new()))).collect::<Vec<_>>();
|
||||
let clients: Vec<Arc<dyn LockClient>> = vec![
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Failure {
|
||||
error: "lock already held",
|
||||
delay: Duration::ZERO,
|
||||
}],
|
||||
seen_ids[0].clone(),
|
||||
)),
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Success {
|
||||
delay: Duration::from_millis(80),
|
||||
}],
|
||||
seen_ids[1].clone(),
|
||||
)),
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Success {
|
||||
delay: Duration::from_millis(120),
|
||||
}],
|
||||
seen_ids[2].clone(),
|
||||
)),
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Success { delay: Duration::ZERO }],
|
||||
seen_ids[3].clone(),
|
||||
)),
|
||||
];
|
||||
let lock = DistributedLock::new("test".to_string(), clients, 3);
|
||||
let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Shared, "owner")
|
||||
.with_acquire_timeout(Duration::from_secs(1));
|
||||
|
||||
let guard = lock
|
||||
.acquire_guard(&request)
|
||||
.await
|
||||
.expect("shared lock acquisition should not error")
|
||||
.expect("read quorum should be reached after replenishing the failed slot");
|
||||
|
||||
assert_eq!(guard.entries.len(), 2, "read quorum should be reached");
|
||||
assert_eq!(
|
||||
seen_ids[3].lock().unwrap().len(),
|
||||
1usize,
|
||||
"fourth client should be started after an early failure"
|
||||
);
|
||||
drop(guard);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acquire_guard_returns_quorum_error_when_rpc_failures_make_quorum_impossible() {
|
||||
let clients: Vec<Arc<dyn LockClient>> = vec![
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::fast_lock::guard::FastLockGuard;
|
||||
use compact_str::CompactString;
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use smartstring::SmartString;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
@@ -143,15 +143,15 @@ impl ObjectKey {
|
||||
}
|
||||
}
|
||||
|
||||
/// Optimized object key using smart strings for better performance
|
||||
/// Optimized object key using compact strings for better performance
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OptimizedObjectKey {
|
||||
/// Bucket name - uses inline storage for small strings
|
||||
pub bucket: SmartString<smartstring::LazyCompact>,
|
||||
pub bucket: CompactString,
|
||||
/// Object name - uses inline storage for small strings
|
||||
pub object: SmartString<smartstring::LazyCompact>,
|
||||
pub object: CompactString,
|
||||
/// Version - optional for latest version semantics
|
||||
pub version: Option<SmartString<smartstring::LazyCompact>>,
|
||||
pub version: Option<CompactString>,
|
||||
/// Cached hash to avoid recomputation
|
||||
hash_cache: OnceLock<u64>,
|
||||
}
|
||||
@@ -189,10 +189,7 @@ impl Ord for OptimizedObjectKey {
|
||||
}
|
||||
|
||||
impl OptimizedObjectKey {
|
||||
pub fn new(
|
||||
bucket: impl Into<SmartString<smartstring::LazyCompact>>,
|
||||
object: impl Into<SmartString<smartstring::LazyCompact>>,
|
||||
) -> Self {
|
||||
pub fn new(bucket: impl Into<CompactString>, object: impl Into<CompactString>) -> Self {
|
||||
Self {
|
||||
bucket: bucket.into(),
|
||||
object: object.into(),
|
||||
@@ -202,9 +199,9 @@ impl OptimizedObjectKey {
|
||||
}
|
||||
|
||||
pub fn with_version(
|
||||
bucket: impl Into<SmartString<smartstring::LazyCompact>>,
|
||||
object: impl Into<SmartString<smartstring::LazyCompact>>,
|
||||
version: impl Into<SmartString<smartstring::LazyCompact>>,
|
||||
bucket: impl Into<CompactString>,
|
||||
object: impl Into<CompactString>,
|
||||
version: impl Into<CompactString>,
|
||||
) -> Self {
|
||||
Self {
|
||||
bucket: bucket.into(),
|
||||
@@ -232,9 +229,9 @@ impl OptimizedObjectKey {
|
||||
/// Convert from regular ObjectKey
|
||||
pub fn from_object_key(key: &ObjectKey) -> Self {
|
||||
Self {
|
||||
bucket: SmartString::from(key.bucket.as_ref()),
|
||||
object: SmartString::from(key.object.as_ref()),
|
||||
version: key.version.as_ref().map(|v| SmartString::from(v.as_ref())),
|
||||
bucket: CompactString::from(key.bucket.as_ref()),
|
||||
object: CompactString::from(key.object.as_ref()),
|
||||
version: key.version.as_ref().map(|v| CompactString::from(v.as_ref())),
|
||||
hash_cache: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,12 +28,17 @@ pub struct LocalLock {
|
||||
/// Global lock manager for fast local locks
|
||||
manager: Arc<GlobalLockManager>,
|
||||
/// Namespace identifier
|
||||
namespace: String,
|
||||
namespace: Arc<str>,
|
||||
}
|
||||
|
||||
impl LocalLock {
|
||||
/// Create new local lock
|
||||
pub fn new(namespace: String, manager: Arc<GlobalLockManager>) -> Self {
|
||||
Self::new_shared(namespace.into(), manager)
|
||||
}
|
||||
|
||||
/// Create a local lock that shares an existing namespace allocation.
|
||||
pub(crate) fn new_shared(namespace: Arc<str>, manager: Arc<GlobalLockManager>) -> Self {
|
||||
Self { namespace, manager }
|
||||
}
|
||||
|
||||
|
||||
@@ -180,6 +180,11 @@ impl NamespaceLock {
|
||||
Self::Local(LocalLock::new(namespace, manager))
|
||||
}
|
||||
|
||||
/// Create a local namespace lock that shares an existing namespace allocation.
|
||||
pub fn with_local_manager_shared(namespace: Arc<str>, manager: Arc<crate::GlobalLockManager>) -> Self {
|
||||
Self::Local(LocalLock::new_shared(namespace, manager))
|
||||
}
|
||||
|
||||
/// Create namespace lock with clients
|
||||
/// Uses DistributedLock with appropriate quorum
|
||||
pub fn with_clients(namespace: String, clients: Vec<Arc<dyn LockClient>>) -> Self {
|
||||
@@ -195,6 +200,11 @@ impl NamespaceLock {
|
||||
Self::Distributed(DistributedLock::new(namespace, clients, quorum))
|
||||
}
|
||||
|
||||
/// Create a namespace lock that shares an existing namespace allocation.
|
||||
pub fn with_clients_and_quorum_shared(namespace: Arc<str>, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
|
||||
Self::Distributed(DistributedLock::new_shared(namespace, clients, quorum))
|
||||
}
|
||||
|
||||
/// Get namespace identifier
|
||||
pub fn namespace(&self) -> &str {
|
||||
match self {
|
||||
|
||||
@@ -356,6 +356,16 @@ async fn test_namespace_lock_with_local_manager() {
|
||||
assert_eq!(lock.namespace(), "local-ns");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_lock_preserves_shared_namespace_storage() {
|
||||
let namespace: Arc<str> = Arc::from("shared-namespace");
|
||||
let namespace_ptr = Arc::as_ptr(&namespace);
|
||||
let local = LocalLock::new_shared(namespace.clone(), Arc::new(GlobalLockManager::new()));
|
||||
|
||||
assert_eq!(local.namespace(), namespace.as_ref());
|
||||
assert_eq!(local.namespace().as_ptr(), namespace_ptr.cast::<u8>());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_with_clients() {
|
||||
let clients = vec![ClientFactory::create_local(), ClientFactory::create_local()];
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user