mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-14 00:53:14 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3f994c59eb | |||
| 7c1d9dec8f | |||
| 4a8759239d |
@@ -34,8 +34,7 @@ e2e-vault = { max-threads = 1 }
|
||||
|
||||
# Reliability / fault-injection e2e tests each spawn a single-node 4-disk RustFS
|
||||
# server and manipulate its disk directories at runtime (crates/e2e_test:
|
||||
# reliability_disk_fault_test, degraded_read_eof_regression_test / dist-13, and
|
||||
# replacement_privileged_e2e_test when explicitly run as root on Linux). They
|
||||
# reliability_disk_fault_test, degraded_read_eof_regression_test / dist-13). They
|
||||
# are correct in isolation but resource-heavy; serialize them under nextest's
|
||||
# process boundary (serial_test's #[serial] does not cross it) so several 4-disk
|
||||
# servers never run at once. ci-7's nightly picks these up via the e2e suite;
|
||||
@@ -91,7 +90,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
# e2e-reliability test-group note above). The matching ci-profile override is at
|
||||
# the end of the file, after [profile.ci] is declared.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
|
||||
test-group = 'e2e-reliability'
|
||||
|
||||
[[profile.default.overrides]]
|
||||
@@ -156,7 +155,7 @@ retries = 2
|
||||
# quarantine: no retries, just single-threaded so several 4-disk servers never
|
||||
# run concurrently when ci-7's nightly runs the full e2e suite.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
|
||||
test-group = 'e2e-reliability'
|
||||
|
||||
# Serialize the multipart crash-consistency scenarios under the ci profile too
|
||||
@@ -384,7 +383,7 @@ path = "junit.xml"
|
||||
# quarantine: no retries, just single-threaded so several 4-disk servers never
|
||||
# run concurrently.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression|replacement_privileged_e2e)_test::/)'
|
||||
filter = 'package(e2e_test) & test(/^(reliability_disk_fault|degraded_read_eof_regression)_test::/)'
|
||||
test-group = 'e2e-reliability'
|
||||
|
||||
[[profile.e2e-full.overrides]]
|
||||
|
||||
@@ -17,11 +17,9 @@
|
||||
# =============================================================================
|
||||
#
|
||||
# Metric source: the KMS operation-policy choke point in
|
||||
# crates/kms/src/policy.rs, except KmsKeyRotationOverdue, which reads the
|
||||
# label-less key-lifecycle gauge published by the deletion worker's sweep
|
||||
# (crates/kms/src/deletion_worker.rs). All label values are bounded static
|
||||
# strings (operation, op_class, outcome, error_class, backend, scope); key
|
||||
# identifiers, key material, and tokens never appear in labels.
|
||||
# crates/kms/src/policy.rs. All label values are bounded static strings
|
||||
# (operation, op_class, outcome, error_class, backend, scope); key identifiers,
|
||||
# key material, and tokens never appear in labels.
|
||||
#
|
||||
# Response procedures: docs/operations/kms-observability-runbook.md
|
||||
#
|
||||
@@ -214,38 +212,3 @@ groups:
|
||||
circuit_open until the half-open probe succeeds or returns
|
||||
a non-retryable failure.
|
||||
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendcircuitopen"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 7. KmsKeyRotationOverdue
|
||||
# The least recently rotated usable key has gone more than 400
|
||||
# days without a rotation (measured from creation for keys with
|
||||
# no recorded rotation). Direct gauge state published by the
|
||||
# deletion worker's sweep, so no traffic guard applies; the
|
||||
# one-hour hold only bridges scrape gaps. The worker runs only
|
||||
# on backends with the schedule_deletion capability, so on the
|
||||
# Static backend the series never exists and this alert cannot
|
||||
# fire — that backend cannot rotate either; see the rotation
|
||||
# driver matrix in docs/operations/kms-backend-security.md.
|
||||
# Threshold: 400 days — conservative default sitting above a
|
||||
# one-year rotation policy. Align it with the rotation period
|
||||
# your compliance policy requires, and with
|
||||
# RUSTFS_KMS_ROTATION_MAX_AGE_SECS so the per-key rotation_due
|
||||
# verdict and this aggregate alert agree.
|
||||
# ------------------------------------------------------------------
|
||||
- alert: KmsKeyRotationOverdue
|
||||
expr: |
|
||||
rustfs_kms_oldest_key_rotation_age_seconds > (400 * 86400)
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
component: kms
|
||||
annotations:
|
||||
summary: "Oldest KMS key unrotated for more than 400 days"
|
||||
description: >-
|
||||
The least recently rotated usable KMS key was last rotated
|
||||
{{ $value | humanizeDuration }} ago (measured from creation
|
||||
for keys with no recorded rotation). List keys through the
|
||||
admin API and read rotation_due / rotation_due_reason for
|
||||
the per-key verdict; an "unsupported" reason means the
|
||||
backend cannot rotate at all.
|
||||
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmskeyrotationoverdue"
|
||||
|
||||
@@ -55,142 +55,3 @@ jobs:
|
||||
|
||||
- name: Build RustFS
|
||||
run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins
|
||||
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
# Vault Transit backends to every for_each_backend spec in
|
||||
# crates/kms/tests/behavior_*.rs (see crates/kms/AGENTS.md). rotate and
|
||||
# versioning are advertised only by the Vault backends, so without this lane
|
||||
# no CI run ever asserts the working half of behavior_rotation.rs — a
|
||||
# rotation that silently dropped historical key versions would stay green.
|
||||
# The same lane runs the dev-Vault #[ignore] tests and the two self-hosting
|
||||
# live scripts (AppRole login, three-node Raft leader failover).
|
||||
#
|
||||
# GitHub-hosted ubuntu-latest, deliberately not the self-hosted sm-standard
|
||||
# fleet: the HA failover script needs a working Docker daemon, and the
|
||||
# self-hosted fleet is heterogeneous — a docker-dependent workflow has been
|
||||
# burned by it before (see the banner in e2e-s3tests.yml, rustfs/backlog#1149).
|
||||
kms-vault-lane:
|
||||
name: KMS live Vault lane
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
# Root token of the ephemeral loopback dev server. Not a secret: the
|
||||
# server lives only for this job, listens on 127.0.0.1, and holds only
|
||||
# keys the tests create. The literal value matters — the dev-Vault
|
||||
# #[ignore] fixtures in crates/kms/src/backends/vault.rs hardcode it.
|
||||
VAULT_LANE_TOKEN: dev-only-token
|
||||
VAULT_LANE_ADDR: http://127.0.0.1:8200
|
||||
# Keeps a runner-level proxy from swallowing the loopback dev-server
|
||||
# traffic (see crates/kms/AGENTS.md). Actions env keys are
|
||||
# case-insensitive, so only the uppercase form is set; reqwest reads
|
||||
# either casing.
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
steps:
|
||||
- name: Checkout main branch
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: main
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
# Dedicated key: rust-cache cannot tell runner images apart, so
|
||||
# sharing a key with an sm-standard lane would let two different
|
||||
# system images overwrite each other's artifacts (same reasoning as
|
||||
# ci.yml's ci-uring lane). Saved from this nightly job itself so the
|
||||
# next night starts warm.
|
||||
cache-shared-key: kms-vault-lane
|
||||
cache-save-if: 'true'
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Install Vault CLI
|
||||
run: |
|
||||
set -euo pipefail
|
||||
wget -qO- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list >/dev/null
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq vault
|
||||
vault version
|
||||
|
||||
- name: Start Vault dev server with KV2 and Transit engines
|
||||
run: |
|
||||
set -euo pipefail
|
||||
nohup vault server -dev \
|
||||
-dev-root-token-id="${VAULT_LANE_TOKEN}" \
|
||||
-dev-listen-address=127.0.0.1:8200 >/tmp/vault-dev.log 2>&1 &
|
||||
for _ in $(seq 1 60); do
|
||||
if curl -fsS "${VAULT_LANE_ADDR}/v1/sys/health" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
curl -fsS "${VAULT_LANE_ADDR}/v1/sys/health"
|
||||
export VAULT_ADDR="${VAULT_LANE_ADDR}" VAULT_TOKEN="${VAULT_LANE_TOKEN}"
|
||||
# Dev mode mounts KV v2 at secret/ by default; Transit is explicit.
|
||||
# Prove both engines actually work rather than assuming the defaults.
|
||||
vault secrets enable transit
|
||||
vault kv put secret/rustfs-ci-lane-probe value=ok >/dev/null
|
||||
vault kv get secret/rustfs-ci-lane-probe >/dev/null
|
||||
vault write -f transit/keys/rustfs-ci-lane-probe >/dev/null
|
||||
|
||||
- name: Run rustfs-kms suite with the Vault lane on
|
||||
env:
|
||||
RUSTFS_KMS_VAULT_TOKEN: ${{ env.VAULT_LANE_TOKEN }}
|
||||
RUSTFS_KMS_VAULT_ADDR: ${{ env.VAULT_LANE_ADDR }}
|
||||
run: cargo test -p rustfs-kms --locked
|
||||
|
||||
- name: Run dev-Vault ignored tests
|
||||
env:
|
||||
RUSTFS_KMS_VAULT_TOKEN: ${{ env.VAULT_LANE_TOKEN }}
|
||||
RUSTFS_KMS_VAULT_ADDR: ${{ env.VAULT_LANE_ADDR }}
|
||||
# Filters select the dev-Vault-only #[ignore] tests. The AWS #[ignore]
|
||||
# tests (backends::aws, service_manager) stay excluded — they need real
|
||||
# AWS credentials and create billable keys. The AppRole and HA #[ignore]
|
||||
# tests are excluded here because their own scripts below provision the
|
||||
# Vault topology they need.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cargo test -p rustfs-kms --locked --lib backends::contract_tests -- --ignored
|
||||
cargo test -p rustfs-kms --locked --lib backends::vault -- --ignored
|
||||
cargo test -p rustfs-kms --locked --test vault_fault_injection -- --ignored
|
||||
|
||||
- name: Run AppRole live checks (self-hosting ephemeral Vault)
|
||||
run: bash scripts/test/vault_approle_kms_live.sh
|
||||
|
||||
- name: Show Vault dev server log on failure
|
||||
if: failure()
|
||||
run: tail -n 200 /tmp/vault-dev.log || true
|
||||
|
||||
# Three-node Raft leader failover (crates/kms/tests/vault_ha_failover_live.rs,
|
||||
# first validated by rustfs/rustfs#5653). Its own job so an election-timing
|
||||
# flake cannot mask the main lane's verdict, and vice versa. The script
|
||||
# provisions and tears down its own Docker cluster.
|
||||
kms-vault-ha-failover:
|
||||
name: KMS Vault HA failover lane
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
steps:
|
||||
- name: Checkout main branch
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: main
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
cache-shared-key: kms-vault-lane
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Run HA leader failover live checks (three-node Raft cluster in Docker)
|
||||
run: bash scripts/test/vault_ha_kms_live.sh
|
||||
|
||||
+20
-64
@@ -1,6 +1,6 @@
|
||||
# ARCHITECTURE.md
|
||||
|
||||
> Last updated: 2026-08-12 · Revision: 3
|
||||
> Last updated: 2026-07-02 · Revision: 2
|
||||
>
|
||||
> This document describes the high-level architecture of RustFS.
|
||||
> If you want to familiarize yourself with the code base, you are in the right place!
|
||||
@@ -119,44 +119,19 @@ module split is tracked under `docs/architecture/`.
|
||||
|
||||
3. **Each type has exactly one definition.** Types shared across crates must be defined
|
||||
in one crate and re-exported or imported by others.
|
||||
- ⚠️ VIOLATED: `ReplicationStats` names three unrelated types
|
||||
(`crates/data-usage/src/data_usage.rs`,
|
||||
`crates/obs/src/metrics/collectors/replication.rs`,
|
||||
`crates/ecstore/src/bucket/replication/replication_state.rs`) — a naming
|
||||
collision, not copies; renaming is tracked in rustfs/backlog#1847.
|
||||
- `LastMinuteLatency` has two deliberately different implementations: the
|
||||
per-second bucketed accumulator in `crates/common/src/last_minute.rs` and
|
||||
the in-memory endpoint-health sample tracker in
|
||||
`crates/ecstore/src/bucket/bucket_target_sys.rs` (its doc comment explains
|
||||
why it stays local).
|
||||
- ✅ RESOLVED: `BackpressureConfig` and `DataUsageInfo` each have exactly one
|
||||
definition (`crates/io-core/src/backpressure.rs`,
|
||||
`crates/data-usage/src/data_usage.rs`). A zero-consumer
|
||||
`BackpressureSettings` copy lingers in `crates/io-metrics/src/config.rs`;
|
||||
its removal is tracked in rustfs/backlog#1833.
|
||||
- ⚠️ VIOLATED: `ReplicationStats` (4 copies), `LastMinuteLatency` (3 copies),
|
||||
`BackpressureConfig` (3 copies), `DataUsageInfo` (2 copies).
|
||||
|
||||
4. **ecstore does not know about HTTP or S3 protocol details.** It operates on
|
||||
storage-level abstractions (objects, buckets, disks, pools).
|
||||
- ⚠️ VIOLATED: 58 files under `crates/ecstore/src` reference `s3s`
|
||||
(`rg -l 's3s' crates/ecstore/src | wc -l`), `crates/ecstore/src/client/`
|
||||
is a ~9.4K-line embedded S3 HTTP client, and `crates/ecstore/Cargo.toml`
|
||||
depends on `s3s`, `http`, `hyper`/`hyper-util`/`hyper-rustls`, and
|
||||
`reqwest`. Target state: the engine's need to act as an S3 client
|
||||
(tiering, replication targets) is served by an extracted client crate,
|
||||
and ecstore holds no wire or DTO types.
|
||||
|
||||
5. **The `rustfs` binary crate is the only place that wires everything together.**
|
||||
Individual crates should be testable in isolation.
|
||||
|
||||
6. **Error types use `thiserror` with descriptive names** (e.g., `StorageError`,
|
||||
not bare `Error`).
|
||||
- ✅ RESOLVED (strategy): `snafu` is gone from source
|
||||
(`rg -l snafu crates/ rustfs/` is empty) and library code no longer uses
|
||||
`anyhow` (remaining hits are test code and the `e2e_test` crate; `heal`
|
||||
uses `thiserror`).
|
||||
- ⚠️ VIOLATED (naming): 6 crates still export a bare `pub enum Error`:
|
||||
`crypto`, `filemeta`, `heal`, `iam`, `policy`, and `replication`
|
||||
(`src/resync.rs`) — all `thiserror`-derived.
|
||||
- ⚠️ VIOLATED: 6 crates use `pub enum Error`; 2 crates use `snafu`;
|
||||
`heal` use `anyhow` in library code.
|
||||
|
||||
## Known Structural Issues
|
||||
|
||||
@@ -165,25 +140,13 @@ module split is tracked under `docs/architecture/`.
|
||||
|
||||
### Critical
|
||||
|
||||
- **scanner/data-usage duplicate `.usage-cache.bin` serialization types.** The
|
||||
original finding ("common/scanner code duplication, ~3K lines") is resolved:
|
||||
`scanner` imports the shared data-usage types from `rustfs-data-usage` (see
|
||||
the `pub use rustfs_data_usage::…` re-exports at the top of
|
||||
`crates/scanner/src/data_usage_define.rs`). What remains: `scanner` and
|
||||
`data-usage` each hold their own serialization types for the scanner cache
|
||||
file (`DataUsageCacheInfo`/`DataUsageEntryInfo` in
|
||||
`crates/scanner/src/data_usage_define.rs` vs
|
||||
`DataUsageCacheInfo`/`DataUsageEntry` in
|
||||
`crates/data-usage/src/data_usage.rs`); convergence is tracked in
|
||||
rustfs/backlog#1828.
|
||||
- **common/scanner code duplication (~3K lines).** `scanner` depends on `common`
|
||||
but maintains its own copies of `DataUsageInfo`, `LastMinuteLatency`, and related
|
||||
types instead of importing them.
|
||||
|
||||
- **ecstore is a monolith (265 files, ~288K lines — roughly half is inline
|
||||
`#[cfg(test)]` code).** Measured with
|
||||
`find crates/ecstore/src -name '*.rs' | xargs wc -l`. It contains disk
|
||||
management, bucket management, erasure coding, replication, lifecycle, RPC,
|
||||
and configuration — all in one crate. It should be decomposed along its
|
||||
existing subdirectories; the split plan lives in
|
||||
[docs/architecture/ecstore-module-split-plan.md](docs/architecture/ecstore-module-split-plan.md).
|
||||
- **ecstore is a monolith (87K lines, 163 files).** It contains disk management,
|
||||
bucket management, erasure coding, replication, lifecycle, RPC, and configuration
|
||||
— all in one crate. It should be decomposed along its existing subdirectories.
|
||||
|
||||
### High
|
||||
|
||||
@@ -191,26 +154,19 @@ module split is tracked under `docs/architecture/`.
|
||||
`common → filemeta/madmin` edges must stay removed so leaf/helper crates do
|
||||
not regain upward dependencies.
|
||||
|
||||
- **Three-layer backpressure/deadlock policy bridging** across io-core,
|
||||
concurrency, and `rustfs/src/storage`. The config types are no longer
|
||||
duplicated (`BackpressureConfig` and `DeadlockDetectorConfig` are each
|
||||
defined once, in io-core). Storage policies expose and consume explicit
|
||||
projections into the concurrency/io-core policy shapes, and workload
|
||||
- **Three-layer BackpressureConfig/DeadlockConfig duplication** across io-core,
|
||||
concurrency, and `rustfs/src/storage`. Storage policies now expose and consume
|
||||
explicit projections into the concurrency/io-core policy shapes, and workload
|
||||
admission snapshots are composed through provider registries; later work
|
||||
should use those bridges before deleting compatibility wrappers.
|
||||
|
||||
### Medium
|
||||
|
||||
- **Bare `Error` naming.** Error-handling strategy has converged on `thiserror`
|
||||
(no `snafu`, no `anyhow` in library code); the remaining inconsistency is the
|
||||
bare `pub enum Error` naming in the 6 crates listed under Invariant 6.
|
||||
- **Inconsistent error handling.** Three strategies (thiserror/snafu/anyhow) and
|
||||
mixed naming (bare `Error` vs descriptive names).
|
||||
|
||||
- **`common` is mostly parked domain code, not shared utilities.** Of its
|
||||
6,724 lines, ~83% is scanner/heal domain code stranded there to break
|
||||
dependency cycles (`metrics.rs`, ~4,810 lines of scanner-domain metrics;
|
||||
`heal_channel.rs`, ~776 lines of heal-domain channel types). The
|
||||
"common vs utils" naming ambiguity is secondary to moving that code to its
|
||||
domain owners.
|
||||
- **Ambiguous common vs utils boundary.** Both described as "utilities and data
|
||||
structures." Need clear ownership rules.
|
||||
|
||||
## Cross-Cutting Concerns
|
||||
|
||||
@@ -276,7 +232,7 @@ The binary (`main.rs`) boots in this order:
|
||||
|
||||
```
|
||||
┌─────────┐
|
||||
│ rustfs │ (binary + lib)
|
||||
│ rustfs │ (binary + lib, 75K lines)
|
||||
│ main │
|
||||
└────┬────┘
|
||||
│
|
||||
@@ -299,7 +255,7 @@ The binary (`main.rs`) boots in this order:
|
||||
│ │ │
|
||||
┌─────▼──────┐ ┌──────▼──────┐ ┌──────▼──────┐
|
||||
│ ecstore │ │ rio │ │ io-core │
|
||||
│ (core) │ │ (readers) │ │ (zero-copy) │
|
||||
│ (87K,core) │ │ (readers) │ │ (zero-copy) │
|
||||
└─────┬──────┘ └─────────────┘ └─────────────┘
|
||||
│
|
||||
┌─────┬──┼──┬─────┬──────┐
|
||||
|
||||
Generated
+100
-151
@@ -104,12 +104,6 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aliasable"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd"
|
||||
|
||||
[[package]]
|
||||
name = "aligned-vec"
|
||||
version = "0.6.4"
|
||||
@@ -272,24 +266,24 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||
|
||||
[[package]]
|
||||
name = "apache-avro"
|
||||
version = "0.22.0"
|
||||
version = "0.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c"
|
||||
checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf"
|
||||
dependencies = [
|
||||
"bigdecimal",
|
||||
"bon",
|
||||
"digest 0.11.3",
|
||||
"digest 0.10.7",
|
||||
"log",
|
||||
"miniz_oxide 0.9.1",
|
||||
"miniz_oxide",
|
||||
"num-bigint 0.4.8",
|
||||
"ouroboros",
|
||||
"quad-rand",
|
||||
"rand 0.10.2",
|
||||
"rand 0.9.5",
|
||||
"regex-lite",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"serde_json",
|
||||
"strum",
|
||||
"strum 0.27.2",
|
||||
"strum_macros 0.27.2",
|
||||
"thiserror 2.0.20",
|
||||
"uuid",
|
||||
]
|
||||
@@ -1464,7 +1458,7 @@ dependencies = [
|
||||
"addr2line",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"miniz_oxide 0.8.9",
|
||||
"miniz_oxide",
|
||||
"object 0.37.3",
|
||||
"rustc-demangle",
|
||||
"windows-link",
|
||||
@@ -1807,15 +1801,6 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
|
||||
|
||||
[[package]]
|
||||
name = "castaway"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a"
|
||||
dependencies = [
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cbc"
|
||||
version = "0.1.2"
|
||||
@@ -2009,7 +1994,7 @@ version = "4.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
@@ -2078,19 +2063,6 @@ dependencies = [
|
||||
"unicode-width 0.2.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "compact_str"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416"
|
||||
dependencies = [
|
||||
"castaway",
|
||||
"cfg-if",
|
||||
"itoa",
|
||||
"static_assertions",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "compression-codecs"
|
||||
version = "0.4.38"
|
||||
@@ -4190,7 +4162,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide 0.8.9",
|
||||
"miniz_oxide",
|
||||
"zlib-rs",
|
||||
]
|
||||
|
||||
@@ -4254,9 +4226,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.34"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3"
|
||||
checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
@@ -4269,9 +4241,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.34"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4"
|
||||
checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
@@ -4279,15 +4251,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.34"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
|
||||
checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.34"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432"
|
||||
checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
@@ -4296,9 +4268,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.34"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed"
|
||||
checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a"
|
||||
|
||||
[[package]]
|
||||
name = "futures-lite"
|
||||
@@ -4315,13 +4287,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.34"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
|
||||
checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.3",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4337,21 +4309,21 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.34"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d"
|
||||
checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.34"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
|
||||
checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.34"
|
||||
version = "0.3.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
|
||||
checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
@@ -4860,12 +4832,6 @@ dependencies = [
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
@@ -5025,9 +4991,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath"
|
||||
version = "0.23.2"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62e810bedda5a467ef5c9b5c8a20763fefebc89b63ef36f7ee44a143085204a2"
|
||||
checksum = "be80823867e0c9820c9237c38b21f9f4aa1ebb0db1f98ff25ac0b1d2c088a470"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -5059,9 +5025,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros"
|
||||
version = "0.23.2"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "01bdc59bfc1a9984bee2ff5da63b2f6fccbaa57cd9a4119d709524632bddf341"
|
||||
checksum = "61d1fb3ee80ae7b4743d29487665766ce5a1442e959521790e86317f89dcd5a3"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5070,15 +5036,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros-meta"
|
||||
version = "0.23.2"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9216e8a01abe1e1671c376dc8736fb1bf772d7a889538d25f9e1200120ced38"
|
||||
checksum = "feede71fa226b0b5d523e58e7b0a1462935c0b8a00584a6669f45d564086209d"
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-meta"
|
||||
version = "0.23.2"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f22a9d20435fb79511b19dae37b3607224cd98f342a410702d84657cc38fc72f"
|
||||
checksum = "424fe0a13105d3731f65237785f5b95c3e4b8bfae4a039d932f56192cd74afc0"
|
||||
dependencies = [
|
||||
"hotpath-macros-meta",
|
||||
]
|
||||
@@ -5454,9 +5420,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "io-uring"
|
||||
version = "0.7.14"
|
||||
version = "0.7.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6"
|
||||
checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"cfg-if",
|
||||
@@ -5930,18 +5896,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "liblzma"
|
||||
version = "0.4.8"
|
||||
version = "0.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2fe0a34ca854fd4f20c07f696fc8675aec78f87d88d29f5e10257a7490a1b2e1"
|
||||
checksum = "45aec2360b3933207e27908049d8e4df4e476b58180afb1e56b2a4fb72efe4ba"
|
||||
dependencies = [
|
||||
"liblzma-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "liblzma-sys"
|
||||
version = "0.4.8"
|
||||
version = "0.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a0dad045e4b1b7b170be4b60b54b780cafb4490165461bac7d1cf7b703f61d5f"
|
||||
checksum = "a046c7f353ba30f810545151e04f63545833803f5b86ee3ddf1517247fe560a5"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
@@ -6403,15 +6369,6 @@ dependencies = [
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "minlz"
|
||||
version = "1.2.3"
|
||||
@@ -6459,9 +6416,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "moka"
|
||||
version = "0.12.16"
|
||||
version = "0.12.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9"
|
||||
checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046"
|
||||
dependencies = [
|
||||
"async-lock",
|
||||
"crossbeam-channel",
|
||||
@@ -6506,7 +6463,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4db8a44120571277accfaa3f3d91e7d3989d601d817c2fc01a9391b86135666"
|
||||
dependencies = [
|
||||
"darling 0.23.0",
|
||||
"heck 0.5.0",
|
||||
"heck",
|
||||
"manyhow",
|
||||
"num-bigint 0.4.8",
|
||||
"proc-macro-crate",
|
||||
@@ -6790,9 +6747,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.47"
|
||||
version = "0.1.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
|
||||
checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
@@ -7211,30 +7168,6 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ouroboros"
|
||||
version = "0.18.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59"
|
||||
dependencies = [
|
||||
"aliasable",
|
||||
"ouroboros_macro",
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ouroboros_macro"
|
||||
version = "0.18.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0"
|
||||
dependencies = [
|
||||
"heck 0.4.1",
|
||||
"proc-macro2",
|
||||
"proc-macro2-diagnostics",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "outref"
|
||||
version = "0.5.2"
|
||||
@@ -7787,9 +7720,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.15.0"
|
||||
version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
||||
checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic-util"
|
||||
@@ -7970,19 +7903,6 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2-diagnostics"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
"version_check",
|
||||
"yansi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prometheus"
|
||||
version = "0.14.0"
|
||||
@@ -8042,7 +7962,7 @@ version = "0.13.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"heck",
|
||||
"itertools 0.14.0",
|
||||
"log",
|
||||
"multimap",
|
||||
@@ -8062,7 +7982,7 @@ version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"heck",
|
||||
"itertools 0.14.0",
|
||||
"log",
|
||||
"multimap",
|
||||
@@ -8154,9 +8074,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pulldown-cmark-to-cmark"
|
||||
version = "22.0.1"
|
||||
version = "22.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab1ad36992cead65f02aa399a373a42730922f1525d988172634fdefdecb8a60"
|
||||
checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90"
|
||||
dependencies = [
|
||||
"pulldown-cmark",
|
||||
]
|
||||
@@ -8510,9 +8430,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rcgen"
|
||||
version = "0.14.9"
|
||||
version = "0.14.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "091e7a8e7d86e6feb87a27ce8e2cba29d49eff9507afeebefab7eeb2ca667fb4"
|
||||
checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"pem",
|
||||
@@ -8917,9 +8837,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "russh"
|
||||
version = "0.62.6"
|
||||
version = "0.62.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b41043523e0edcbd4e31d00903e26f12994f63b21bae9904f7405c1ed92752a5"
|
||||
checksum = "da7c230e0ed9cbeb92fbad6c8848985d6df2a1464c0dc247a021abd666e9005e"
|
||||
dependencies = [
|
||||
"aes 0.9.2",
|
||||
"aws-lc-rs",
|
||||
@@ -9345,6 +9265,7 @@ dependencies = [
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.0-rc.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
"rmp-serde",
|
||||
"rustfs-filemeta",
|
||||
@@ -9781,7 +9702,6 @@ name = "rustfs-lock"
|
||||
version = "1.0.0-rc.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"compact_str",
|
||||
"crossbeam-queue",
|
||||
"futures",
|
||||
"hotpath",
|
||||
@@ -9792,6 +9712,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
"smartstring",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tonic",
|
||||
@@ -9981,7 +9902,7 @@ dependencies = [
|
||||
"rustfs-crypto",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum",
|
||||
"strum 0.28.0",
|
||||
"temp-env",
|
||||
"test-case",
|
||||
"thiserror 2.0.20",
|
||||
@@ -10625,9 +10546,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.14"
|
||||
version = "0.103.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a"
|
||||
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
@@ -11007,9 +10928,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_with"
|
||||
version = "3.22.0"
|
||||
version = "3.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a"
|
||||
checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bs58",
|
||||
@@ -11017,7 +10938,6 @@ dependencies = [
|
||||
"hex",
|
||||
"indexmap 1.9.3",
|
||||
"indexmap 2.14.0",
|
||||
"jiff",
|
||||
"schemars 0.9.0",
|
||||
"schemars 1.2.2",
|
||||
"serde_core",
|
||||
@@ -11028,9 +10948,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_with_macros"
|
||||
version = "3.22.0"
|
||||
version = "3.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46"
|
||||
checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
|
||||
dependencies = [
|
||||
"darling 0.23.0",
|
||||
"proc-macro2",
|
||||
@@ -11324,6 +11244,17 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smartstring"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"static_assertions",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "snafu"
|
||||
version = "0.6.10"
|
||||
@@ -11570,13 +11501,31 @@ version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "strum"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
|
||||
|
||||
[[package]]
|
||||
name = "strum"
|
||||
version = "0.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd"
|
||||
dependencies = [
|
||||
"strum_macros",
|
||||
"strum_macros 0.28.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strum_macros"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11585,7 +11534,7 @@ version = "0.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -12858,9 +12807,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "whoami"
|
||||
version = "2.1.3"
|
||||
version = "2.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c"
|
||||
checksum = "998767ef88740d1f5b0682a9c53c24431453923962269c2db68ee43788c5a40d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"libredox",
|
||||
|
||||
+9
-9
@@ -142,10 +142,10 @@ async-recursion = "1.1.1"
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.50.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.34"
|
||||
futures-core = "0.3.34"
|
||||
futures = "0.3.33"
|
||||
futures-core = "0.3.33"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
futures-util = "0.3.33"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "6.8.0" }
|
||||
lapin = { default-features = false, version = "4.10.0" }
|
||||
@@ -171,7 +171,7 @@ tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.0" }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = "0.22.0"
|
||||
apache-avro = "0.21.0"
|
||||
bytes = { version = "1.12.1" }
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
@@ -268,7 +268,7 @@ lz4 = "1.28.1"
|
||||
matchit = "0.9.2"
|
||||
md-5 = "0.11.0"
|
||||
mime_guess = "2.0.5"
|
||||
moka = { version = "0.12.16" }
|
||||
moka = { version = "0.12.15" }
|
||||
netif = "0.1.6"
|
||||
num_cpus = { version = "1.17.0" }
|
||||
nvml-wrapper = "0.12.1"
|
||||
@@ -294,7 +294,7 @@ serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
smallvec = { version = "1.15.2" }
|
||||
compact_str = "0.10.0"
|
||||
smartstring = "1.0.1"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.2.2" }
|
||||
strum = { version = "0.28.0" }
|
||||
@@ -339,8 +339,8 @@ pyroscope = { version = "2.1.1" }
|
||||
libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.1" }
|
||||
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.6" }
|
||||
rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.5" }
|
||||
russh-sftp = "2.4.0"
|
||||
|
||||
# WebDAV
|
||||
@@ -349,7 +349,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "ce6338661179c8be22e516b00af7483f151485a7", features = ["extended"] }
|
||||
hotpath = { version = "0.23.2", default-features = false }
|
||||
hotpath = { version = "0.23.1", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@ pub mod heal_channel;
|
||||
pub mod last_minute;
|
||||
pub mod metrics;
|
||||
mod readiness;
|
||||
pub mod table_catalog;
|
||||
|
||||
pub use globals::*;
|
||||
pub use readiness::{GlobalReadiness, SystemStage};
|
||||
|
||||
@@ -915,13 +915,11 @@ const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1;
|
||||
const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
|
||||
const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3;
|
||||
const SCAN_CYCLE_RESULT_SUPERSEDED: u8 = 4;
|
||||
const SCAN_CYCLE_RESULT_DEFERRED: u8 = 5;
|
||||
const SCAN_CYCLE_RESULT_UNKNOWN_LABEL: &str = "unknown";
|
||||
const SCAN_CYCLE_RESULT_SUCCESS_LABEL: &str = "success";
|
||||
const SCAN_CYCLE_RESULT_ERROR_LABEL: &str = "error";
|
||||
const SCAN_CYCLE_RESULT_PARTIAL_LABEL: &str = "partial";
|
||||
const SCAN_CYCLE_RESULT_SUPERSEDED_LABEL: &str = "superseded";
|
||||
const SCAN_CYCLE_RESULT_DEFERRED_LABEL: &str = "deferred";
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum ScanCyclePartialReason {
|
||||
@@ -1426,7 +1424,6 @@ fn scan_cycle_result_label(result: u8) -> &'static str {
|
||||
SCAN_CYCLE_RESULT_ERROR => SCAN_CYCLE_RESULT_ERROR_LABEL,
|
||||
SCAN_CYCLE_RESULT_PARTIAL => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
|
||||
SCAN_CYCLE_RESULT_SUPERSEDED => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL,
|
||||
SCAN_CYCLE_RESULT_DEFERRED => SCAN_CYCLE_RESULT_DEFERRED_LABEL,
|
||||
_ => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
|
||||
}
|
||||
}
|
||||
@@ -1755,11 +1752,6 @@ pub fn emit_scan_cycle_superseded(duration: Duration) {
|
||||
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL).increment(1);
|
||||
}
|
||||
|
||||
pub fn emit_scan_cycle_deferred(duration: Duration) {
|
||||
global_metrics().record_scan_cycle_deferred(duration);
|
||||
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_DEFERRED_LABEL).increment(1);
|
||||
}
|
||||
|
||||
pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
|
||||
let result = if success { "success" } else { "error" };
|
||||
global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
|
||||
@@ -2557,17 +2549,6 @@ impl Metrics {
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_deferred(&self, duration: Duration) {
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.last_scan_cycle_result
|
||||
.store(SCAN_CYCLE_RESULT_DEFERRED, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason
|
||||
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) {
|
||||
self.record_scan_cycle_partial_with_source(duration, reason, None);
|
||||
}
|
||||
@@ -4283,21 +4264,6 @@ mod tests {
|
||||
assert_eq!(report.partial_cycles, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_tracks_deferred_cycle_without_failed_increment() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scan_cycle_deferred(Duration::from_millis(250));
|
||||
|
||||
let report = metrics.report().await;
|
||||
|
||||
assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_DEFERRED_LABEL);
|
||||
assert_eq!(report.last_cycle_result_code, u64::from(SCAN_CYCLE_RESULT_DEFERRED));
|
||||
assert_eq!(report.last_cycle_duration_seconds, 0.25);
|
||||
assert_eq!(report.failed_cycles, 0);
|
||||
assert_eq!(report.superseded_cycles, 0);
|
||||
assert_eq!(report.partial_cycles, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_tracks_successful_scan_cycle_without_failed_increment() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
/// Cross-crate lock identity used to fence table-bucket publication against
|
||||
/// object mutations that bypass the S3 request authorization layer.
|
||||
pub const TABLE_BUCKET_PUBLICATION_LOCK_PATH: &str = ".rustfs-table/warehouses/default/publication.lock";
|
||||
@@ -81,8 +81,6 @@ pub const ENV_TEST_IAM_FAIL_INIT_ATTEMPTS: &str = "RUSTFS_TEST_IAM_FAIL_INIT_ATT
|
||||
pub const ENV_TEST_IAM_RETRY_INTERVAL_MS: &str = "RUSTFS_TEST_IAM_RETRY_INTERVAL_MS";
|
||||
/// Runtime env var controlling the transition worker count.
|
||||
pub const ENV_TRANSITION_WORKERS: &str = "RUSTFS_MAX_TRANSITION_WORKERS";
|
||||
/// Runtime env var controlling the expiry worker count.
|
||||
pub const ENV_MAX_EXPIRY_WORKERS: &str = "RUSTFS_MAX_EXPIRY_WORKERS";
|
||||
/// Runtime env var controlling the absolute maximum transition workers.
|
||||
pub const ENV_TRANSITION_WORKERS_ABSOLUTE_MAX: &str = "RUSTFS_ABSOLUTE_MAX_WORKERS";
|
||||
/// Runtime env var controlling the transition queue capacity.
|
||||
|
||||
@@ -37,6 +37,7 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "rustfs-filemeta/hotpath-cpu"]
|
||||
hotpath.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
rmp-serde = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
|
||||
[lib]
|
||||
|
||||
@@ -846,15 +846,8 @@ impl DataUsageEntry {
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-only projection of the scanner's `.usage-cache.bin` info block.
|
||||
///
|
||||
/// The canonical wire format is written by the hand-written map-encoded
|
||||
/// `Serialize` on the scanner-side `DataUsageCacheInfo`
|
||||
/// (`crates/scanner/src/data_usage_define.rs`), which carries 16 fields.
|
||||
/// This type decodes only the shared subset and is deliberately not
|
||||
/// `Serialize`: a derived (array) encoding of this 6-field subset would
|
||||
/// corrupt the cache for scanner readers, so no write path may exist here.
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
/// Data usage cache info
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct DataUsageCacheInfo {
|
||||
pub name: String,
|
||||
pub next_cycle: u64,
|
||||
@@ -870,12 +863,8 @@ pub struct DataUsageCacheInfo {
|
||||
pub snapshot_complete: bool,
|
||||
}
|
||||
|
||||
/// Read-only projection of a scanner-written `.usage-cache.bin` file.
|
||||
///
|
||||
/// The scanner-side `DataUsageCache` (`crates/scanner/src/data_usage_define.rs`)
|
||||
/// owns the persisted format; this type only decodes it (see
|
||||
/// [`DataUsageCacheInfo`]) and must never grow a serialization path.
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
/// Data usage cache
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct DataUsageCache {
|
||||
pub info: DataUsageCacheInfo,
|
||||
pub cache: HashMap<String, DataUsageEntry>,
|
||||
@@ -1197,10 +1186,31 @@ impl DataUsageCache {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn marshal_msg(&self) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut buf = Vec::new();
|
||||
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
pub fn unmarshal(buf: &[u8]) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let t: Self = rmp_serde::from_slice(buf)?;
|
||||
Ok(t)
|
||||
}
|
||||
|
||||
// Note: load and save methods are storage-specific and should be implemented
|
||||
// in the ecstore crate where storage access is available
|
||||
}
|
||||
|
||||
/// Trait for storage-specific operations on DataUsageCache
|
||||
#[async_trait::async_trait]
|
||||
pub trait DataUsageCacheStorage {
|
||||
/// Load data usage cache from backend storage
|
||||
async fn load(store: &dyn std::any::Any, name: &str) -> Result<Self, Box<dyn std::error::Error + Send + Sync>>
|
||||
where
|
||||
Self: Sized;
|
||||
|
||||
/// Save data usage cache to backend storage
|
||||
async fn save(&self, name: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
}
|
||||
|
||||
// Helper structs and functions for cache operations
|
||||
@@ -1822,82 +1832,6 @@ mod tests {
|
||||
assert!(decoded.all_tier_stats.is_none());
|
||||
}
|
||||
|
||||
/// Scanner-written `.usage-cache.bin` bytes: a 2-element array of the
|
||||
/// canonical 16-field map-encoded info block and one map-encoded entry.
|
||||
/// Captured from the canonical writer's `marshal_msg` — see
|
||||
/// `usage_cache_wire_format_is_pinned` in
|
||||
/// `crates/scanner/src/data_usage_define.rs`, which pins these exact
|
||||
/// bytes and documents regeneration. Hardcoded here because a
|
||||
/// dev-dependency on rustfs-scanner would pull the whole ecstore tree
|
||||
/// into this crate's test build, and a fixture generated at test runtime
|
||||
/// could not detect writer drift anyway.
|
||||
const SCANNER_USAGE_CACHE_WIRE_FIXTURE: &[u8] = &[
|
||||
0x92, 0xde, 0x00, 0x10, 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
||||
0x74, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x07, 0xac, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72,
|
||||
0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x09, 0xab, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x92,
|
||||
0xce, 0x65, 0x53, 0xf1, 0x00, 0x00, 0xac, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0xc3,
|
||||
0xa9, 0x6c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0xc0, 0xab, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74,
|
||||
0x69, 0x6f, 0x6e, 0xc0, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x81,
|
||||
0xb0, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2f, 0x6c, 0x6f, 0x73, 0x74, 0x0b, 0xb1, 0x73,
|
||||
0x63, 0x61, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72, 0xb2, 0x77, 0x69, 0x72,
|
||||
0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0xaf, 0x73, 0x63, 0x61, 0x6e,
|
||||
0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0xc0, 0xad, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67,
|
||||
0x5f, 0x68, 0x65, 0x61, 0x6c, 0x73, 0x91, 0x9a, 0xa6, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0xab, 0x77, 0x69, 0x72, 0x65,
|
||||
0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0xa6, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x6e, 0xc0, 0x01, 0x64, 0xcc, 0xc8, 0x03,
|
||||
0xa8, 0x64, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0xa6, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0xab, 0x6f, 0x62, 0x6a,
|
||||
0x65, 0x63, 0x74, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0xc0, 0xa6, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x92, 0x01, 0x02, 0xb1,
|
||||
0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0xc3, 0xb0, 0x73,
|
||||
0x63, 0x61, 0x6e, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0xdc, 0x00, 0x20, 0x03, 0x03,
|
||||
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
|
||||
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0xb0, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6b, 0x65, 0x79,
|
||||
0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x01, 0x81, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
||||
0x74, 0x8b, 0xa8, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x90, 0xa4, 0x73, 0x69, 0x7a, 0x65, 0xcd, 0x10, 0x00,
|
||||
0xa7, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x03, 0xa8, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x05, 0xae,
|
||||
0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x01, 0xa9, 0x6f, 0x62, 0x6a, 0x5f,
|
||||
0x73, 0x69, 0x7a, 0x65, 0x73, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x6f, 0x62,
|
||||
0x6a, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x72,
|
||||
0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0xc0, 0xa9, 0x63, 0x6f,
|
||||
0x6d, 0x70, 0x61, 0x63, 0x74, 0x65, 0x64, 0xc3, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65,
|
||||
0x63, 0x74, 0x73, 0x02, 0xae, 0x61, 0x6c, 0x6c, 0x5f, 0x74, 0x69, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x91,
|
||||
0x81, 0xa4, 0x57, 0x41, 0x52, 0x4d, 0x93, 0xcd, 0x08, 0x00, 0x02, 0x01,
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn thin_usage_cache_decodes_scanner_wire_fixture() {
|
||||
let decoded =
|
||||
DataUsageCache::unmarshal(SCANNER_USAGE_CACHE_WIRE_FIXTURE).expect("thin projection decodes a scanner-written cache");
|
||||
|
||||
// The six fields shared with the scanner's 16-field info block; the
|
||||
// remaining ten (lifecycle, replication, checkpoint, heals, ...) must
|
||||
// be skipped, not error.
|
||||
assert_eq!(decoded.info.name, "wire-bucket");
|
||||
assert_eq!(decoded.info.next_cycle, 7);
|
||||
assert_eq!(
|
||||
decoded.info.last_update,
|
||||
Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000))
|
||||
);
|
||||
assert!(decoded.info.skip_healing);
|
||||
assert_eq!(decoded.info.failed_objects.get("wire-bucket/lost"), Some(&11));
|
||||
assert!(decoded.info.snapshot_complete);
|
||||
|
||||
// Entries use the shared canonical map-encoded type end to end.
|
||||
let entry = decoded.cache.get("wire-bucket").expect("fixture entry decodes");
|
||||
assert_eq!(entry.size, 4096);
|
||||
assert_eq!(entry.objects, 3);
|
||||
assert_eq!(entry.versions, 5);
|
||||
assert_eq!(entry.delete_markers, 1);
|
||||
assert!(entry.compacted);
|
||||
assert_eq!(entry.failed_objects, 2);
|
||||
assert_eq!(
|
||||
entry.all_tier_stats.as_ref().and_then(|tiers| tiers.tiers.get("WARM")),
|
||||
Some(&TierStats {
|
||||
total_size: 2048,
|
||||
num_versions: 2,
|
||||
num_objects: 1,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_path_uses_portable_slash_semantics() {
|
||||
for (input, expected) in [
|
||||
|
||||
+18
-102
@@ -40,8 +40,7 @@ use http::header::{CONTENT_TYPE, HOST};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::collections::BTreeSet;
|
||||
use std::error::Error;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::info;
|
||||
@@ -60,26 +59,13 @@ pub(crate) struct VersionShardCensus {
|
||||
pub version_id: Option<String>,
|
||||
pub has_xl_meta: bool,
|
||||
pub data_dir: Option<String>,
|
||||
pub erasure_index: Option<usize>,
|
||||
pub expected_part_numbers: BTreeSet<usize>,
|
||||
pub present_part_fingerprints: BTreeMap<usize, PartShardFingerprint>,
|
||||
pub inline_data_fingerprint: Option<PartShardFingerprint>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct PartShardFingerprint {
|
||||
pub size: u64,
|
||||
pub sha256: String,
|
||||
pub present_part_numbers: BTreeSet<usize>,
|
||||
}
|
||||
|
||||
impl VersionShardCensus {
|
||||
pub(crate) fn is_complete(&self) -> bool {
|
||||
self.has_xl_meta
|
||||
&& self.expected_part_numbers.len() == self.present_part_fingerprints.len()
|
||||
&& self
|
||||
.expected_part_numbers
|
||||
.iter()
|
||||
.all(|part_number| self.present_part_fingerprints.contains_key(part_number))
|
||||
self.has_xl_meta && self.expected_part_numbers == self.present_part_numbers
|
||||
}
|
||||
|
||||
pub(crate) fn matches_manifest(&self, manifest: &Self) -> bool {
|
||||
@@ -87,25 +73,10 @@ impl VersionShardCensus {
|
||||
&& self.is_complete()
|
||||
&& manifest.is_complete()
|
||||
&& self.data_dir == manifest.data_dir
|
||||
&& self.erasure_index == manifest.erasure_index
|
||||
&& self.expected_part_numbers == manifest.expected_part_numbers
|
||||
&& self.present_part_fingerprints == manifest.present_part_fingerprints
|
||||
&& self.inline_data_fingerprint == manifest.inline_data_fingerprint
|
||||
}
|
||||
}
|
||||
|
||||
fn sha256_hex(data: &[u8]) -> String {
|
||||
let digest = Sha256::digest(data);
|
||||
digest.iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
|
||||
fn shard_fingerprint(data: &[u8]) -> ChaosResult<PartShardFingerprint> {
|
||||
Ok(PartShardFingerprint {
|
||||
size: u64::try_from(data.len())?,
|
||||
sha256: sha256_hex(data),
|
||||
})
|
||||
}
|
||||
|
||||
/// Single-node RustFS server with `disk_count` local volume directories that
|
||||
/// can be faulted individually while the server is running.
|
||||
pub struct DiskFaultHarness {
|
||||
@@ -312,10 +283,8 @@ pub(crate) fn census_object_version_on_disk(
|
||||
version_id,
|
||||
has_xl_meta: false,
|
||||
data_dir: None,
|
||||
erasure_index: None,
|
||||
expected_part_numbers: BTreeSet::new(),
|
||||
present_part_fingerprints: BTreeMap::new(),
|
||||
inline_data_fingerprint: None,
|
||||
present_part_numbers: BTreeSet::new(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -327,31 +296,20 @@ pub(crate) fn census_object_version_on_disk(
|
||||
file_info.parts.iter().map(|part| part.number).collect()
|
||||
};
|
||||
let data_dir = file_info.data_dir.map(|id| id.to_string());
|
||||
let erasure_index = Some(file_info.erasure.index);
|
||||
let inline_data_fingerprint = file_info.data.as_deref().map(shard_fingerprint).transpose()?;
|
||||
let part_dir = data_dir.as_ref().map_or_else(|| object_dir.clone(), |id| object_dir.join(id));
|
||||
let present_part_fingerprints = match std::fs::read_dir(&part_dir) {
|
||||
Ok(entries) => {
|
||||
let mut fingerprints = BTreeMap::new();
|
||||
for entry in entries {
|
||||
let entry = entry?;
|
||||
if !entry.file_type()?.is_file() {
|
||||
continue;
|
||||
}
|
||||
let file_name = entry.file_name();
|
||||
let Some(part_number) = file_name
|
||||
.to_str()
|
||||
.and_then(|name| name.strip_prefix("part."))
|
||||
.and_then(|number| number.parse::<usize>().ok())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let data = std::fs::read(entry.path())?;
|
||||
fingerprints.insert(part_number, shard_fingerprint(&data)?);
|
||||
}
|
||||
fingerprints
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => BTreeMap::new(),
|
||||
let present_part_numbers = match std::fs::read_dir(&part_dir) {
|
||||
Ok(entries) => entries
|
||||
.filter_map(Result::ok)
|
||||
.filter_map(|entry| {
|
||||
entry
|
||||
.file_type()
|
||||
.ok()
|
||||
.filter(|kind| kind.is_file())
|
||||
.and_then(|_| entry.file_name().to_str().map(str::to_owned))
|
||||
})
|
||||
.filter_map(|name| name.strip_prefix("part.").and_then(|number| number.parse::<usize>().ok()))
|
||||
.collect(),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => BTreeSet::new(),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
|
||||
@@ -359,10 +317,8 @@ pub(crate) fn census_object_version_on_disk(
|
||||
version_id,
|
||||
has_xl_meta: true,
|
||||
data_dir,
|
||||
erasure_index,
|
||||
expected_part_numbers,
|
||||
present_part_fingerprints,
|
||||
inline_data_fingerprint,
|
||||
present_part_numbers,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -402,43 +358,3 @@ pub async fn signed_admin_post(url: &str, body: Option<&str>, access_key: &str,
|
||||
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn complete_census() -> VersionShardCensus {
|
||||
VersionShardCensus {
|
||||
version_id: Some("version".to_string()),
|
||||
has_xl_meta: true,
|
||||
data_dir: Some("data-dir".to_string()),
|
||||
erasure_index: Some(3),
|
||||
expected_part_numbers: BTreeSet::from([1]),
|
||||
present_part_fingerprints: BTreeMap::from([(1, shard_fingerprint(b"part").unwrap())]),
|
||||
inline_data_fingerprint: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shard_fingerprint_uses_physical_length_and_sha256() {
|
||||
assert_eq!(
|
||||
shard_fingerprint(b"abc").unwrap(),
|
||||
PartShardFingerprint {
|
||||
size: 3,
|
||||
sha256: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad".to_string(),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_requires_matching_inline_payload() {
|
||||
let mut expected = complete_census();
|
||||
expected.expected_part_numbers.clear();
|
||||
expected.present_part_fingerprints.clear();
|
||||
expected.inline_data_fingerprint = Some(shard_fingerprint(b"expected").unwrap());
|
||||
let mut changed = expected.clone();
|
||||
changed.inline_data_fingerprint = Some(shard_fingerprint(b"changed").unwrap());
|
||||
assert!(expected.matches_manifest(&expected));
|
||||
assert!(!changed.matches_manifest(&expected));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,16 +67,6 @@ fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
fn capture_command_logs(command: &mut Command, log_path: Option<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let Some(log_path) = log_path else {
|
||||
return Ok(());
|
||||
};
|
||||
let file = stdfs::OpenOptions::new().create(true).append(true).open(log_path)?;
|
||||
let stderr_file = file.try_clone()?;
|
||||
command.stdout(Stdio::from(file)).stderr(Stdio::from(stderr_file));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn build_test_s3_config(
|
||||
endpoint_url: &str,
|
||||
access_key: &str,
|
||||
@@ -567,7 +557,13 @@ impl RustFSTestEnvironment {
|
||||
for (key, value) in extra_env {
|
||||
command.env(key, value);
|
||||
}
|
||||
capture_command_logs(&mut command, self.capture_log_path.as_deref())?;
|
||||
// Optionally capture the child's stdout+stderr to a file so the test can
|
||||
// grep server logs (e.g. to confirm which GET reader path was taken).
|
||||
if let Some(log_path) = &self.capture_log_path {
|
||||
let file = stdfs::OpenOptions::new().create(true).append(true).open(log_path)?;
|
||||
let stderr_file = file.try_clone()?;
|
||||
command.stdout(Stdio::from(file)).stderr(Stdio::from(stderr_file));
|
||||
}
|
||||
let process = command.args(&args).spawn()?;
|
||||
|
||||
self.process = Some(process);
|
||||
@@ -1055,7 +1051,6 @@ pub struct RustFSTestClusterEnvironment {
|
||||
pub secret_key: String,
|
||||
pub extra_env: Vec<(String, String)>,
|
||||
pub node_extra_env: Vec<Vec<(String, String)>>,
|
||||
pub node_capture_log_paths: Vec<Option<String>>,
|
||||
pub topology: ClusterTopology,
|
||||
}
|
||||
|
||||
@@ -1155,7 +1150,6 @@ impl RustFSTestClusterEnvironment {
|
||||
secret_key: "rustfs-cluster-test-secret".to_string(),
|
||||
extra_env,
|
||||
node_extra_env: vec![Vec::new(); topology.node_count],
|
||||
node_capture_log_paths: vec![None; topology.node_count],
|
||||
topology,
|
||||
})
|
||||
}
|
||||
@@ -1185,20 +1179,6 @@ impl RustFSTestClusterEnvironment {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Capture stdout+stderr for a single cluster node process.
|
||||
pub fn set_node_capture_log_path<P>(
|
||||
&mut self,
|
||||
node_idx: usize,
|
||||
path: P,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
where
|
||||
P: Into<String>,
|
||||
{
|
||||
self.ensure_node_index(node_idx)?;
|
||||
self.node_capture_log_paths[node_idx] = Some(path.into());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_node_index(&self, node_idx: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
if node_idx >= self.nodes.len() {
|
||||
return Err(format!("node_idx {node_idx} is invalid").into());
|
||||
@@ -1288,7 +1268,6 @@ impl RustFSTestClusterEnvironment {
|
||||
for (key, value) in &self.node_extra_env[i] {
|
||||
command.env(key, value);
|
||||
}
|
||||
capture_command_logs(&mut command, self.node_capture_log_paths[i].as_deref())?;
|
||||
|
||||
let process = command.current_dir(&node.data_dir).spawn()?;
|
||||
|
||||
@@ -1315,7 +1294,6 @@ impl RustFSTestClusterEnvironment {
|
||||
|
||||
let binary_path = rustfs_binary_path();
|
||||
let volumes_arg = self.build_volumes_arg();
|
||||
let log_path = self.node_capture_log_paths[node_idx].clone();
|
||||
let node = &mut self.nodes[node_idx];
|
||||
info!("Starting cluster node {} on {}", node_idx, node.address);
|
||||
|
||||
@@ -1334,7 +1312,6 @@ impl RustFSTestClusterEnvironment {
|
||||
for (key, value) in &self.node_extra_env[node_idx] {
|
||||
command.env(key, value);
|
||||
}
|
||||
capture_command_logs(&mut command, log_path.as_deref())?;
|
||||
|
||||
let process = command.current_dir(&node.data_dir).spawn()?;
|
||||
node.process = Some(process);
|
||||
@@ -1586,7 +1563,6 @@ mod tests {
|
||||
secret_key: DEFAULT_SECRET_KEY.to_string(),
|
||||
extra_env: Vec::new(),
|
||||
node_extra_env: vec![Vec::new(); topology.node_count],
|
||||
node_capture_log_paths: vec![None; topology.node_count],
|
||||
topology,
|
||||
}
|
||||
}
|
||||
@@ -1682,16 +1658,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_node_log_capture_supports_per_node_paths() {
|
||||
let mut env = fake_cluster(ClusterTopology::single_pool(3));
|
||||
env.set_node_capture_log_path(1, "/tmp/node1.log").unwrap();
|
||||
assert_eq!(env.node_capture_log_paths[0], None);
|
||||
assert_eq!(env.node_capture_log_paths[1], Some("/tmp/node1.log".to_string()));
|
||||
assert_eq!(env.node_capture_log_paths[2], None);
|
||||
assert!(env.set_node_capture_log_path(3, "/tmp/invalid.log").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_node_env_rejects_invalid_index() {
|
||||
let mut env = fake_cluster(ClusterTopology::single_pool(4));
|
||||
|
||||
@@ -189,6 +189,8 @@ mod tests {
|
||||
("RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT", "100"),
|
||||
("RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED", "true"),
|
||||
("RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED", "true"),
|
||||
// Lower the min-size floor so every non-inline object below is eligible.
|
||||
("RUSTFS_GET_CODEC_STREAMING_MIN_SIZE", "4096"),
|
||||
// Route multipart objects through per-part codec streaming too.
|
||||
("RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE", "true"),
|
||||
// Lock optimization is on by default, but pin it so the gate's
|
||||
@@ -313,13 +315,6 @@ mod tests {
|
||||
},
|
||||
payload(64 * 1024, 2),
|
||||
),
|
||||
(
|
||||
Shape {
|
||||
key: "small-non-inline-256kib-plus",
|
||||
expect_large: true,
|
||||
},
|
||||
payload(256 * 1024 + 1, 6),
|
||||
),
|
||||
(
|
||||
Shape {
|
||||
key: "mid-1_5mib",
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
//! E2E tests for group management (fixes #2028).
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
|
||||
use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_get, awscurl_put, init_logging};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use serial_test::serial;
|
||||
@@ -32,56 +32,6 @@ fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_k
|
||||
Client::from_conf(config)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let invalid_groups = [
|
||||
("test group", "group name contains whitespace"),
|
||||
("test=group", "group name contains reserved characters =,"),
|
||||
("test,group", "group name contains reserved characters =,"),
|
||||
];
|
||||
|
||||
for (group, expected_message) in invalid_groups {
|
||||
let body = serde_json::json!({
|
||||
"group": group,
|
||||
"members": [],
|
||||
"isRemove": false,
|
||||
"groupStatus": "enabled"
|
||||
})
|
||||
.to_string();
|
||||
let (status, response_body) = admin_request(
|
||||
&env.url,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/update-group-members",
|
||||
Some(body),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"invalid group {group:?} must return HTTP 400, body: {response_body}"
|
||||
);
|
||||
assert!(
|
||||
response_body.contains("<Code>InvalidArgument</Code>"),
|
||||
"invalid group {group:?} must return InvalidArgument, body: {response_body}"
|
||||
);
|
||||
assert!(
|
||||
response_body.contains(&format!("<Message>{expected_message}</Message>")),
|
||||
"invalid group {group:?} returned an unexpected message: {response_body}"
|
||||
);
|
||||
}
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Test that deleting a group with members fails, and deleting an empty group succeeds.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
|
||||
@@ -39,10 +39,6 @@ pub mod fault_proxy;
|
||||
#[cfg(test)]
|
||||
mod reliability_disk_fault_test;
|
||||
|
||||
// Privileged Linux-only 3x4 replacement rebuild proof for rustfs#5869/#1791.
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
mod replacement_privileged_e2e_test;
|
||||
|
||||
// dist-13 (backlog#1150/#1155): e2e regression net proving a large-object
|
||||
// degraded EC read never returns a silently truncated body (rustfs#4594/#4560/#4585).
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -285,198 +285,6 @@ async fn allow_anonymous_put_object(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One rejected POST Object upload driven end-to-end (backlog#1838): starts a
|
||||
/// fresh server, allows anonymous PutObject on `bucket`, posts an anonymous
|
||||
/// POST Object form whose policy carries `policy_conditions` and whose form
|
||||
/// carries `form_fields` on top of the mandatory key+policy fields, then
|
||||
/// asserts the expected status, error code, and lowercase-body mention.
|
||||
/// `case` prefixes every assertion message so a failing table row is
|
||||
/// identifiable at a glance.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn run_post_object_policy_case(
|
||||
bucket: &str,
|
||||
object_key: &str,
|
||||
policy_conditions: Vec<serde_json::Value>,
|
||||
form_fields: &[(&str, &str)],
|
||||
file_body: &[u8],
|
||||
expected_status: reqwest::StatusCode,
|
||||
expected_code: &str,
|
||||
expected_mention: &str,
|
||||
case: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(policy_conditions);
|
||||
|
||||
let mut post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy);
|
||||
for (name, value) in form_fields {
|
||||
post_form = post_form.text(name.to_string(), value.to_string());
|
||||
}
|
||||
let post_form = post_form.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(file_body.to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, expected_status, "[{case}] unexpected status, body: {response_body}");
|
||||
assert!(
|
||||
response_body.contains(expected_code),
|
||||
"[{case}] response should contain {expected_code}, got: {response_body}"
|
||||
);
|
||||
assert!(
|
||||
response_body_lower.contains(expected_mention),
|
||||
"[{case}] response should mention {expected_mention}, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Table-driven fold of the nine `*_missing_from_policy_conditions` POST
|
||||
/// Object tests (backlog#1838 PR1). Every row keeps its original test's exact
|
||||
/// bucket, key, form field, file body, and expected error strings; the shared
|
||||
/// shape is: policy pins bucket + key + content-length-range only, the form
|
||||
/// smuggles one extra field the policy never declared, and the upload must be
|
||||
/// rejected with 403 AccessDenied naming the offending field.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_fields_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
// (case, bucket, object_key, form field, file body, expected code, expected mention)
|
||||
type Case = (
|
||||
&'static str,
|
||||
&'static str,
|
||||
&'static str,
|
||||
(&'static str, &'static str),
|
||||
&'static [u8],
|
||||
&'static str,
|
||||
&'static str,
|
||||
);
|
||||
let cases: &[Case] = &[
|
||||
(
|
||||
"cache-control",
|
||||
"anon-post-policy-cache-control-missing",
|
||||
"uploads/cache-control-missing.txt",
|
||||
("Cache-Control", "max-age=60"),
|
||||
b"post-policy-cache-control-missing",
|
||||
"AccessDenied",
|
||||
"cache-control",
|
||||
),
|
||||
(
|
||||
"content-language",
|
||||
"anon-post-policy-content-language-missing",
|
||||
"uploads/content-language-missing.txt",
|
||||
("Content-Language", "en-US"),
|
||||
b"post-policy-content-language-missing",
|
||||
"AccessDenied",
|
||||
"content-language",
|
||||
),
|
||||
(
|
||||
"content-encoding",
|
||||
"anon-post-policy-content-encoding-missing",
|
||||
"uploads/content-encoding-missing.txt",
|
||||
("Content-Encoding", "gzip"),
|
||||
b"post-policy-content-encoding-missing",
|
||||
"AccessDenied",
|
||||
"content-encoding",
|
||||
),
|
||||
(
|
||||
"website-redirect-location",
|
||||
"anon-post-policy-website-redirect-missing",
|
||||
"uploads/website-redirect-missing.txt",
|
||||
("x-amz-website-redirect-location", "/docs/landing.html"),
|
||||
b"post-policy-website-redirect-missing",
|
||||
"AccessDenied",
|
||||
"x-amz-website-redirect-location",
|
||||
),
|
||||
(
|
||||
"expires",
|
||||
"anon-post-policy-expires-missing",
|
||||
"uploads/expires-missing-object.txt",
|
||||
("Expires", "Wed, 21 Oct 2037 07:28:00 GMT"),
|
||||
b"post-policy-expires-missing",
|
||||
"AccessDenied",
|
||||
"expires",
|
||||
),
|
||||
(
|
||||
"tagging",
|
||||
"anon-post-policy-tagging-missing",
|
||||
"uploads/tagging-missing-object.txt",
|
||||
("x-amz-tagging", "project=alpha&env=test"),
|
||||
b"post-policy-tagging-missing",
|
||||
"AccessDenied",
|
||||
"x-amz-tagging",
|
||||
),
|
||||
(
|
||||
"metadata",
|
||||
"anon-post-policy-meta-reject",
|
||||
"uploads/meta-reject-object.txt",
|
||||
("x-amz-meta-project", "alpha-demo"),
|
||||
b"post-policy-body",
|
||||
"<Code>AccessDenied</Code>",
|
||||
"x-amz-meta-project",
|
||||
),
|
||||
(
|
||||
"metadata-new-key",
|
||||
"anon-post-policy-meta-name-missing",
|
||||
"uploads/meta-name-missing.txt",
|
||||
("x-amz-meta-name", "demo-name"),
|
||||
b"post-policy-meta-name-missing",
|
||||
"<Code>AccessDenied</Code>",
|
||||
"x-amz-meta-name",
|
||||
),
|
||||
(
|
||||
"content-type",
|
||||
"anon-post-policy-content-type-missing",
|
||||
"uploads/content-type-missing.txt",
|
||||
("Content-Type", "text/plain"),
|
||||
b"post-policy-content-type-missing",
|
||||
"AccessDenied",
|
||||
"content-type",
|
||||
),
|
||||
];
|
||||
|
||||
for (case, bucket, object_key, form_field, file_body, expected_code, expected_mention) in cases {
|
||||
run_post_object_policy_case(
|
||||
bucket,
|
||||
object_key,
|
||||
vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
],
|
||||
&[*form_field],
|
||||
file_body,
|
||||
reqwest::StatusCode::FORBIDDEN,
|
||||
expected_code,
|
||||
expected_mention,
|
||||
case,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_multipart_control_apis_require_auth() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
@@ -3061,6 +2869,59 @@ async fn test_anonymous_post_object_rejects_cache_control_policy_mismatch() -> R
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_cache_control_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-cache-control-missing";
|
||||
let object_key = "uploads/cache-control-missing.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("Cache-Control", "max-age=60")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-cache-control-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert!(
|
||||
response_body_lower.contains("cache-control"),
|
||||
"response should mention cache-control, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_content_language_field_exact_policy_match()
|
||||
@@ -3173,6 +3034,59 @@ async fn test_anonymous_post_object_rejects_content_language_policy_mismatch()
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_content_language_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-content-language-missing";
|
||||
let object_key = "uploads/content-language-missing.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("Content-Language", "en-US")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-content-language-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert!(
|
||||
response_body_lower.contains("content-language"),
|
||||
"response should mention content-language, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_content_encoding_field_exact_policy_match()
|
||||
@@ -3285,6 +3199,59 @@ async fn test_anonymous_post_object_rejects_content_encoding_policy_mismatch()
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_content_encoding_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-content-encoding-missing";
|
||||
let object_key = "uploads/content-encoding-missing.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("Content-Encoding", "gzip")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-content-encoding-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert!(
|
||||
response_body_lower.contains("content-encoding"),
|
||||
"response should mention content-encoding, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_accepts_website_redirect_location_exact_policy_match()
|
||||
@@ -3343,6 +3310,59 @@ async fn test_anonymous_post_object_accepts_website_redirect_location_exact_poli
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_website_redirect_location_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-website-redirect-missing";
|
||||
let object_key = "uploads/website-redirect-missing.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("x-amz-website-redirect-location", "/docs/landing.html")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-website-redirect-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert!(
|
||||
response_body_lower.contains("x-amz-website-redirect-location"),
|
||||
"response should mention x-amz-website-redirect-location, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_website_redirect_location_policy_mismatch()
|
||||
@@ -3509,6 +3529,59 @@ async fn test_anonymous_post_object_rejects_expires_field_policy_mismatch() -> R
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_expires_field_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-expires-missing";
|
||||
let object_key = "uploads/expires-missing-object.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("Expires", "Wed, 21 Oct 2037 07:28:00 GMT")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-expires-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert!(
|
||||
response_body_lower.contains("expires"),
|
||||
"response should mention Expires, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_object_lock_retention_without_permission()
|
||||
@@ -4037,6 +4110,115 @@ async fn test_anonymous_post_object_rejects_tagging_field_policy_mismatch() -> R
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_tagging_field_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-tagging-missing";
|
||||
let object_key = "uploads/tagging-missing-object.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("x-amz-tagging", "project=alpha&env=test")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-tagging-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert!(
|
||||
response_body_lower.contains("x-amz-tagging"),
|
||||
"response should mention x-amz-tagging, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_metadata_field_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-meta-reject";
|
||||
let object_key = "uploads/meta-reject-object.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("x-amz-meta-project", "alpha-demo")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-body".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(
|
||||
response_body.contains("<Code>AccessDenied</Code>"),
|
||||
"response should contain AccessDenied code, got: {response_body}"
|
||||
);
|
||||
assert!(
|
||||
response_body_lower.contains("x-amz-meta-project"),
|
||||
"response should mention the missing metadata field, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_metadata_field_exact_policy_mismatch()
|
||||
@@ -4206,6 +4388,59 @@ async fn test_anonymous_post_object_allows_x_ignore_fields_outside_policy_condit
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_metadata_field_missing_from_policy_conditions_for_new_key()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-meta-name-missing";
|
||||
let object_key = "uploads/meta-name-missing.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("x-amz-meta-name", "demo-name")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-meta-name-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("<Code>AccessDenied</Code>"));
|
||||
assert!(
|
||||
response_body_lower.contains("x-amz-meta-name"),
|
||||
"response should mention x-amz-meta-name, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_metadata_uuid_exact_policy_mismatch()
|
||||
@@ -4641,6 +4876,59 @@ async fn test_anonymous_post_object_rejects_content_type_policy_mismatch() -> Re
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_anonymous_post_object_rejects_content_type_missing_from_policy_conditions()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "anon-post-policy-content-type-missing";
|
||||
let object_key = "uploads/content-type-missing.txt";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
allow_anonymous_put_object(&admin_client, bucket).await?;
|
||||
|
||||
let policy = encode_post_policy(vec![
|
||||
serde_json::json!({ "bucket": bucket }),
|
||||
serde_json::json!({ "key": object_key }),
|
||||
serde_json::json!(["content-length-range", 0, 1024]),
|
||||
]);
|
||||
|
||||
let post_form = reqwest::multipart::Form::new()
|
||||
.text("key", object_key.to_string())
|
||||
.text("policy", policy)
|
||||
.text("Content-Type", "text/plain")
|
||||
.part(
|
||||
"file",
|
||||
reqwest::multipart::Part::bytes(b"post-policy-content-type-missing".to_vec())
|
||||
.file_name("upload.txt")
|
||||
.mime_str("text/plain")?,
|
||||
);
|
||||
|
||||
let post_resp = local_http_client()
|
||||
.post(format!("{}/{}", env.url, bucket))
|
||||
.multipart(post_form)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let status = post_resp.status();
|
||||
let response_body = post_resp.text().await?;
|
||||
let response_body_lower = response_body.to_ascii_lowercase();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
|
||||
assert!(response_body.contains("AccessDenied"));
|
||||
assert!(
|
||||
response_body_lower.contains("content-type"),
|
||||
"response should mention content-type, got: {response_body}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_signed_put_object_extract_expands_tar_entries_with_prefix_headers()
|
||||
|
||||
@@ -252,7 +252,6 @@ impl QuotaTestEnv {
|
||||
#[cfg(test)]
|
||||
mod integration_tests {
|
||||
use super::*;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
@@ -964,27 +963,9 @@ mod integration_tests {
|
||||
.send()
|
||||
.await;
|
||||
|
||||
let complete_error = complete_result.expect_err("multipart completion above quota must be rejected");
|
||||
assert_eq!(complete_error.as_service_error().and_then(|error| error.code()), Some("InvalidRequest"));
|
||||
assert!(complete_result.is_err());
|
||||
assert!(!env.object_exists("over_quota.txt").await?);
|
||||
|
||||
let staged_parts = env
|
||||
.client
|
||||
.list_parts()
|
||||
.bucket(&env.bucket_name)
|
||||
.key("over_quota.txt")
|
||||
.upload_id(upload_id2)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(staged_parts.parts().len(), 2, "quota rejection must preserve the multipart upload");
|
||||
env.client
|
||||
.abort_multipart_upload()
|
||||
.bucket(&env.bucket_name)
|
||||
.key("over_quota.txt")
|
||||
.upload_id(upload_id2)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
env.cleanup_bucket().await?;
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -349,32 +349,11 @@ mod tests {
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let first_inline = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("versions/inline.bin")
|
||||
.body(ByteStream::from(payload(8 * 1024, 40)))
|
||||
.send()
|
||||
.await?;
|
||||
let first_inline_version = first_inline
|
||||
.version_id()
|
||||
.ok_or("first inline PUT did not return a version ID")?;
|
||||
let second_inline = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("versions/inline.bin")
|
||||
.body(ByteStream::from(payload(8 * 1024, 41)))
|
||||
.send()
|
||||
.await?;
|
||||
let second_inline_version = second_inline
|
||||
.version_id()
|
||||
.ok_or("second inline PUT did not return a version ID")?;
|
||||
|
||||
let first = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload(128 * 1024, 41)))
|
||||
.body(ByteStream::from(payload(256 * 1024, 41)))
|
||||
.send()
|
||||
.await?;
|
||||
let first_version = first.version_id().ok_or("first PUT did not return a version ID")?;
|
||||
@@ -382,36 +361,16 @@ mod tests {
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload(3 * 1024 * 1024, 42)))
|
||||
.body(ByteStream::from(payload(256 * 1024, 42)))
|
||||
.send()
|
||||
.await?;
|
||||
let second_version = second.version_id().ok_or("second PUT did not return a version ID")?;
|
||||
let delete = client.delete_object().bucket(bucket).key(key).send().await?;
|
||||
let delete_version = delete.version_id().ok_or("delete marker did not return a version ID")?;
|
||||
|
||||
let first_inline_census = harness.census_object_version(0, bucket, "versions/inline.bin", Some(first_inline_version))?;
|
||||
let second_inline_census =
|
||||
harness.census_object_version(0, bucket, "versions/inline.bin", Some(second_inline_version))?;
|
||||
let first_census = harness.census_object_version(0, bucket, key, Some(first_version))?;
|
||||
let first_other_disk_census = harness.census_object_version(1, bucket, key, Some(first_version))?;
|
||||
let second_census = harness.census_object_version(0, bucket, key, Some(second_version))?;
|
||||
let delete_census = harness.census_object_version(0, bucket, key, Some(delete_version))?;
|
||||
assert!(
|
||||
first_inline_census.is_complete() && second_inline_census.is_complete(),
|
||||
"inline version physical census is incomplete: first={first_inline_census:?} second={second_inline_census:?}"
|
||||
);
|
||||
assert!(
|
||||
first_inline_census.present_part_fingerprints.is_empty() && second_inline_census.present_part_fingerprints.is_empty(),
|
||||
"inline versions must not select external shard files: first={first_inline_census:?} second={second_inline_census:?}"
|
||||
);
|
||||
assert!(
|
||||
first_inline_census.inline_data_fingerprint.is_some() && second_inline_census.inline_data_fingerprint.is_some(),
|
||||
"inline versions must fingerprint payload bytes stored in xl.meta"
|
||||
);
|
||||
assert_ne!(
|
||||
first_inline_census.inline_data_fingerprint, second_inline_census.inline_data_fingerprint,
|
||||
"same-size inline versions with different payloads must retain distinct xl.meta fingerprints"
|
||||
);
|
||||
assert!(
|
||||
first_census.is_complete(),
|
||||
"first version physical census is incomplete: {first_census:?}"
|
||||
@@ -420,14 +379,6 @@ mod tests {
|
||||
second_census.is_complete(),
|
||||
"second version physical census is incomplete: {second_census:?}"
|
||||
);
|
||||
assert!(
|
||||
first_other_disk_census.is_complete(),
|
||||
"first version physical census on the second disk is incomplete: {first_other_disk_census:?}"
|
||||
);
|
||||
assert_ne!(
|
||||
first_census.erasure_index, first_other_disk_census.erasure_index,
|
||||
"physical census must preserve each disk's erasure index"
|
||||
);
|
||||
assert_ne!(
|
||||
first_census.data_dir, second_census.data_dir,
|
||||
"distinct object versions must select distinct physical data directories"
|
||||
@@ -436,24 +387,6 @@ mod tests {
|
||||
first_census.expected_part_numbers, second_census.expected_part_numbers,
|
||||
"same single-part shape should expose the same part numbers"
|
||||
);
|
||||
let first_part = first_census
|
||||
.present_part_fingerprints
|
||||
.values()
|
||||
.next()
|
||||
.ok_or("first version did not expose a physical part fingerprint")?;
|
||||
let second_part = second_census
|
||||
.present_part_fingerprints
|
||||
.values()
|
||||
.next()
|
||||
.ok_or("second version did not expose a physical part fingerprint")?;
|
||||
assert_ne!(
|
||||
first_part.size, second_part.size,
|
||||
"different shard lengths must retain their physical sizes"
|
||||
);
|
||||
assert_ne!(
|
||||
first_part.sha256, second_part.sha256,
|
||||
"different shard contents must retain their physical hashes"
|
||||
);
|
||||
assert!(
|
||||
delete_census.is_complete(),
|
||||
"delete marker physical census is incomplete: {delete_census:?}"
|
||||
@@ -463,7 +396,7 @@ mod tests {
|
||||
"delete marker must not declare object shards: {delete_census:?}"
|
||||
);
|
||||
assert!(
|
||||
delete_census.present_part_fingerprints.is_empty(),
|
||||
delete_census.present_part_numbers.is_empty(),
|
||||
"delete marker must not select stale object shards: {delete_census:?}"
|
||||
);
|
||||
Ok(())
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -69,7 +69,6 @@ fn build_non_inline_writers(config: &BenchConfig) -> Vec<Option<BitrotWriterWrap
|
||||
fn bench_single_block_non_inline_fast_path(c: &mut Criterion) {
|
||||
let configs = vec![
|
||||
BenchConfig::new(4 * 1024, 4, 2, 128 * 1024),
|
||||
BenchConfig::new(16 * 1024, 4, 2, 128 * 1024),
|
||||
BenchConfig::new(64 * 1024, 4, 2, 128 * 1024),
|
||||
BenchConfig::new(128 * 1024, 4, 2, 128 * 1024),
|
||||
];
|
||||
@@ -113,12 +112,7 @@ fn bench_single_block_non_inline_fast_path(c: &mut Criterion) {
|
||||
rt.block_on(async {
|
||||
erasure
|
||||
.clone()
|
||||
.encode_single_block_non_inline_with_size_hint(
|
||||
reader,
|
||||
&mut writers,
|
||||
config.data_shards,
|
||||
config.payload_size,
|
||||
)
|
||||
.encode_single_block_non_inline(reader, &mut writers, config.data_shards)
|
||||
.await
|
||||
.expect("single block candidate benchmark");
|
||||
});
|
||||
|
||||
@@ -308,8 +308,6 @@ pub mod config {
|
||||
}
|
||||
|
||||
pub mod data_usage {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::data_usage::seed_bucket_usage_memory_for_test;
|
||||
pub use crate::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
|
||||
@@ -411,10 +409,10 @@ pub mod object {
|
||||
pub use crate::object_api::{
|
||||
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
|
||||
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
|
||||
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer,
|
||||
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
|
||||
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader,
|
||||
ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest, StreamConsumer, get_object_body_cache_plaintext_len,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
|
||||
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
};
|
||||
pub use crate::store::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
|
||||
@@ -64,41 +64,10 @@ impl BucketDurabilityConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Default durability tier seeded into a newly created bucket's metadata
|
||||
/// (rustfs/backlog#1811). `relaxed` aligns new buckets with MinIO's default
|
||||
/// posture: object data is still fdatasynced, while xl.meta and directory-entry
|
||||
/// fsyncs follow the relaxed durability gate.
|
||||
pub const ENV_NEW_BUCKET_DURABILITY_MODE: &str = "RUSTFS_NEW_BUCKET_DURABILITY_MODE";
|
||||
pub const DEFAULT_NEW_BUCKET_DURABILITY_MODE: &str = BUCKET_DURABILITY_MODE_RELAXED;
|
||||
|
||||
/// The `durability.json` bytes to seed into a freshly created bucket's metadata.
|
||||
/// Empty means "no override" (the bucket then follows the global
|
||||
/// `RUSTFS_DURABILITY_MODE`); otherwise the serialized chosen tier. Operators
|
||||
/// can set `inherit` to disable the new-bucket override. Invalid values also
|
||||
/// fail closed to inherit the global mode instead of seeding a surprising tier.
|
||||
pub fn new_bucket_durability_config_json() -> Vec<u8> {
|
||||
let raw = std::env::var(ENV_NEW_BUCKET_DURABILITY_MODE).unwrap_or_else(|_| DEFAULT_NEW_BUCKET_DURABILITY_MODE.to_string());
|
||||
let mode = raw.trim();
|
||||
if mode.eq_ignore_ascii_case("inherit") || mode.is_empty() || !BucketDurabilityConfig::is_valid_mode(mode) {
|
||||
return Vec::new();
|
||||
}
|
||||
serde_json::to_vec(&BucketDurabilityConfig::new(mode)).expect("BucketDurabilityConfig serialization cannot fail")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn new_bucket_seeded_mode() -> Option<String> {
|
||||
let json = new_bucket_durability_config_json();
|
||||
if json.is_empty() {
|
||||
return None;
|
||||
}
|
||||
serde_json::from_slice::<BucketDurabilityConfig>(&json)
|
||||
.expect("new-bucket durability config must serialize")
|
||||
.normalized_mode()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn valid_modes_are_recognized() {
|
||||
assert!(BucketDurabilityConfig::is_valid_mode("strict"));
|
||||
@@ -130,33 +99,4 @@ mod tests {
|
||||
let empty: BucketDurabilityConfig = serde_json::from_slice(b"{}").expect("deserialize empty");
|
||||
assert_eq!(empty.normalized_mode(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_default_seeds_relaxed_when_unset() {
|
||||
temp_env::with_var_unset(ENV_NEW_BUCKET_DURABILITY_MODE, || {
|
||||
assert_eq!(new_bucket_seeded_mode().as_deref(), Some(BUCKET_DURABILITY_MODE_RELAXED));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_default_honors_explicit_tiers() {
|
||||
for mode in [
|
||||
BUCKET_DURABILITY_MODE_STRICT,
|
||||
BUCKET_DURABILITY_MODE_RELAXED,
|
||||
BUCKET_DURABILITY_MODE_NONE,
|
||||
] {
|
||||
temp_env::with_var(ENV_NEW_BUCKET_DURABILITY_MODE, Some(mode), || {
|
||||
assert_eq!(new_bucket_seeded_mode().as_deref(), Some(mode));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_default_can_inherit_global_mode() {
|
||||
for mode in ["inherit", "", "bogus"] {
|
||||
temp_env::with_var(ENV_NEW_BUCKET_DURABILITY_MODE, Some(mode), || {
|
||||
assert_eq!(new_bucket_seeded_mode(), None);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,8 +78,8 @@ use rustfs_common::metrics::{
|
||||
};
|
||||
use rustfs_config::{
|
||||
DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_QUEUE_SEND_TIMEOUT_MS, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX,
|
||||
DEFAULT_TRANSITION_WORKERS_CAP, ENV_MAX_EXPIRY_WORKERS, ENV_TRANSITION_QUEUE_CAPACITY, ENV_TRANSITION_QUEUE_SEND_TIMEOUT_MS,
|
||||
ENV_TRANSITION_WORKERS, ENV_TRANSITION_WORKERS_ABSOLUTE_MAX,
|
||||
DEFAULT_TRANSITION_WORKERS_CAP, ENV_TRANSITION_QUEUE_CAPACITY, ENV_TRANSITION_QUEUE_SEND_TIMEOUT_MS, ENV_TRANSITION_WORKERS,
|
||||
ENV_TRANSITION_WORKERS_ABSOLUTE_MAX,
|
||||
};
|
||||
use rustfs_data_usage::TierStats;
|
||||
use rustfs_filemeta::{
|
||||
@@ -159,8 +159,6 @@ const TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL: StdDuration = StdDuration::f
|
||||
const TIER_FREE_VERSION_RECOVERY_JITTER_PERCENT: u64 = 10;
|
||||
const DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS: i64 = 5;
|
||||
const EXPIRY_WORKER_QUEUE_CAPACITY: usize = 1000;
|
||||
/// Maximum expiry workers used as a local fallback when runtime env is unset.
|
||||
const DEFAULT_EXPIRY_WORKERS_CAP: usize = 16;
|
||||
const DEFAULT_MANUAL_TRANSITION_JOB_RECOVERY_LIMIT: usize = 100;
|
||||
|
||||
// Phase 5 (backlog#939): lifecycle expiry/transition state moved into the
|
||||
@@ -206,15 +204,6 @@ fn resolve_transition_queue_send_timeout() -> StdDuration {
|
||||
)
|
||||
}
|
||||
|
||||
fn resolve_expiry_worker_count() -> usize {
|
||||
let fallback = std::cmp::min(num_cpus::get(), DEFAULT_EXPIRY_WORKERS_CAP);
|
||||
env::var(ENV_MAX_EXPIRY_WORKERS)
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.filter(|value| *value > 0)
|
||||
.unwrap_or(fallback)
|
||||
}
|
||||
|
||||
fn is_immediate_transition_source(src: &LcEventSrc) -> bool {
|
||||
matches!(
|
||||
src,
|
||||
@@ -2028,7 +2017,17 @@ fn is_slow_down(err: &Error) -> bool {
|
||||
}
|
||||
|
||||
pub async fn init_background_expiry(api: Arc<ECStore>) {
|
||||
let workers = resolve_expiry_worker_count();
|
||||
let mut workers = get_env_usize("RUSTFS_MAX_EXPIRY_WORKERS", std::cmp::min(num_cpus::get(), 16));
|
||||
//globalILMConfig.getExpirationWorkers()
|
||||
if let Ok(env_expiration_workers) = env::var("_RUSTFS_ILM_EXPIRATION_WORKERS")
|
||||
&& let Ok(num_expirations) = env_expiration_workers.parse::<usize>()
|
||||
{
|
||||
workers = num_expirations;
|
||||
}
|
||||
|
||||
if workers == 0 {
|
||||
workers = get_env_usize("RUSTFS_DEFAULT_EXPIRY_WORKERS", 8);
|
||||
}
|
||||
|
||||
ExpiryState::resize_workers(workers, api.clone()).await;
|
||||
let _ = spawn_tier_free_version_recovery_once(api.clone(), &TIER_FREE_VERSION_RECOVERY_STARTED);
|
||||
@@ -3319,9 +3318,6 @@ pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) {
|
||||
return;
|
||||
}
|
||||
};
|
||||
if configs.table_bucket_enabled {
|
||||
return;
|
||||
}
|
||||
let Some(lifecycle) = configs.lifecycle else {
|
||||
return;
|
||||
};
|
||||
@@ -3982,9 +3978,6 @@ async fn enqueue_expiry_for_existing_object_group(
|
||||
|
||||
pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
|
||||
let configs = metadata_boundary::get_expiry_configs(&api, bucket).await?;
|
||||
if configs.table_bucket_enabled {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(lc) = configs.lifecycle else {
|
||||
return Ok(());
|
||||
};
|
||||
@@ -4201,16 +4194,12 @@ pub async fn expire_transitioned_object(
|
||||
_src: &LcEventSrc,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Result<ObjectInfo, std::io::Error> {
|
||||
let publication_guard = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id)
|
||||
.await
|
||||
.ok_or_else(|| std::io::Error::other("lifecycle expiry is not allowed for this bucket"))?;
|
||||
let snapshot = lifecycle_delete_config_snapshot(&api, oi)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let (versioned, version_suspended) = snapshot.versioning_config().delete_state(&oi.name);
|
||||
let mut opts = transitioned_object_delete_opts(oi, lc_event.action, versioned, version_suspended, bucket_incarnation_id)
|
||||
.map_err(std::io::Error::other)?;
|
||||
opts.add_namespace_lock_guard(&publication_guard);
|
||||
opts.delete_replication_config_snapshot = Some(Arc::new(snapshot));
|
||||
//let tags = LcAuditEvent::new(src, lcEvent).Tags();
|
||||
if lc_event.action.delete_restored() {
|
||||
@@ -4800,43 +4789,6 @@ pub async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, o
|
||||
.await
|
||||
}
|
||||
|
||||
async fn lifecycle_expiry_publication_guard(
|
||||
api: &ECStore,
|
||||
oi: &ObjectInfo,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||
let result = async {
|
||||
let lock = api
|
||||
.new_ns_lock(&oi.bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await?;
|
||||
let guard = lock.get_read_lock(get_lock_acquire_timeout()).await.map_err(Error::other)?;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(Error::other("table-bucket publication lock was lost before lifecycle delete admission"));
|
||||
}
|
||||
if !metadata_boundary::lifecycle_expiry_allowed(api, &oi.bucket, bucket_incarnation_id).await? {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(guard))
|
||||
}
|
||||
.await;
|
||||
match result {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_DELETE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
operation = "authorize_lifecycle_expiry",
|
||||
error = %err,
|
||||
"Lifecycle delete admission failed"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn apply_expiry_on_transitioned_object(
|
||||
api: Arc<ECStore>,
|
||||
oi: &ObjectInfo,
|
||||
@@ -4860,9 +4812,6 @@ pub async fn apply_expiry_on_non_transitioned_objects(
|
||||
_src: &LcEventSrc,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> bool {
|
||||
let Some(publication_guard) = lifecycle_expiry_publication_guard(&api, oi, bucket_incarnation_id).await else {
|
||||
return false;
|
||||
};
|
||||
let snapshot = match lifecycle_delete_config_snapshot(&api, oi).await {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(err) => {
|
||||
@@ -4888,7 +4837,6 @@ pub async fn apply_expiry_on_non_transitioned_objects(
|
||||
expected_bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_namespace_lock_guard(&publication_guard);
|
||||
|
||||
if lc_event.action.delete_versioned() {
|
||||
opts.version_id = oi.version_id.map(|v| v.to_string());
|
||||
@@ -5076,27 +5024,26 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc,
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, DEFAULT_EXPIRY_WORKERS_CAP, DEFAULT_TRANSITION_QUEUE_CAPACITY,
|
||||
DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX, DEFAULT_TRANSITION_WORKERS_CAP, EVENT_LIFECYCLE_EVALUATION_FAILED,
|
||||
EVENT_LIFECYCLE_EXPIRED_DETECTED, EVENT_LIFECYCLE_NOT_ENQUEUED, ExpiryState, ExpiryTask, FreeVersionTask,
|
||||
ManualTransitionJobRecoveryOutcome, ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport,
|
||||
StaleMultipartUploadCandidate, TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL, TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL,
|
||||
TRANSITION_COMPLETE, TierFreeVersionRecoverySchedule, TransitionEnqueueOutcome, TransitionState, TransitionedObject,
|
||||
VersionReplicationScan, cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at,
|
||||
DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX,
|
||||
DEFAULT_TRANSITION_WORKERS_CAP, EVENT_LIFECYCLE_EVALUATION_FAILED, EVENT_LIFECYCLE_EXPIRED_DETECTED,
|
||||
EVENT_LIFECYCLE_NOT_ENQUEUED, ExpiryState, ExpiryTask, FreeVersionTask, ManualTransitionJobRecoveryOutcome,
|
||||
ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, StaleMultipartUploadCandidate,
|
||||
TIER_FREE_VERSION_RECOVERY_BASE_INTERVAL, TIER_FREE_VERSION_RECOVERY_MAX_IDLE_INTERVAL, TRANSITION_COMPLETE,
|
||||
TierFreeVersionRecoverySchedule, TransitionEnqueueOutcome, TransitionState, TransitionedObject, VersionReplicationScan,
|
||||
cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at,
|
||||
enqueue_recovered_free_version_with_state, enqueue_transition_for_existing_objects_scoped,
|
||||
enqueue_transition_with_lifecycle, enqueue_transition_with_lifecycle_report, eval_action_from_lifecycle,
|
||||
get_lock_acquire_timeout, jitter_tier_free_version_recovery_delay, lifecycle_action_blocked_by_replication,
|
||||
jitter_tier_free_version_recovery_delay, lifecycle_action_blocked_by_replication,
|
||||
lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object, lifecycle_replication_blocks_action,
|
||||
lifecycle_rule_has_date_expiration, manual_transition_duration_elapsed, manual_transition_has_more_after_limit,
|
||||
manual_transition_recovery_progress_sink, manual_transition_version_marker, manual_transition_worker_failure_reason,
|
||||
mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate,
|
||||
persist_manual_transition_job_progress, persist_manual_transition_page_checkpoint, recover_manual_transition_job,
|
||||
recover_manual_transition_jobs, resolve_expiry_worker_count, resolve_tier_free_version_recovery_enabled,
|
||||
resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, resolve_transition_worker_count,
|
||||
resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop, select_restore_s3_location,
|
||||
set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer,
|
||||
should_defer_date_expiry_for_recent_config_update, transitioned_cleanup_tuple, transitioned_object_delete_opts,
|
||||
wait_for_tier_free_version_recovery,
|
||||
recover_manual_transition_jobs, resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity,
|
||||
resolve_transition_queue_send_timeout, resolve_transition_worker_count, resolve_transition_workers_absolute_max,
|
||||
run_tier_free_version_recovery_loop, select_restore_s3_location, set_lifecycle_observability_observer,
|
||||
set_recovered_free_version_enqueue_observer, should_defer_date_expiry_for_recent_config_update,
|
||||
transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
use super::{delete_free_version_remote_object_then, encode_dir_object, get_transitioned_object_reader_with_tier_manager};
|
||||
@@ -5143,6 +5090,7 @@ mod tests {
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::services::tier::warm_backend::WarmBackend as _;
|
||||
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
|
||||
@@ -5157,7 +5105,7 @@ mod tests {
|
||||
#[cfg(feature = "test-util")]
|
||||
use http::HeaderMap;
|
||||
use rustfs_common::metrics::{IlmAction, global_metrics};
|
||||
use rustfs_config::{ENV_MAX_EXPIRY_WORKERS, ENV_TRANSITION_WORKERS_ABSOLUTE_MAX};
|
||||
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
|
||||
use rustfs_data_usage::TierStats;
|
||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||
use s3s::dto::{
|
||||
@@ -7409,76 +7357,6 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
// SAFETY: this helper is only used from `#[serial]` tests and those tests run under a
|
||||
// single-thread runtime (`worker_threads = 1`), so no concurrent reader/writer can access
|
||||
// process environment while `env::set_var`/`env::remove_var` is active.
|
||||
#[allow(unsafe_code)]
|
||||
fn with_expiry_worker_env<F>(value: Option<&str>, test_fn: F)
|
||||
where
|
||||
F: FnOnce(),
|
||||
{
|
||||
let original = env::var_os(ENV_MAX_EXPIRY_WORKERS);
|
||||
|
||||
match value {
|
||||
Some(value) => unsafe {
|
||||
env::set_var(ENV_MAX_EXPIRY_WORKERS, value);
|
||||
},
|
||||
None => unsafe {
|
||||
env::remove_var(ENV_MAX_EXPIRY_WORKERS);
|
||||
},
|
||||
}
|
||||
|
||||
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(test_fn));
|
||||
|
||||
match original {
|
||||
Some(value) => unsafe {
|
||||
env::set_var(ENV_MAX_EXPIRY_WORKERS, value);
|
||||
},
|
||||
None => unsafe {
|
||||
env::remove_var(ENV_MAX_EXPIRY_WORKERS);
|
||||
},
|
||||
}
|
||||
|
||||
if let Err(e) = result {
|
||||
std::panic::resume_unwind(e);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn resolve_expiry_worker_count_uses_fallback_when_env_missing() {
|
||||
with_expiry_worker_env(None, || {
|
||||
let fallback = std::cmp::min(num_cpus::get(), DEFAULT_EXPIRY_WORKERS_CAP);
|
||||
assert_eq!(resolve_expiry_worker_count(), fallback);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn resolve_expiry_worker_count_honors_positive_env_value() {
|
||||
with_expiry_worker_env(Some("6"), || {
|
||||
assert_eq!(resolve_expiry_worker_count(), 6);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn resolve_expiry_worker_count_falls_back_for_zero_value() {
|
||||
with_expiry_worker_env(Some("0"), || {
|
||||
let fallback = std::cmp::min(num_cpus::get(), DEFAULT_EXPIRY_WORKERS_CAP);
|
||||
assert_eq!(resolve_expiry_worker_count(), fallback);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn resolve_expiry_worker_count_falls_back_for_invalid_value() {
|
||||
with_expiry_worker_env(Some("not-a-number"), || {
|
||||
let fallback = std::cmp::min(num_cpus::get(), DEFAULT_EXPIRY_WORKERS_CAP);
|
||||
assert_eq!(resolve_expiry_worker_count(), fallback);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn resolve_transition_queue_capacity_uses_default_when_env_missing() {
|
||||
@@ -10441,85 +10319,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn queued_lifecycle_expiry_does_not_delete_from_table_bucket() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("table-bucket-lifecycle-{}", Uuid::new_v4().simple());
|
||||
let object = "tables/table-id/data/part-00001.parquet";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
|
||||
let mut reader = PutObjReader::from_vec(b"referenced table data".to_vec());
|
||||
let object_info = ecstore
|
||||
.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("table data object should be created");
|
||||
|
||||
let publication_lock = ecstore
|
||||
.new_ns_lock(&bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("table-bucket publication lock should be created");
|
||||
let enable_guard = publication_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.expect("table-bucket enablement should acquire the publication lock");
|
||||
let expiry_store = ecstore.clone();
|
||||
let expiry_object = object_info.clone();
|
||||
let (expiry_started_tx, expiry_started_rx) = tokio::sync::oneshot::channel();
|
||||
let mut expiry = tokio::spawn(async move {
|
||||
let event = crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: IlmAction::DeleteAction,
|
||||
..Default::default()
|
||||
};
|
||||
let bucket_incarnation_id = expiry_store
|
||||
.bucket_incarnation_id_from_disk(&expiry_object.bucket)
|
||||
.await
|
||||
.expect("bucket incarnation should be available");
|
||||
expiry_started_tx.send(()).expect("lifecycle expiry start should be observed");
|
||||
super::apply_expiry_on_non_transitioned_objects(
|
||||
expiry_store,
|
||||
&expiry_object,
|
||||
&event,
|
||||
&LcEventSrc::Scanner,
|
||||
bucket_incarnation_id,
|
||||
)
|
||||
.await
|
||||
});
|
||||
expiry_started_rx.await.expect("lifecycle expiry should start");
|
||||
assert!(
|
||||
tokio::time::timeout(StdDuration::from_millis(100), &mut expiry)
|
||||
.await
|
||||
.is_err(),
|
||||
"queued lifecycle expiry must wait for table-bucket enablement"
|
||||
);
|
||||
|
||||
let sys = metadata_sys::bucket_metadata_sys_of(&ecstore.ctx).expect("metadata system should be initialized");
|
||||
let sys = sys.read().await.clone();
|
||||
let mut metadata = (*sys.get(&bucket).await.expect("bucket metadata should exist")).clone();
|
||||
metadata.table_bucket_config_json = br#"{"enabled":true}"#.to_vec();
|
||||
sys.persist_and_set(metadata)
|
||||
.await
|
||||
.expect("table bucket marker should be persisted");
|
||||
sys.reload_from_store(&bucket)
|
||||
.await
|
||||
.expect("table bucket marker should become authoritative");
|
||||
drop(enable_guard);
|
||||
assert!(
|
||||
!tokio::time::timeout(StdDuration::from_secs(2), expiry)
|
||||
.await
|
||||
.expect("queued lifecycle expiry should resume after enablement")
|
||||
.expect("queued lifecycle expiry task should join"),
|
||||
"a queued lifecycle task must be rejected after the bucket becomes table-enabled"
|
||||
);
|
||||
assert!(
|
||||
ecstore
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.is_ok(),
|
||||
"table data must remain readable after lifecycle admission rejects the delete"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_object_lifecycle_skips_current_expiration_for_explicit_legal_hold() {
|
||||
let lc = latest_expiration_lifecycle();
|
||||
|
||||
@@ -18,7 +18,6 @@ use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::bucket::metadata::BucketMetadata;
|
||||
use crate::bucket::metadata_sys::{self, ObjectLockConfigState};
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
@@ -27,37 +26,16 @@ pub(crate) struct LifecycleExpiryConfigs {
|
||||
pub(crate) lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
|
||||
pub(crate) object_lock: Option<Arc<ObjectLockConfiguration>>,
|
||||
pub(crate) bucket_incarnation_id: Uuid,
|
||||
pub(crate) table_bucket_enabled: bool,
|
||||
}
|
||||
|
||||
async fn get_authoritative_metadata(
|
||||
api: &crate::store::ECStore,
|
||||
bucket: &str,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Result<Arc<BucketMetadata>> {
|
||||
pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result<LifecycleExpiryConfigs> {
|
||||
let bucket_incarnation_id = api.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let sys = metadata_sys::bucket_metadata_sys_of(&api.ctx)?;
|
||||
let sys = sys.read().await.clone();
|
||||
let metadata = sys.get_authoritative_metadata(bucket).await?;
|
||||
if !metadata.bucket_incarnation_sidecar || metadata.bucket_incarnation_id != bucket_incarnation_id {
|
||||
return Err(Error::other(format!("bucket lifecycle metadata is not authoritative: {bucket}")));
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
pub(crate) async fn lifecycle_expiry_allowed(
|
||||
api: &crate::store::ECStore,
|
||||
bucket: &str,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Result<bool> {
|
||||
Ok(!get_authoritative_metadata(api, bucket, bucket_incarnation_id)
|
||||
.await?
|
||||
.table_bucket_enabled())
|
||||
}
|
||||
|
||||
pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result<LifecycleExpiryConfigs> {
|
||||
let bucket_incarnation_id = api.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let metadata = get_authoritative_metadata(api, bucket, bucket_incarnation_id).await?;
|
||||
let table_bucket_enabled = metadata.table_bucket_enabled();
|
||||
|
||||
let lifecycle = if metadata.lifecycle_config.is_none() && !metadata.lifecycle_config_xml.is_empty() {
|
||||
return Err(Error::other("persisted bucket lifecycle configuration is invalid"));
|
||||
@@ -73,7 +51,6 @@ pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str
|
||||
lifecycle: None,
|
||||
object_lock: None,
|
||||
bucket_incarnation_id,
|
||||
table_bucket_enabled,
|
||||
});
|
||||
}
|
||||
let object_lock = match metadata_sys::object_lock_config_state_from_authoritative_metadata(&metadata)? {
|
||||
@@ -88,7 +65,6 @@ pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str
|
||||
lifecycle,
|
||||
object_lock,
|
||||
bucket_incarnation_id,
|
||||
table_bucket_enabled,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -149,7 +125,6 @@ mod tests {
|
||||
let lifecycle = lifecycle_config();
|
||||
metadata.lifecycle_config_xml = crate::bucket::utils::serialize(&lifecycle).unwrap();
|
||||
metadata.lifecycle_config = Some(lifecycle);
|
||||
metadata.table_bucket_config_json = br#"{"enabled":true}"#.to_vec();
|
||||
metadata_sys::set_new_bucket_metadata_in(&store_a.ctx, metadata)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -157,14 +132,7 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let configs = get_expiry_configs(&store_a, bucket).await.unwrap();
|
||||
assert!(configs.lifecycle.is_some());
|
||||
assert!(configs.table_bucket_enabled);
|
||||
assert!(
|
||||
!lifecycle_expiry_allowed(&store_a, bucket, configs.bucket_incarnation_id)
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
assert!(get_expiry_configs(&store_a, bucket).await.unwrap().lifecycle.is_some());
|
||||
assert!(get_expiry_configs(&store_b, bucket).await.unwrap().lifecycle.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -425,15 +425,6 @@ impl BucketMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
/// Metadata for a physically new user bucket. Existing or fabricated legacy
|
||||
/// metadata must use [`Self::new`] so upgrades do not rewrite their
|
||||
/// durability posture.
|
||||
pub fn new_with_default_durability(name: &str) -> Self {
|
||||
let mut metadata = Self::new(name);
|
||||
metadata.durability_config_json = super::durability::new_bucket_durability_config_json();
|
||||
metadata
|
||||
}
|
||||
|
||||
pub fn save_file_path(&self) -> String {
|
||||
format!("{}/{}/{}", BUCKET_META_PREFIX, self.name.as_str(), BUCKET_METADATA_FILE)
|
||||
}
|
||||
@@ -1311,7 +1302,7 @@ mod test {
|
||||
assert!(bm.object_locking(), "object lock active via parsed config");
|
||||
}
|
||||
|
||||
/// backlog#580: KNOWN GAP (flagged 2026-03-06: "inline_data 前缀不同"). RustFS's
|
||||
/// backlog#580: KNOWN GAP (weisd 2026-03-06 "inline_data 前缀不同"). RustFS's
|
||||
/// inline-data extraction does not yet recover the object body from a
|
||||
/// MinIO-written bucket-metadata object: `into_fileinfo(read_data=true).data`
|
||||
/// returns bytes that are not the `.metadata.bin` blob (no `format|version`
|
||||
@@ -1319,7 +1310,7 @@ mod test {
|
||||
/// inline-data framing is handled on the read path.
|
||||
/// backlog#580: prove RustFS reads a MinIO-written **inlined** bucket-metadata
|
||||
/// object end-to-end. MinIO stores inline data as `[bitrot hash][object body]`
|
||||
/// (the "`inline_data` 前缀不同" gap flagged on 2026-03-06 is that
|
||||
/// (the "`inline_data` 前缀不同" that weisd flagged on 2026-03-06 is that
|
||||
/// bitrot prefix, not a format incompatibility). Running the raw inline shard
|
||||
/// through RustFS's `BitrotReader` with the default `HighwayHash256S` must
|
||||
/// verify the checksum and yield the exact `.metadata.bin` blob.
|
||||
@@ -1387,43 +1378,6 @@ mod test {
|
||||
assert_ne!(old.bucket_incarnation_id, new.bucket_incarnation_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regular_bucket_metadata_constructor_does_not_seed_durability() {
|
||||
temp_env::with_var_unset(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, || {
|
||||
let metadata = BucketMetadata::new("legacy-or-fabricated");
|
||||
assert!(metadata.durability_config_json.is_empty());
|
||||
assert!(metadata.durability_config().is_none());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_metadata_constructor_seeds_default_durability() {
|
||||
temp_env::with_var_unset(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, || {
|
||||
let metadata = BucketMetadata::new_with_default_durability("new-user-bucket");
|
||||
assert_eq!(
|
||||
metadata.durability_config().and_then(|cfg| cfg.normalized_mode()).as_deref(),
|
||||
Some(crate::bucket::durability::BUCKET_DURABILITY_MODE_RELAXED)
|
||||
);
|
||||
|
||||
let encoded = metadata.marshal_msg().expect("marshal metadata");
|
||||
let decoded = BucketMetadata::unmarshal(&encoded).expect("unmarshal metadata");
|
||||
assert_eq!(decoded.durability_config_json, metadata.durability_config_json);
|
||||
assert_eq!(
|
||||
decoded.durability_config().and_then(|cfg| cfg.normalized_mode()).as_deref(),
|
||||
Some(crate::bucket::durability::BUCKET_DURABILITY_MODE_RELAXED)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_bucket_metadata_constructor_can_inherit_global_durability() {
|
||||
temp_env::with_var(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, Some("inherit"), || {
|
||||
let metadata = BucketMetadata::new_with_default_durability("strict-fleet-new-bucket");
|
||||
assert!(metadata.durability_config_json.is_empty());
|
||||
assert!(metadata.durability_config().is_none());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn site_replication_config_updates_cannot_replace_bucket_incarnation() {
|
||||
let mut metadata = BucketMetadata::new("site-replication-update");
|
||||
|
||||
@@ -288,13 +288,6 @@ pub(crate) fn bucket_metadata_sys_of(ctx: &crate::runtime::instance::InstanceCon
|
||||
get_bucket_metadata_sys()
|
||||
}
|
||||
|
||||
pub(crate) fn require_bucket_metadata_sys_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
) -> Result<Arc<RwLock<BucketMetadataSys>>> {
|
||||
ctx.bucket_metadata_sys()
|
||||
.ok_or_else(|| Error::other("bucket metadata sys not initialized for this instance"))
|
||||
}
|
||||
|
||||
pub(crate) async fn object_store_in(ctx: &crate::runtime::instance::InstanceContext) -> Result<Arc<ECStore>> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
Ok(sys.read().await.api.clone())
|
||||
@@ -383,15 +376,6 @@ pub async fn update(bucket: &str, config_file: &str, data: Vec<u8>) -> Result<Of
|
||||
Box::pin(update_with_sys(get_bucket_metadata_sys()?, bucket, config_file, data)).await
|
||||
}
|
||||
|
||||
pub(crate) async fn update_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
) -> Result<OffsetDateTime> {
|
||||
Box::pin(update_with_sys(require_bucket_metadata_sys_in(ctx)?, bucket, config_file, data)).await
|
||||
}
|
||||
|
||||
pub async fn delete(bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
|
||||
delete_with_sys(get_bucket_metadata_sys()?, bucket, config_file).await
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
use super::metadata_sys::get_bucket_metadata_sys;
|
||||
use crate::error::{Result, StorageError};
|
||||
use crate::store::ECStore;
|
||||
use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
|
||||
|
||||
pub struct PolicySys {}
|
||||
@@ -28,10 +27,6 @@ impl PolicySys {
|
||||
Self::is_allowed_with_policy(args, Self::get(args.bucket).await).await
|
||||
}
|
||||
|
||||
pub async fn try_is_allowed_for_store(store: &ECStore, args: &BucketPolicyArgs<'_>) -> Result<bool> {
|
||||
Self::is_allowed_with_policy(args, store.get_bucket_policy(args.bucket).await.map(|(policy, _)| policy)).await
|
||||
}
|
||||
|
||||
async fn is_allowed_with_policy(args: &BucketPolicyArgs<'_>, policy: Result<BucketPolicy>) -> Result<bool> {
|
||||
match policy {
|
||||
Ok(policy) => Ok(policy.is_allowed(args).await),
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::{HeaderMap, StatusCode};
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Body;
|
||||
use hyper::body::Bytes;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::http_resp_to_error_response,
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
||||
};
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn set_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
|
||||
if policy == "" {
|
||||
return self.remove_bucket_policy(bucket_name).await;
|
||||
}
|
||||
|
||||
self.put_bucket_policy(bucket_name, policy).await
|
||||
}
|
||||
|
||||
pub async fn put_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("policy".to_string(), "".to_string());
|
||||
|
||||
let mut req_metadata = RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
query_values: url_values,
|
||||
content_body: ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())),
|
||||
content_length: policy.len() as i64,
|
||||
object_name: "".to_string(),
|
||||
custom_header: HeaderMap::new(),
|
||||
content_md5_base64: "".to_string(),
|
||||
content_sha256_hex: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
};
|
||||
|
||||
let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
|
||||
//defer closeResponse(resp)
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
|
||||
//if resp != nil {
|
||||
if resp_status != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(
|
||||
resp_status,
|
||||
&h,
|
||||
vec![],
|
||||
bucket_name,
|
||||
"",
|
||||
)));
|
||||
}
|
||||
//}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn remove_bucket_policy(&self, bucket_name: &str) -> Result<(), std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("policy".to_string(), "".to_string());
|
||||
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::DELETE,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
query_values: url_values,
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
object_name: "".to_string(),
|
||||
custom_header: HeaderMap::new(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
//defer closeResponse(resp)
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
|
||||
if resp_status != StatusCode::NO_CONTENT {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(
|
||||
resp_status,
|
||||
&h,
|
||||
vec![],
|
||||
bucket_name,
|
||||
"",
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result<String, std::io::Error> {
|
||||
let bucket_policy = self.get_bucket_policy_inner(bucket_name).await?;
|
||||
Ok(bucket_policy)
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("policy".to_string(), "".to_string());
|
||||
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::GET,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
query_values: url_values,
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
object_name: "".to_string(),
|
||||
custom_header: HeaderMap::new(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
let policy = String::from_utf8_lossy(&body_vec).to_string();
|
||||
Ok(policy)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::http_resp_to_error_response,
|
||||
api_get_options::GetObjectOptions,
|
||||
transition_api::{ObjectInfo, ReaderImpl, RequestMetadata, TransitionClient},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use http_body_util::BodyExt;
|
||||
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
||||
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
|
||||
use s3s::dto::Owner;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Grantee {
|
||||
pub id: String,
|
||||
pub display_name: String,
|
||||
pub uri: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Grant {
|
||||
pub grantee: Grantee,
|
||||
pub permission: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct AccessControlList {
|
||||
pub grant: Vec<Grant>,
|
||||
pub permission: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub struct AccessControlPolicy {
|
||||
#[serde(skip)]
|
||||
owner: Owner,
|
||||
pub access_control_list: AccessControlList,
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn get_object_acl(&self, bucket_name: &str, object_name: &str) -> Result<ObjectInfo, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("acl".to_string(), "".to_string());
|
||||
let mut resp = self
|
||||
.execute_method(
|
||||
http::Method::GET,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
query_values: url_values,
|
||||
custom_header: HeaderMap::new(),
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
content_md5_base64: "".to_string(),
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
|
||||
if resp_status != http::StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(
|
||||
resp_status,
|
||||
&h,
|
||||
body_vec,
|
||||
bucket_name,
|
||||
object_name,
|
||||
)));
|
||||
}
|
||||
|
||||
let mut res = match quick_xml::de::from_str::<AccessControlPolicy>(&String::from_utf8(body_vec).unwrap()) {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
let mut obj_info = self
|
||||
.stat_object(bucket_name, object_name, &GetObjectOptions::default())
|
||||
.await?;
|
||||
|
||||
obj_info.owner.display_name = res.owner.display_name.clone();
|
||||
obj_info.owner.id = res.owner.id.clone();
|
||||
|
||||
//obj_info.grant.extend(res.access_control_list.grant);
|
||||
|
||||
let canned_acl = get_canned_acl(&res);
|
||||
if canned_acl != "" {
|
||||
obj_info
|
||||
.metadata
|
||||
.insert("X-Amz-Acl", HeaderValue::from_str(&canned_acl).unwrap());
|
||||
return Ok(obj_info);
|
||||
}
|
||||
|
||||
let grant_acl = get_amz_grant_acl(&res);
|
||||
/*for (k, v) in grant_acl {
|
||||
obj_info.metadata.insert(HeaderName::from_bytes(k.as_bytes()).unwrap(), HeaderValue::from_str(&v.to_string()).unwrap());
|
||||
}*/
|
||||
|
||||
Ok(obj_info)
|
||||
}
|
||||
}
|
||||
|
||||
fn get_canned_acl(ac_policy: &AccessControlPolicy) -> String {
|
||||
let grants = ac_policy.access_control_list.grant.clone();
|
||||
|
||||
if grants.len() == 1 {
|
||||
if grants[0].grantee.uri == "" && grants[0].permission == "FULL_CONTROL" {
|
||||
return "private".to_string();
|
||||
}
|
||||
} else if grants.len() == 2 {
|
||||
for g in grants {
|
||||
if g.grantee.uri == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && &g.permission == "READ" {
|
||||
return "authenticated-read".to_string();
|
||||
}
|
||||
if g.grantee.uri == "http://acs.amazonaws.com/groups/global/AllUsers" && &g.permission == "READ" {
|
||||
return "public-read".to_string();
|
||||
}
|
||||
if g.permission == "READ" && g.grantee.id == ac_policy.owner.id.clone().unwrap() {
|
||||
return "bucket-owner-read".to_string();
|
||||
}
|
||||
}
|
||||
} else if grants.len() == 3 {
|
||||
for g in grants {
|
||||
if g.grantee.uri == "http://acs.amazonaws.com/groups/global/AllUsers" && g.permission == "WRITE" {
|
||||
return "public-read-write".to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
"".to_string()
|
||||
}
|
||||
|
||||
pub fn get_amz_grant_acl(ac_policy: &AccessControlPolicy) -> HashMap<String, Vec<String>> {
|
||||
let grants = ac_policy.access_control_list.grant.clone();
|
||||
let mut res = HashMap::<String, Vec<String>>::new();
|
||||
|
||||
for g in grants {
|
||||
let mut id = "id=".to_string();
|
||||
id.push_str(&g.grantee.id);
|
||||
let permission: &str = &g.permission;
|
||||
match permission {
|
||||
"READ" => {
|
||||
res.entry("X-Amz-Grant-Read".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
"WRITE" => {
|
||||
res.entry("X-Amz-Grant-Write".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
"READ_ACP" => {
|
||||
res.entry("X-Amz-Grant-Read-Acp".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
"WRITE_ACP" => {
|
||||
res.entry("X-Amz-Grant-Write-Acp".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
"FULL_CONTROL" => {
|
||||
res.entry("X-Amz-Grant-Full-Control".to_string()).or_insert(vec![]).push(id);
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::client::constants::{GET_OBJECT_ATTRIBUTES_MAX_PARTS, GET_OBJECT_ATTRIBUTES_TAGS, ISO8601_DATEFORMAT};
|
||||
use crate::client::{
|
||||
api_get_object_acl::AccessControlPolicy,
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
||||
};
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Body;
|
||||
use hyper::body::Bytes;
|
||||
use hyper::body::Incoming;
|
||||
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
|
||||
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
|
||||
use s3s::header::{X_AMZ_MAX_PARTS, X_AMZ_OBJECT_ATTRIBUTES, X_AMZ_PART_NUMBER_MARKER, X_AMZ_VERSION_ID};
|
||||
|
||||
pub struct ObjectAttributesOptions {
|
||||
pub max_parts: i64,
|
||||
pub version_id: String,
|
||||
pub part_number_marker: i64,
|
||||
//server_side_encryption: encrypt::ServerSide,
|
||||
}
|
||||
|
||||
pub struct ObjectAttributes {
|
||||
pub version_id: String,
|
||||
pub last_modified: OffsetDateTime,
|
||||
pub object_attributes_response: ObjectAttributesResponse,
|
||||
}
|
||||
|
||||
impl ObjectAttributes {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
version_id: "".to_string(),
|
||||
last_modified: OffsetDateTime::now_utc(),
|
||||
object_attributes_response: ObjectAttributesResponse::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub struct Checksum {
|
||||
checksum_crc32: String,
|
||||
checksum_crc32c: String,
|
||||
checksum_sha1: String,
|
||||
checksum_sha256: String,
|
||||
}
|
||||
|
||||
impl Checksum {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
checksum_crc32: "".to_string(),
|
||||
checksum_crc32c: "".to_string(),
|
||||
checksum_sha1: "".to_string(),
|
||||
checksum_sha256: "".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub struct ObjectParts {
|
||||
pub parts_count: i64,
|
||||
pub part_number_marker: i64,
|
||||
pub next_part_number_marker: i64,
|
||||
pub max_parts: i64,
|
||||
is_truncated: bool,
|
||||
parts: Vec<ObjectAttributePart>,
|
||||
}
|
||||
|
||||
impl ObjectParts {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
parts_count: 0,
|
||||
part_number_marker: 0,
|
||||
next_part_number_marker: 0,
|
||||
max_parts: 0,
|
||||
is_truncated: false,
|
||||
parts: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
pub struct ObjectAttributesResponse {
|
||||
pub etag: String,
|
||||
pub storage_class: String,
|
||||
pub object_size: i64,
|
||||
pub checksum: Checksum,
|
||||
pub object_parts: ObjectParts,
|
||||
}
|
||||
|
||||
impl ObjectAttributesResponse {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
etag: "".to_string(),
|
||||
storage_class: "".to_string(),
|
||||
object_size: 0,
|
||||
checksum: Checksum::new(),
|
||||
object_parts: ObjectParts::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
struct ObjectAttributePart {
|
||||
checksum_crc32: String,
|
||||
checksum_crc32c: String,
|
||||
checksum_sha1: String,
|
||||
checksum_sha256: String,
|
||||
part_number: i64,
|
||||
size: i64,
|
||||
}
|
||||
|
||||
impl ObjectAttributes {
|
||||
pub async fn parse_response(&mut self, h: &HeaderMap, body_vec: Vec<u8>) -> Result<(), std::io::Error> {
|
||||
let last_modified = h
|
||||
.get("Last-Modified")
|
||||
.ok_or_else(|| std::io::Error::other("missing Last-Modified header"))?
|
||||
.to_str()
|
||||
.map_err(|e| std::io::Error::other(format!("invalid Last-Modified header: {e}")))?;
|
||||
let mod_time = OffsetDateTime::parse(last_modified, ISO8601_DATEFORMAT)
|
||||
.map_err(|e| std::io::Error::other(format!("invalid Last-Modified date: {e}")))?;
|
||||
self.last_modified = mod_time;
|
||||
|
||||
let version_id = h
|
||||
.get(X_AMZ_VERSION_ID)
|
||||
.ok_or_else(|| std::io::Error::other("missing version ID header"))?
|
||||
.to_str()
|
||||
.map_err(|e| std::io::Error::other(format!("invalid version ID header: {e}")))?;
|
||||
self.version_id = version_id.to_string();
|
||||
|
||||
let body_str = String::from_utf8(body_vec).map_err(|e| std::io::Error::other(format!("invalid UTF-8 body: {e}")))?;
|
||||
let mut response = match quick_xml::de::from_str::<ObjectAttributesResponse>(&body_str) {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
self.object_attributes_response = response;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn get_object_attributes(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
opts: ObjectAttributesOptions,
|
||||
) -> Result<ObjectAttributes, std::io::Error> {
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("attributes".to_string(), "".to_string());
|
||||
if opts.version_id != "" {
|
||||
url_values.insert("versionId".to_string(), opts.version_id);
|
||||
}
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
X_AMZ_OBJECT_ATTRIBUTES,
|
||||
HeaderValue::from_str(GET_OBJECT_ATTRIBUTES_TAGS).expect("valid header value"),
|
||||
);
|
||||
|
||||
if opts.part_number_marker > 0 {
|
||||
headers.insert(
|
||||
X_AMZ_PART_NUMBER_MARKER,
|
||||
HeaderValue::from_str(&opts.part_number_marker.to_string()).expect("valid header value"),
|
||||
);
|
||||
}
|
||||
|
||||
if opts.max_parts > 0 {
|
||||
headers.insert(
|
||||
X_AMZ_MAX_PARTS,
|
||||
HeaderValue::from_str(&opts.max_parts.to_string()).expect("valid header value"),
|
||||
);
|
||||
} else {
|
||||
headers.insert(
|
||||
X_AMZ_MAX_PARTS,
|
||||
HeaderValue::from_str(&GET_OBJECT_ATTRIBUTES_MAX_PARTS.to_string()).expect("valid header value"),
|
||||
);
|
||||
}
|
||||
|
||||
/*if opts.server_side_encryption.is_some() {
|
||||
opts.server_side_encryption.Marshal(headers);
|
||||
}*/
|
||||
|
||||
let mut resp = self
|
||||
.execute_method(
|
||||
http::Method::HEAD,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
query_values: url_values,
|
||||
custom_header: headers,
|
||||
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
|
||||
content_md5_base64: "".to_string(),
|
||||
content_body: ReaderImpl::Body(Bytes::new()),
|
||||
content_length: 0,
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
let has_etag = h.get("ETag").and_then(|v| v.to_str().ok()).unwrap_or("");
|
||||
if !has_etag.is_empty() {
|
||||
return Err(std::io::Error::other(
|
||||
"get_object_attributes is not supported by the current endpoint version",
|
||||
));
|
||||
}
|
||||
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
|
||||
if resp_status != http::StatusCode::OK {
|
||||
let err_body =
|
||||
String::from_utf8(body_vec).map_err(|e| std::io::Error::other(format!("invalid UTF-8 error body: {e}")))?;
|
||||
let mut er = match quick_xml::de::from_str::<AccessControlPolicy>(&err_body) {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
return Err(std::io::Error::other(err.to_string()));
|
||||
}
|
||||
};
|
||||
|
||||
return Err(std::io::Error::other(er.access_control_list.permission));
|
||||
}
|
||||
|
||||
let mut oa = ObjectAttributes::new();
|
||||
oa.parse_response(&h, body_vec).await?;
|
||||
|
||||
Ok(oa)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[cfg(not(windows))]
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
use tokio::fs::{self, OpenOptions};
|
||||
use tokio::io::{AsyncSeekExt, AsyncWriteExt, SeekFrom};
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::err_invalid_argument, api_get_options::GetObjectOptions, transition_api::TransitionClient,
|
||||
};
|
||||
|
||||
async fn prepare_download_target(file_path: &Path) -> io::Result<()> {
|
||||
match fs::metadata(file_path).await {
|
||||
Ok(metadata) if metadata.is_dir() => {
|
||||
return Err(io::Error::other(err_invalid_argument("filename is a directory.")));
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
if let Some(parent) = file_path.parent()
|
||||
&& !parent.as_os_str().is_empty()
|
||||
{
|
||||
fs::create_dir_all(parent).await?;
|
||||
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
let mut permissions = fs::metadata(parent).await?.permissions();
|
||||
permissions.set_mode(0o700);
|
||||
fs::set_permissions(parent, permissions).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn build_part_path(file_path: &Path) -> PathBuf {
|
||||
PathBuf::from(format!("{}.part.rustfs", file_path.display()))
|
||||
}
|
||||
|
||||
async fn open_download_part_file(file_part_path: &Path) -> io::Result<tokio::fs::File> {
|
||||
let mut options = OpenOptions::new();
|
||||
options.create(true).truncate(false).read(true).write(true);
|
||||
|
||||
#[cfg(not(windows))]
|
||||
options.mode(0o600);
|
||||
|
||||
options.open(file_part_path).await
|
||||
}
|
||||
|
||||
async fn cleanup_part_file(file_part_path: &Path) {
|
||||
let _ = fs::remove_file(file_part_path).await;
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn fget_object(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
file_path: &str,
|
||||
mut opts: GetObjectOptions,
|
||||
) -> Result<(), io::Error> {
|
||||
let file_path = Path::new(file_path);
|
||||
prepare_download_target(file_path).await?;
|
||||
|
||||
let file_part_path = build_part_path(file_path);
|
||||
let mut file_part = open_download_part_file(&file_part_path).await?;
|
||||
let existing_len = file_part.metadata().await?.len();
|
||||
if existing_len > 0 {
|
||||
opts.set_range(existing_len as i64, 0)?;
|
||||
file_part.seek(SeekFrom::Start(existing_len)).await?;
|
||||
}
|
||||
|
||||
let (_object_info, _headers, mut object_reader) = self.get_object_inner(bucket_name, object_name, &opts).await?;
|
||||
if let Err(err) = tokio::io::copy(&mut object_reader, &mut file_part).await {
|
||||
cleanup_part_file(&file_part_path).await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if let Err(err) = file_part.flush().await {
|
||||
cleanup_part_file(&file_part_path).await;
|
||||
return Err(err);
|
||||
}
|
||||
drop(file_part);
|
||||
|
||||
if let Err(err) = fs::rename(&file_part_path, file_path).await {
|
||||
cleanup_part_file(&file_part_path).await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepare_download_target_allows_missing_file_and_creates_parent_dirs() {
|
||||
let dir = tempdir().expect("temp dir");
|
||||
let target = dir.path().join("nested").join("object.bin");
|
||||
|
||||
prepare_download_target(&target)
|
||||
.await
|
||||
.expect("missing target should be accepted");
|
||||
|
||||
assert!(target.parent().expect("parent").exists(), "parent directory should be created");
|
||||
assert!(
|
||||
fs::metadata(&target).await.is_err(),
|
||||
"preparing the target should not create the final file eagerly"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepare_download_target_rejects_directory_paths() {
|
||||
let dir = tempdir().expect("temp dir");
|
||||
let target_dir = dir.path().join("download-dir");
|
||||
fs::create_dir_all(&target_dir).await.expect("target dir");
|
||||
|
||||
let err = prepare_download_target(&target_dir)
|
||||
.await
|
||||
.expect_err("directory targets must be rejected");
|
||||
|
||||
assert!(err.to_string().contains("directory"), "unexpected error for directory target: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn open_download_part_file_creates_part_file() {
|
||||
let dir = tempdir().expect("temp dir");
|
||||
let target = dir.path().join("object.bin");
|
||||
let part_path = build_part_path(&target);
|
||||
|
||||
let file = open_download_part_file(&part_path)
|
||||
.await
|
||||
.expect("part file should be created");
|
||||
drop(file);
|
||||
|
||||
assert!(part_path.exists(), "part file should exist after creation");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use crate::client::{
|
||||
api_error_response::{err_invalid_argument, http_resp_to_error_response},
|
||||
api_get_object_acl::AccessControlList,
|
||||
api_get_options::GetObjectOptions,
|
||||
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::body::Body;
|
||||
use hyper::body::Bytes;
|
||||
use s3s::dto::RestoreRequest;
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::BufReader;
|
||||
|
||||
const TIER_STANDARD: &str = "Standard";
|
||||
const TIER_BULK: &str = "Bulk";
|
||||
const TIER_EXPEDITED: &str = "Expedited";
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Encryption {
|
||||
pub encryption_type: String,
|
||||
pub kms_context: String,
|
||||
pub kms_key_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct MetadataEntry {
|
||||
pub name: String,
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, serde::Serialize)]
|
||||
pub struct S3 {
|
||||
pub access_control_list: AccessControlList,
|
||||
pub bucket_name: String,
|
||||
pub prefix: String,
|
||||
pub canned_acl: String,
|
||||
pub encryption: Encryption,
|
||||
pub storage_class: String,
|
||||
//tagging: Tags,
|
||||
pub user_metadata: MetadataEntry,
|
||||
}
|
||||
|
||||
impl TransitionClient {
|
||||
pub async fn restore_object(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
object_name: &str,
|
||||
version_id: &str,
|
||||
restore_req: &RestoreRequest,
|
||||
) -> Result<(), std::io::Error> {
|
||||
/*let restore_request = match quick_xml::se::to_string(restore_req) {
|
||||
Ok(buf) => buf,
|
||||
Err(e) => {
|
||||
return Err(std::io::Error::other(e));
|
||||
}
|
||||
};*/
|
||||
let restore_request = "".to_string();
|
||||
let restore_request_bytes = restore_request.as_bytes().to_vec();
|
||||
|
||||
let mut url_values = HashMap::new();
|
||||
url_values.insert("restore".to_string(), "".to_string());
|
||||
if version_id != "" {
|
||||
url_values.insert("versionId".to_string(), version_id.to_string());
|
||||
}
|
||||
|
||||
let restore_request_buffer = Bytes::from(restore_request_bytes.clone());
|
||||
let resp = self
|
||||
.execute_method(
|
||||
http::Method::HEAD,
|
||||
&mut RequestMetadata {
|
||||
bucket_name: bucket_name.to_string(),
|
||||
object_name: object_name.to_string(),
|
||||
query_values: url_values,
|
||||
custom_header: HeaderMap::new(),
|
||||
content_sha256_hex: "".to_string(), //sum_sha256_hex(&restore_request_bytes),
|
||||
content_md5_base64: "".to_string(), //sum_md5_base64(&restore_request_bytes),
|
||||
content_body: ReaderImpl::Body(restore_request_buffer),
|
||||
content_length: restore_request_bytes.len() as i64,
|
||||
stream_sha256: false,
|
||||
trailer: HeaderMap::new(),
|
||||
pre_sign_url: Default::default(),
|
||||
add_crc: Default::default(),
|
||||
extra_pre_sign_header: Default::default(),
|
||||
bucket_location: Default::default(),
|
||||
expires: Default::default(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
let resp_status = resp.status();
|
||||
let h = resp.headers().clone();
|
||||
|
||||
let mut body_vec = Vec::new();
|
||||
let mut body = resp.into_body();
|
||||
while let Some(frame) = body.frame().await {
|
||||
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
|
||||
if let Some(data) = frame.data_ref() {
|
||||
body_vec.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
if resp_status != http::StatusCode::ACCEPTED && resp_status != http::StatusCode::OK {
|
||||
return Err(std::io::Error::other(http_resp_to_error_response(
|
||||
resp_status,
|
||||
&h,
|
||||
body_vec,
|
||||
bucket_name,
|
||||
"",
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -37,3 +37,6 @@ pub const TOTAL_WORKERS: i64 = 4;
|
||||
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
|
||||
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
|
||||
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
|
||||
|
||||
pub const GET_OBJECT_ATTRIBUTES_TAGS: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts";
|
||||
pub const GET_OBJECT_ATTRIBUTES_MAX_PARTS: i64 = 1000;
|
||||
|
||||
@@ -16,8 +16,12 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod admin_handler_utils;
|
||||
pub mod api_bucket_policy;
|
||||
pub mod api_error_response;
|
||||
pub mod api_get_object;
|
||||
pub mod api_get_object_acl;
|
||||
pub mod api_get_object_attributes;
|
||||
pub mod api_get_object_file;
|
||||
pub mod api_get_options;
|
||||
pub mod api_list;
|
||||
pub mod api_put_object;
|
||||
@@ -25,6 +29,7 @@ pub mod api_put_object_common;
|
||||
pub mod api_put_object_multipart;
|
||||
pub mod api_put_object_streaming;
|
||||
pub mod api_remove;
|
||||
pub mod api_restore;
|
||||
pub mod api_s3_datatypes;
|
||||
pub mod api_stat;
|
||||
pub mod bucket_cache;
|
||||
|
||||
@@ -1006,6 +1006,16 @@ impl TransitionCore {
|
||||
client.abort_multipart_upload(bucket_name, object, upload_id).await
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result<String, std::io::Error> {
|
||||
let client = self.0.clone();
|
||||
client.get_bucket_policy(bucket_name).await
|
||||
}
|
||||
|
||||
pub async fn put_bucket_policy(&self, bucket_name: &str, bucket_policy: &str) -> Result<(), std::io::Error> {
|
||||
let client = self.0.clone();
|
||||
client.put_bucket_policy(bucket_name, bucket_policy).await
|
||||
}
|
||||
|
||||
pub async fn get_object(
|
||||
&self,
|
||||
bucket_name: &str,
|
||||
|
||||
@@ -1638,7 +1638,7 @@ fn preserve_unknown_dirty_usage(
|
||||
Some(preserved)
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[cfg(test)]
|
||||
async fn replace_bucket_usage_memory_from_authoritative(bucket: &str, usage: BucketUsageInfo, refresh_started_at: SystemTime) {
|
||||
let mut cache = memory_cache().write().await;
|
||||
if let Some(existing) = cache.get(bucket)
|
||||
@@ -1650,19 +1650,6 @@ async fn replace_bucket_usage_memory_from_authoritative(bucket: &str, usage: Buc
|
||||
cache.insert(bucket.to_string(), cached_bucket_usage_from_backend(usage, refresh_started_at, true));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub async fn seed_bucket_usage_memory_for_test(bucket: &str, size: u64) {
|
||||
replace_bucket_usage_memory_from_authoritative(
|
||||
bucket,
|
||||
BucketUsageInfo {
|
||||
size,
|
||||
..Default::default()
|
||||
},
|
||||
SystemTime::now(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Fast in-memory update for immediate quota and admin usage consistency.
|
||||
pub async fn record_bucket_object_write_memory(bucket: &str, previous_current_size: Option<u64>, new_size: u64) {
|
||||
record_bucket_object_write_memory_inner(bucket, previous_current_size, new_size, false).await;
|
||||
@@ -2150,6 +2137,30 @@ pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str
|
||||
Ok(d)
|
||||
}
|
||||
|
||||
#[instrument(skip(cache))]
|
||||
pub async fn save_data_usage_cache(cache: &DataUsageCache, name: &str) -> crate::error::Result<()> {
|
||||
use crate::config::com::save_config;
|
||||
use crate::disk::BUCKET_META_PREFIX;
|
||||
use std::path::Path;
|
||||
|
||||
let Some(store) = runtime_sources::object_store_handle() else {
|
||||
return Err(Error::other("errServerNotInitialized"));
|
||||
};
|
||||
let buf = cache.marshal_msg().map_err(Error::other)?;
|
||||
let buf_clone = buf.clone();
|
||||
|
||||
let store_clone = store.clone();
|
||||
|
||||
let name = Path::new(BUCKET_META_PREFIX).join(name).to_string_lossy().to_string();
|
||||
|
||||
let name_clone = name.clone();
|
||||
tokio::spawn(async move {
|
||||
let _ = save_config(store_clone, &format!("{}{}", name_clone, ".bkp"), buf_clone).await;
|
||||
});
|
||||
save_config(store, &name, buf).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Persist the current in-memory compression total to the backend.
|
||||
/// Resets the debounce counter so the next auto-persist won't fire
|
||||
/// immediately after this manual flush (intended for shutdown paths).
|
||||
|
||||
@@ -12,9 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::cluster::rpc::{
|
||||
ScannerBucketListing, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
};
|
||||
use crate::cluster::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend_cached};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::{
|
||||
@@ -25,7 +23,6 @@ use crate::{
|
||||
|
||||
use crate::data_usage::load_data_usage_cache;
|
||||
use crate::storage_api_contracts::admin::StorageAdminApi;
|
||||
use crate::storage_api_contracts::bucket::BucketOptions;
|
||||
use rustfs_common::heal_channel::DriveState;
|
||||
use rustfs_madmin::{
|
||||
BackendDisks, Disk, ErasureSetInfo, ITEM_INITIALIZING, ITEM_OFFLINE, ITEM_ONLINE, ITEM_UNKNOWN, InfoMessage, MemStats,
|
||||
@@ -77,19 +74,6 @@ fn apply_data_usage_result(
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_bucket_namespace_count(result: Result<ScannerBucketListing>, buckets: &mut rustfs_madmin::Buckets) {
|
||||
if let Ok(listing) = result
|
||||
&& listing.topology_complete
|
||||
{
|
||||
let count = listing.buckets.iter().filter(|bucket| !bucket.name.starts_with('.')).count();
|
||||
let Ok(count) = u64::try_from(count) else {
|
||||
return;
|
||||
};
|
||||
buckets.count = count;
|
||||
buckets.error = None;
|
||||
}
|
||||
}
|
||||
|
||||
// pub const ITEM_OFFLINE: &str = "offline";
|
||||
// pub const ITEM_INITIALIZING: &str = "initializing";
|
||||
// pub const ITEM_ONLINE: &str = "online";
|
||||
@@ -301,18 +285,6 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
&mut delete_markers,
|
||||
&mut usage,
|
||||
);
|
||||
if buckets.error.is_some() {
|
||||
apply_bucket_namespace_count(
|
||||
store
|
||||
.list_bucket_for_scanner(&BucketOptions {
|
||||
cached: true,
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await,
|
||||
&mut buckets,
|
||||
);
|
||||
}
|
||||
|
||||
let after3 = OffsetDateTime::now_utc();
|
||||
|
||||
@@ -733,13 +705,12 @@ mod tests {
|
||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::storage_api_contracts::bucket::BucketInfo;
|
||||
use rustfs_madmin::{Disk, ITEM_OFFLINE, ITEM_ONLINE, ITEM_UNKNOWN, ServerProperties};
|
||||
|
||||
use super::{
|
||||
DATA_USAGE_ROOT, DATA_USAGE_UNAVAILABLE_ERROR, apply_bucket_namespace_count, apply_data_usage_result,
|
||||
apply_erasure_set_usage, get_local_server_property, get_online_offline_disks_stats, get_server_info,
|
||||
reconcile_servers_with_endpoint_topology, server_topology_completeness_report,
|
||||
DATA_USAGE_ROOT, DATA_USAGE_UNAVAILABLE_ERROR, apply_data_usage_result, apply_erasure_set_usage,
|
||||
get_local_server_property, get_online_offline_disks_stats, get_server_info, reconcile_servers_with_endpoint_topology,
|
||||
server_topology_completeness_report,
|
||||
};
|
||||
|
||||
fn disk_with_state(endpoint: &str, state: &str) -> Disk {
|
||||
@@ -989,75 +960,6 @@ mod tests {
|
||||
assert_eq!(usage.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn live_bucket_namespace_count_survives_unavailable_data_usage() {
|
||||
let mut buckets = rustfs_madmin::Buckets {
|
||||
count: 0,
|
||||
error: Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
|
||||
};
|
||||
|
||||
apply_bucket_namespace_count(
|
||||
Ok(crate::cluster::rpc::ScannerBucketListing {
|
||||
buckets: vec![
|
||||
BucketInfo {
|
||||
name: "bucket-a".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
BucketInfo {
|
||||
name: ".rustfs.sys".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
BucketInfo {
|
||||
name: "bucket-b".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
set_buckets: Vec::new(),
|
||||
topology_complete: true,
|
||||
}),
|
||||
&mut buckets,
|
||||
);
|
||||
|
||||
assert_eq!(buckets.count, 2);
|
||||
assert_eq!(buckets.error, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incomplete_bucket_namespace_lookup_preserves_usage_state() {
|
||||
let mut buckets = rustfs_madmin::Buckets {
|
||||
count: 7,
|
||||
error: Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
|
||||
};
|
||||
|
||||
apply_bucket_namespace_count(
|
||||
Ok(crate::cluster::rpc::ScannerBucketListing {
|
||||
buckets: vec![BucketInfo {
|
||||
name: "bucket-a".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
set_buckets: Vec::new(),
|
||||
topology_complete: false,
|
||||
}),
|
||||
&mut buckets,
|
||||
);
|
||||
|
||||
assert_eq!(buckets.count, 7);
|
||||
assert_eq!(buckets.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_bucket_namespace_lookup_preserves_usage_state() {
|
||||
let mut buckets = rustfs_madmin::Buckets {
|
||||
count: 7,
|
||||
error: Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
|
||||
};
|
||||
|
||||
apply_bucket_namespace_count(Err(crate::error::Error::DiskNotFound), &mut buckets);
|
||||
|
||||
assert_eq!(buckets.count, 7);
|
||||
assert_eq!(buckets.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incomplete_erasure_set_cache_is_not_reported_as_zero() {
|
||||
let mut cache = rustfs_data_usage::DataUsageCache::default();
|
||||
|
||||
@@ -331,14 +331,7 @@ impl From<std::io::Error> for DiskError {
|
||||
}
|
||||
match e.downcast::<DiskError>() {
|
||||
Ok(disk_error) => disk_error,
|
||||
// Mirror `From<io::Error> for StorageError`: a StorageError boxed
|
||||
// through `From<StorageError> for io::Error` must recover its typed
|
||||
// classification instead of degrading to `DiskError::Io`, which
|
||||
// quorum aggregation (`reduce_errs`) would count as a distinct error.
|
||||
Err(io_error) => match io_error.downcast::<crate::error::StorageError>() {
|
||||
Ok(storage_error) => storage_error.into(),
|
||||
Err(io_error) => DiskError::Io(io_error),
|
||||
},
|
||||
Err(io_error) => DiskError::Io(io_error),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -960,27 +953,6 @@ mod tests {
|
||||
assert_eq!(original_disk_error, recovered_disk_error);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_io_error_with_storage_error_inside() {
|
||||
use crate::error::StorageError;
|
||||
|
||||
// An io::Error boxing a disk-representable StorageError (as produced by
|
||||
// `From<StorageError> for io::Error`) must recover the typed DiskError
|
||||
// variant instead of degrading to an opaque DiskError::Io.
|
||||
let io_with_storage_error: std::io::Error = StorageError::FaultyRemoteDisk.into();
|
||||
let recovered: DiskError = io_with_storage_error.into();
|
||||
assert_eq!(recovered, DiskError::FaultyRemoteDisk);
|
||||
|
||||
let io_with_storage_error: std::io::Error = StorageError::FileAccessDenied.into();
|
||||
let recovered: DiskError = io_with_storage_error.into();
|
||||
assert_eq!(recovered, DiskError::FileAccessDenied);
|
||||
|
||||
// A StorageError with no DiskError analog stays an opaque Io error.
|
||||
let io_with_bucket_error: std::io::Error = StorageError::BucketNotFound("bucket".to_string()).into();
|
||||
let recovered: DiskError = io_with_bucket_error.into();
|
||||
assert!(matches!(recovered, DiskError::Io(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_io_error_different_kinds() {
|
||||
use std::io::ErrorKind;
|
||||
|
||||
@@ -2027,9 +2027,6 @@ type InlinePreparationHook = Box<dyn FnOnce() + Send>;
|
||||
static INLINE_PREPARATION_BEFORE_BACKUP: std::sync::LazyLock<std::sync::Mutex<HashMap<String, InlinePreparationHook>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
#[cfg(test)]
|
||||
static INLINE_BEFORE_FILE_SYNC_ADMISSION: std::sync::LazyLock<std::sync::Mutex<HashMap<String, InlinePreparationHook>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
#[cfg(test)]
|
||||
static RENAME_DATA_AFTER_FIRST_PUBLICATION: std::sync::LazyLock<std::sync::Mutex<HashMap<String, InlinePreparationHook>>> =
|
||||
std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
|
||||
#[cfg(test)]
|
||||
@@ -2094,14 +2091,6 @@ fn set_inline_preparation_before_backup(dst_path: &str, hook: impl FnOnce() + Se
|
||||
.insert(dst_path.to_string(), Box::new(hook));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn set_inline_before_file_sync_admission(dst_path: &str, hook: impl FnOnce() + Send + 'static) {
|
||||
INLINE_BEFORE_FILE_SYNC_ADMISSION
|
||||
.lock()
|
||||
.expect("test admission hook lock should not be poisoned")
|
||||
.insert(dst_path.to_string(), Box::new(hook));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn set_rename_data_after_first_publication(dst_path: &str, hook: impl FnOnce() + Send + 'static) {
|
||||
RENAME_DATA_AFTER_FIRST_PUBLICATION
|
||||
@@ -2247,17 +2236,6 @@ fn run_inline_preparation_before_backup(dst_path: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn run_inline_before_file_sync_admission(dst_path: &str) {
|
||||
let hook = INLINE_BEFORE_FILE_SYNC_ADMISSION
|
||||
.lock()
|
||||
.expect("test admission hook lock should not be poisoned")
|
||||
.remove(dst_path);
|
||||
if let Some(hook) = hook {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn run_rename_data_after_first_publication(dst_path: &str) {
|
||||
let hook = RENAME_DATA_AFTER_FIRST_PUBLICATION
|
||||
@@ -2929,82 +2907,23 @@ pub(crate) trait LocalIoBackend: Send + Sync + Debug + 'static {
|
||||
/// Default [`LocalIoBackend`]: tokio blocking-pool file I/O plus the
|
||||
/// mmap-copy / direct-read-copy positioned read, moved verbatim from the
|
||||
/// former `DiskAPI` method bodies on `LocalDisk`.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct StdBackend {
|
||||
root: PathBuf,
|
||||
#[cfg(target_os = "linux")]
|
||||
direct_io: Arc<DirectIoReadState>,
|
||||
#[cfg(target_os = "linux")]
|
||||
direct_io_write: Arc<DirectIoWriteState>,
|
||||
/// Per-disk descriptor cache for buffered reads (rustfs/backlog#1801).
|
||||
/// `None` when disabled by env, blocked by a low `RLIMIT_NOFILE`, or on
|
||||
/// non-Linux (where the cache type is unavailable). Like the io_uring
|
||||
/// cache, only the buffered read path populates it; O_DIRECT reads keep
|
||||
/// opening their own aligned descriptors.
|
||||
#[cfg(target_os = "linux")]
|
||||
fd_cache: Option<FdCache>,
|
||||
}
|
||||
|
||||
// Manual `Debug` mirrors `UringBackend`: the fd cache (and the Linux-only
|
||||
// direct-IO state) hold types that do not implement `Debug`, so a derive would
|
||||
// force `FdCache: Debug`. `finish_non_exhaustive` skips them.
|
||||
impl std::fmt::Debug for StdBackend {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("StdBackend").field("root", &self.root).finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl StdBackend {
|
||||
pub(crate) fn new(root: PathBuf) -> Self {
|
||||
Self::build(root, true)
|
||||
}
|
||||
|
||||
/// Construct without the descriptor cache.
|
||||
///
|
||||
/// `UringBackend` wraps a `StdBackend` and runs its own `FdCache` over the
|
||||
/// same positioned reads. If the inner `StdBackend` also built a cache, a
|
||||
/// fallback read (`UringBackend::pread_bytes` delegates to the inner backend
|
||||
/// on latch-off / O_DIRECT / buffered errors) would populate a *second*
|
||||
/// cache that `UringBackend`'s invalidation never touches — re-opening the
|
||||
/// stale-inode hazard `FdCache` exists to close (rustfs/backlog#1176/#1801).
|
||||
/// The wrapper therefore owns the only cache for the disk; the inner backend
|
||||
/// opens per read. This also avoids double-counting `FD_CACHE_CAPACITY`
|
||||
/// against `RLIMIT_NOFILE` (rustfs/backlog#1178).
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(crate) fn new_without_fd_cache(root: PathBuf) -> Self {
|
||||
Self::build(root, false)
|
||||
}
|
||||
|
||||
fn build(root: PathBuf, build_fd_cache: bool) -> Self {
|
||||
// Gate the fd cache on RLIMIT_NOFILE headroom (rustfs/backlog#1178):
|
||||
// 512 fds/disk with a low soft limit and several disks would hit EMFILE.
|
||||
// Fall back to open-per-read when the limit is too small.
|
||||
#[cfg(target_os = "linux")]
|
||||
let fd_cache = if build_fd_cache && is_local_fd_cache_enabled() {
|
||||
if rlimit_allows_fd_cache() {
|
||||
Some(FdCache::new())
|
||||
} else {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
"std fd cache disabled: RLIMIT_NOFILE soft limit too low for 512 fds/disk; using open-per-read"
|
||||
);
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// `build_fd_cache` is only consulted on Linux (for the fd cache); on
|
||||
// other platforms it has no effect and would trip the unused-variable lint.
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let _ = build_fd_cache;
|
||||
Self {
|
||||
root,
|
||||
#[cfg(target_os = "linux")]
|
||||
direct_io: Arc::new(DirectIoReadState::new()),
|
||||
#[cfg(target_os = "linux")]
|
||||
direct_io_write: Arc::new(DirectIoWriteState::new()),
|
||||
#[cfg(target_os = "linux")]
|
||||
fd_cache,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3082,9 +3001,6 @@ impl LocalIoBackend for StdBackend {
|
||||
direct_read_copy_fault_delta: MmapPageFaultDelta,
|
||||
blocking_task_duration: StdDuration,
|
||||
used_direct_io: bool,
|
||||
/// The descriptor opened by THIS call (None on a cache hit), handed
|
||||
/// back so the async caller can index it in the fd cache.
|
||||
opened_fd: Option<Arc<std::fs::File>>,
|
||||
}
|
||||
|
||||
enum MmapCopyReadError {
|
||||
@@ -3112,72 +3028,28 @@ impl LocalIoBackend for StdBackend {
|
||||
let direct_io_state = self.direct_io.clone();
|
||||
let offset_u64 = u64::try_from(offset).map_err(|_| DiskError::FileCorrupt)?;
|
||||
let end_offset_u64 = u64::try_from(end_offset).map_err(|_| DiskError::FileCorrupt)?;
|
||||
|
||||
// Descriptor cache (rustfs/backlog#1801): on a hit the read reuses an
|
||||
// already-open descriptor (via dup below) and skips `access` +
|
||||
// `File::open`. Linux-only — on other Unix `cached_fd` is None and the
|
||||
// read opens per call exactly as before. `fd_lookup` snapshots the
|
||||
// invalidation generation BEFORE the open so a heal/delete that lands
|
||||
// while the blocking open is in flight prevents the now-stale descriptor
|
||||
// from being inserted (rustfs/backlog#1176).
|
||||
#[cfg(target_os = "linux")]
|
||||
let fd_lookup = self.fd_cache.as_ref().map(|cache| {
|
||||
let key = FdKey {
|
||||
volume: volume.to_owned(),
|
||||
path: path.to_owned(),
|
||||
direct: false,
|
||||
};
|
||||
let gen_at_open = cache.generation();
|
||||
(cache, key, gen_at_open)
|
||||
});
|
||||
#[cfg(target_os = "linux")]
|
||||
let cached_fd: Option<Arc<std::fs::File>> = match &fd_lookup {
|
||||
Some((cache, key, _)) => cache.get(key).await,
|
||||
None => None,
|
||||
};
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let cached_fd: Option<Arc<std::fs::File>> = None;
|
||||
|
||||
let blocking_wait_start = metrics_enabled.then(std::time::Instant::now);
|
||||
let read_result = tokio::task::spawn_blocking(move || {
|
||||
let blocking_task_start = metrics_enabled.then(StdInstant::now);
|
||||
|
||||
// Resolve the part path unconditionally: the O_DIRECT branch (large
|
||||
// reads) opens its own aligned descriptor by path even on a cache hit.
|
||||
let access_check_start = metrics_enabled.then(StdInstant::now);
|
||||
let volume_dir = local_disk_bucket_path(&root, &volume_owned)?;
|
||||
if !skip_access_checks(&volume_owned) {
|
||||
crate::disk::fs::access_std(&volume_dir)
|
||||
.map_err(|e| DiskError::from(to_access_error(e, DiskError::VolumeAccessDenied)))?;
|
||||
}
|
||||
let access_check_duration = access_check_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
|
||||
let path_resolve_start = metrics_enabled.then(StdInstant::now);
|
||||
let file_path = local_disk_object_path(&root, &volume_owned, &path_owned)?;
|
||||
check_path_length(file_path.to_string_lossy().as_ref())?;
|
||||
let path_resolve_duration = path_resolve_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
|
||||
let file_open_start = metrics_enabled.then(StdInstant::now);
|
||||
// Acquire the read handle (rustfs/backlog#1801). On a descriptor-cache
|
||||
// hit this reuses the cached descriptor via `dup` (one syscall, no path
|
||||
// resolution or permission re-check) and skips the volume access probe;
|
||||
// on a miss it resolves the volume, access-checks, and opens the file.
|
||||
// `File::try_clone` shares the cached descriptor's open-file offset, so
|
||||
// the read below is positioned (mmap offset argument / `read_exact_at`)
|
||||
// and never depends on the descriptor's current offset. `cached_fd` being
|
||||
// None also marks this call as a miss for the cache-insert side-channel.
|
||||
let (file, access_check_duration) = if let Some(cached) = cached_fd.as_ref() {
|
||||
(cached.as_ref().try_clone().map_err(DiskError::from)?, StdDuration::ZERO)
|
||||
} else {
|
||||
// Measure the volume access probe only — the part-path resolution
|
||||
// above is accounted in `path_resolve_duration` (rustfs/backlog#1801).
|
||||
let access_check_start = metrics_enabled.then(StdInstant::now);
|
||||
let volume_dir = local_disk_bucket_path(&root, &volume_owned)?;
|
||||
if !skip_access_checks(&volume_owned) {
|
||||
crate::disk::fs::access_std(&volume_dir)
|
||||
.map_err(|e| DiskError::from(to_access_error(e, DiskError::VolumeAccessDenied)))?;
|
||||
}
|
||||
let access_check_duration = access_check_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
(std::fs::File::open(&file_path).map_err(DiskError::from)?, access_check_duration)
|
||||
};
|
||||
let mut file = std::fs::File::open(&file_path).map_err(DiskError::from)?;
|
||||
let file_open_duration = file_open_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
|
||||
let metadata_lookup_start = metrics_enabled.then(StdInstant::now);
|
||||
// On a cache hit this fstats the cached descriptor — the inode it was
|
||||
// opened against, which invalidation keeps current for live entries. EC
|
||||
// shards are fixed-length, so a still-cached pre-heal length is benign.
|
||||
let meta = file.metadata().map_err(DiskError::from)?;
|
||||
let metadata_lookup_duration = metadata_lookup_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
|
||||
@@ -3288,15 +3160,13 @@ impl LocalIoBackend for StdBackend {
|
||||
bytes
|
||||
}
|
||||
LocalReadCopyMethod::DirectReadCopy => {
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::io::{Read as _, Seek as _};
|
||||
|
||||
let direct_read_copy_start = metrics_enabled.then(StdInstant::now);
|
||||
let direct_read_copy_faults_before = read_mmap_page_fault_counts(metrics_enabled);
|
||||
file.seek(SeekFrom::Start(offset_u64)).map_err(DiskError::from)?;
|
||||
let mut buffer = vec![0; length];
|
||||
// Positioned read: a cache hit reads through a `dup`'d handle
|
||||
// that shares the cached descriptor's offset, so this must not
|
||||
// touch the descriptor offset (rustfs/backlog#1801).
|
||||
file.read_exact_at(&mut buffer, offset_u64).map_err(DiskError::from)?;
|
||||
file.read_exact(&mut buffer).map_err(DiskError::from)?;
|
||||
let direct_read_copy_faults_after = read_mmap_page_fault_counts(metrics_enabled);
|
||||
direct_read_copy_duration =
|
||||
direct_read_copy_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
@@ -3323,16 +3193,6 @@ impl LocalIoBackend for StdBackend {
|
||||
|
||||
let blocking_task_duration = blocking_task_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
|
||||
// Hand the freshly opened descriptor back so the async caller can index
|
||||
// the cache — None on a hit (the cache already holds it). mmap/reclaim
|
||||
// above only borrowed `file`, so it is still owned here and moves into the
|
||||
// Arc; `cached_fd.is_none()` is true exactly when this call did the open.
|
||||
// Non-Linux has no fd cache, so skip the Arc allocation there.
|
||||
#[cfg(target_os = "linux")]
|
||||
let opened_fd: Option<Arc<std::fs::File>> = cached_fd.is_none().then(|| Arc::new(file));
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let opened_fd: Option<Arc<std::fs::File>> = None;
|
||||
|
||||
Ok::<MmapCopyReadResult, MmapCopyReadError>(MmapCopyReadResult {
|
||||
bytes,
|
||||
access_check_duration,
|
||||
@@ -3348,7 +3208,6 @@ impl LocalIoBackend for StdBackend {
|
||||
direct_read_copy_fault_delta,
|
||||
blocking_task_duration,
|
||||
used_direct_io,
|
||||
opened_fd,
|
||||
})
|
||||
})
|
||||
.await
|
||||
@@ -3454,16 +3313,6 @@ impl LocalIoBackend for StdBackend {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Index the freshly opened descriptor for future cache hits
|
||||
// (rustfs/backlog#1801). `insert_if_fresh` refuses to cache if an
|
||||
// invalidation (heal/delete/rename) bumped the generation between the
|
||||
// open snapshot and now, so a stale pre-mutation inode is never served
|
||||
// (rustfs/backlog#1176). On a cache hit `opened_fd` is None; on non-Linux
|
||||
// there is no fd cache, so this is gated out entirely.
|
||||
#[cfg(target_os = "linux")]
|
||||
if let (Some((cache, key, gen_at_open)), Some(opened)) = (fd_lookup, read_result.opened_fd) {
|
||||
cache.insert_if_fresh(key, opened, gen_at_open).await;
|
||||
}
|
||||
let bytes = read_result.bytes;
|
||||
|
||||
// Log successful mmap read metrics
|
||||
@@ -3667,39 +3516,6 @@ impl LocalIoBackend for StdBackend {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Descriptor-cache invalidation for StdBackend (rustfs/backlog#1801). On
|
||||
// non-Linux `fd_cache` does not exist, so these overrides are absent and the
|
||||
// trait's default no-op impls apply. On Linux they mirror UringBackend so
|
||||
// the existing LocalDisk mutation hooks (rename_data/rename_file/delete/
|
||||
// delete_volume/close) drop stale descriptors on every inode swap.
|
||||
#[cfg(target_os = "linux")]
|
||||
async fn invalidate_cached_fd(&self, volume: &str, path: &str) {
|
||||
if let Some(cache) = self.fd_cache.as_ref() {
|
||||
cache.invalidate_exact(volume, path).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn invalidate_cached_fds_under(&self, volume: &str, path: &str) {
|
||||
if let Some(cache) = self.fd_cache.as_ref() {
|
||||
cache.invalidate_under(volume, path);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn invalidate_cached_fds_for_volume(&self, volume: &str) {
|
||||
if let Some(cache) = self.fd_cache.as_ref() {
|
||||
cache.invalidate_volume(volume);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
async fn clear_cached_fds(&self) {
|
||||
if let Some(cache) = self.fd_cache.as_ref() {
|
||||
cache.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable the per-disk descriptor cache for io_uring reads (backlog#1145).
|
||||
@@ -3725,20 +3541,6 @@ fn is_io_uring_fd_cache_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_IO_URING_FD_CACHE, DEFAULT_RUSTFS_IO_URING_FD_CACHE)
|
||||
}
|
||||
|
||||
/// Enable the per-disk descriptor cache for the default `StdBackend` reads
|
||||
/// (rustfs/backlog#1801). Independent of the io_uring switch so each backend is
|
||||
/// separately controllable; both share the same `rlimit_allows_fd_cache` guard
|
||||
/// because each may hold up to `FD_CACHE_CAPACITY` (512) descriptors per disk.
|
||||
#[cfg(target_os = "linux")]
|
||||
const ENV_RUSTFS_LOCAL_FD_CACHE: &str = "RUSTFS_LOCAL_FD_CACHE";
|
||||
#[cfg(target_os = "linux")]
|
||||
const DEFAULT_RUSTFS_LOCAL_FD_CACHE: bool = true;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn is_local_fd_cache_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_LOCAL_FD_CACHE, DEFAULT_RUSTFS_LOCAL_FD_CACHE)
|
||||
}
|
||||
|
||||
/// Whether the soft `RLIMIT_NOFILE` has enough headroom to run the fd cache
|
||||
/// safely (rustfs/backlog#1178). The cache holds up to `FD_CACHE_CAPACITY` (512)
|
||||
/// descriptors PER DISK and `try_new` cannot know the disk count, so a low limit
|
||||
@@ -4144,7 +3946,7 @@ impl UringBackend {
|
||||
// struct (rustfs/backlog#1185).
|
||||
let root_label = root.display().to_string();
|
||||
Some(Self {
|
||||
inner: StdBackend::new_without_fd_cache(root.clone()),
|
||||
inner: StdBackend::new(root.clone()),
|
||||
root,
|
||||
root_label,
|
||||
driver: std::mem::ManuallyDrop::new(driver),
|
||||
@@ -9219,20 +9021,7 @@ impl DiskAPI for LocalDisk {
|
||||
#[cfg(windows)]
|
||||
let source_parent = src_file_parent.to_path_buf();
|
||||
let rename_commit_guard_for_preparation = rename_commit_guard.clone();
|
||||
let sync = durability.syncs_commit_metadata();
|
||||
#[cfg(test)]
|
||||
run_inline_before_file_sync_admission(dst_path);
|
||||
let mut file_sync_admission = if sync {
|
||||
Some(
|
||||
os::acquire_file_sync_admission(self.file_sync_permits.clone())
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let prepare_inline_metadata = move || {
|
||||
let inline_preparation = os::run_blocking_namespace_operation(mutation_lease.clone(), move || {
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&src, &dst, &rename_commit_guard_for_preparation)?;
|
||||
#[cfg(windows)]
|
||||
@@ -9269,6 +9058,7 @@ impl DiskAPI for LocalDisk {
|
||||
None
|
||||
}
|
||||
});
|
||||
let sync = durability.syncs_commit_metadata();
|
||||
let mut staged_rollback_path = None;
|
||||
if let Some(d) = old_data_dir.as_ref() {
|
||||
let _ = xlmeta.data.remove_two(version_id, *d);
|
||||
@@ -9313,12 +9103,8 @@ impl DiskAPI for LocalDisk {
|
||||
has_dst_buf.is_none(),
|
||||
prepared_metadata_source,
|
||||
))
|
||||
};
|
||||
let inline_preparation = if let Some(admission) = file_sync_admission.as_ref() {
|
||||
os::run_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, prepare_inline_metadata).await
|
||||
} else {
|
||||
os::run_blocking_namespace_operation(mutation_lease.clone(), prepare_inline_metadata).await
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from);
|
||||
|
||||
@@ -9368,26 +9154,14 @@ impl DiskAPI for LocalDisk {
|
||||
let backup_path = dst_parent
|
||||
.join(rollback_data_dir.to_string())
|
||||
.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
// rename_all acquires the backup path's namespace lease. Do not
|
||||
// hold a disk admission while acquiring another namespace lock.
|
||||
drop(file_sync_admission.take());
|
||||
if let Err(err) = rename_all(staged_backup, &backup_path, &dst_volume_dir, &self.publication_root).await {
|
||||
let _ = remove_file_if_exists(staged_backup);
|
||||
return Err(err);
|
||||
}
|
||||
run_rename_data_after_first_publication(dst_path);
|
||||
if sync {
|
||||
file_sync_admission = Some(
|
||||
os::acquire_file_sync_admission(self.file_sync_permits.clone())
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)?,
|
||||
);
|
||||
}
|
||||
if let Some(admission) = file_sync_admission.as_ref()
|
||||
if durability.syncs_commit_metadata()
|
||||
&& let Some(backup_parent) = backup_path.parent()
|
||||
&& let Err(err) =
|
||||
os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await
|
||||
&& let Err(err) = os::fsync_dir(backup_parent).await
|
||||
{
|
||||
return Err(DiskError::from(to_file_error(err)));
|
||||
}
|
||||
@@ -9425,10 +9199,9 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
// Persist the commit rename's directory entry across power loss.
|
||||
if let Some(admission) = file_sync_admission.as_ref()
|
||||
if durability.syncs_commit_metadata()
|
||||
&& let Some(dst_parent) = dst_file_path.parent()
|
||||
&& let Err(err) =
|
||||
os::fsync_dir_with_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission).await
|
||||
&& let Err(err) = os::fsync_dir(dst_parent).await
|
||||
{
|
||||
rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?;
|
||||
return Err(err);
|
||||
@@ -9441,17 +9214,13 @@ impl DiskAPI for LocalDisk {
|
||||
// not its own entry, so for a new inline object fsync the ancestor
|
||||
// chain up to and including the bucket. Overwrites already have a
|
||||
// durable object dir; the starts_with guard bounds the walk.
|
||||
if let Some(admission) = file_sync_admission.as_ref()
|
||||
&& destination_was_absent
|
||||
{
|
||||
if durability.syncs_commit_metadata() && destination_was_absent {
|
||||
let mut ancestor = dst_file_path.parent().and_then(|object_dir| object_dir.parent());
|
||||
while let Some(ancestor_dir) = ancestor {
|
||||
if !ancestor_dir.starts_with(&dst_volume_dir) {
|
||||
break;
|
||||
}
|
||||
if let Err(err) =
|
||||
os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await
|
||||
{
|
||||
if let Err(err) = os::fsync_dir(ancestor_dir).await {
|
||||
rollback_inline_metadata_commit_std(
|
||||
&dst_file_path,
|
||||
rollback_data_dir,
|
||||
@@ -9470,10 +9239,6 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
.await;
|
||||
|
||||
// The disk admission protects the durability chain, not staging
|
||||
// cleanup or cache invalidation after that chain has completed.
|
||||
drop(file_sync_admission.take());
|
||||
|
||||
// A post-commit rollback (for example, a commit-metadata fsync
|
||||
// failure under strict durability) restores the old metadata; drop any
|
||||
// descriptors cached during the committed window before propagating the
|
||||
@@ -12640,7 +12405,7 @@ mod test {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn windows_and_unix_rename_data_new_inline_object_fsyncs_new_ancestor_dirs() {
|
||||
async fn test_rename_data_new_inline_object_fsyncs_new_ancestor_dirs() {
|
||||
// The inline commit path (fi.data present) has the same mkdir gap as the
|
||||
// non-inline path: a first PUT under a new prefix must fsync the newly
|
||||
// created prefix and bucket dirs.
|
||||
@@ -12669,122 +12434,10 @@ mod test {
|
||||
os::fsync_dir_recorder::was_fsynced(&prefix_dir),
|
||||
"the newly created prefix dir must be fsynced on an inline first PUT"
|
||||
);
|
||||
assert_eq!(
|
||||
os::fsync_dir_recorder::was_limited(&prefix_dir),
|
||||
cfg!(unix),
|
||||
"only Unix inline prefix fsyncs should use the disk file-sync limit"
|
||||
);
|
||||
assert!(
|
||||
os::fsync_dir_recorder::was_fsynced(&bucket_dir),
|
||||
"the bucket dir must be fsynced on an inline first PUT"
|
||||
);
|
||||
assert_eq!(
|
||||
os::fsync_dir_recorder::was_limited(&bucket_dir),
|
||||
cfg!(unix),
|
||||
"only Unix inline bucket fsyncs should use the disk file-sync limit"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn strict_inline_rename_retains_admission_until_commit_fsync() {
|
||||
use std::sync::atomic::{AtomicU8, Ordering};
|
||||
use std::sync::mpsc;
|
||||
use tempfile::tempdir;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
const FIRST_BARRIER: u8 = 1;
|
||||
const SECOND_PREPARATION: u8 = 2;
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let mut disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
disk.file_sync_permits = Arc::new(Semaphore::new(1));
|
||||
let disk = Arc::new(disk);
|
||||
let bucket = "inline-admission-order";
|
||||
let first_object = "first-object";
|
||||
let second_object = "second-object";
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
|
||||
let (first_prepared_tx, first_prepared_rx) = mpsc::channel();
|
||||
let (release_first_tx, release_first_rx) = mpsc::channel();
|
||||
set_inline_preparation_before_backup(first_object, move || {
|
||||
first_prepared_tx.send(()).expect("signal first preparation");
|
||||
release_first_rx.recv().expect("wait for queued rename");
|
||||
});
|
||||
let first_disk = disk.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
first_disk
|
||||
.rename_data(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"first-stage",
|
||||
test_file_info(first_object, Uuid::new_v4(), None, Some(Bytes::from_static(b"first"))),
|
||||
bucket,
|
||||
first_object,
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::task::spawn_blocking(move || first_prepared_rx.recv_timeout(Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("first preparation waiter should run")
|
||||
.expect("first rename should hold the only disk admission");
|
||||
|
||||
let first_event = Arc::new(AtomicU8::new(0));
|
||||
let first_barrier_event = first_event.clone();
|
||||
let first_object_dir = disk
|
||||
.get_object_path_for_io(bucket, first_object)
|
||||
.expect("first object path should resolve");
|
||||
os::fsync_dir_recorder::set_before_limited(&first_object_dir, move || {
|
||||
let _ = first_barrier_event.compare_exchange(0, FIRST_BARRIER, Ordering::SeqCst, Ordering::SeqCst);
|
||||
});
|
||||
|
||||
let second_preparation_event = first_event.clone();
|
||||
set_inline_preparation_before_backup(second_object, move || {
|
||||
second_preparation_event.fetch_or(SECOND_PREPARATION, Ordering::SeqCst);
|
||||
});
|
||||
let (second_admission_tx, second_admission_rx) = oneshot::channel();
|
||||
set_inline_before_file_sync_admission(second_object, move || {
|
||||
second_admission_tx.send(()).expect("signal second admission attempt");
|
||||
});
|
||||
let second_disk = disk.clone();
|
||||
let mut second = Box::pin(async move {
|
||||
second_disk
|
||||
.rename_data(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"second-stage",
|
||||
test_file_info(second_object, Uuid::new_v4(), None, Some(Bytes::from_static(b"second"))),
|
||||
bucket,
|
||||
second_object,
|
||||
)
|
||||
.await
|
||||
});
|
||||
let mut second_admission_rx = Box::pin(second_admission_rx);
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
tokio::select! {
|
||||
_ = &mut second => panic!("second rename must wait for disk admission"),
|
||||
signal = &mut second_admission_rx => signal.expect("second admission hook should run"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second rename should reach the admission queue");
|
||||
|
||||
release_first_tx.send(()).expect("release first preparation");
|
||||
let (first_result, second_result) = tokio::time::timeout(Duration::from_secs(30), async { tokio::join!(first, second) })
|
||||
.await
|
||||
.expect("both inline renames should complete");
|
||||
first_result
|
||||
.expect("first rename task should join")
|
||||
.expect("first inline rename should commit");
|
||||
second_result.expect("second inline rename should commit");
|
||||
|
||||
assert_eq!(
|
||||
first_event.load(Ordering::SeqCst),
|
||||
FIRST_BARRIER | SECOND_PREPARATION,
|
||||
"the queued rename must not overtake the admitted rename before its commit fsync"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
@@ -14031,16 +13684,14 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
#[tokio::test]
|
||||
async fn test_rename_data_writes_old_metadata_backup_for_inline_overwrite() {
|
||||
use std::sync::mpsc;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk should be created");
|
||||
|
||||
let bucket = "bucket";
|
||||
let object = "inline-object";
|
||||
@@ -14066,32 +13717,10 @@ mod test {
|
||||
.await
|
||||
.expect("tmp object dir should be created");
|
||||
|
||||
let (published_tx, published_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
set_rename_data_after_first_publication(object, move || {
|
||||
published_tx.send(()).expect("signal backup publication");
|
||||
release_rx.recv().expect("wait for lock-order assertion");
|
||||
});
|
||||
let new_fi = test_file_info(object, version_id, None, Some(Bytes::from_static(b"inline-new")));
|
||||
let rename_disk = disk.clone();
|
||||
let rename = tokio::spawn(async move {
|
||||
rename_disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, new_fi, bucket, object)
|
||||
.await
|
||||
});
|
||||
tokio::task::spawn_blocking(move || published_rx.recv_timeout(Duration::from_secs(10)))
|
||||
let resp = disk
|
||||
.rename_data(RUSTFS_META_TMP_BUCKET, tmp_object, new_fi, bucket, object)
|
||||
.await
|
||||
.expect("publication waiter should run")
|
||||
.expect("rollback backup must be published");
|
||||
assert_eq!(
|
||||
disk.file_sync_permits.available_permits(),
|
||||
os::MAX_PARALLEL_FILE_SYNCS,
|
||||
"backup publication must not acquire namespace while holding disk admission"
|
||||
);
|
||||
release_tx.send(()).expect("release backup publication");
|
||||
let resp = rename
|
||||
.await
|
||||
.expect("inline rename task should join")
|
||||
.expect("inline rename_data should commit");
|
||||
|
||||
assert_eq!(resp.old_data_dir, Some(old_data_dir));
|
||||
@@ -14102,20 +13731,10 @@ mod test {
|
||||
os::fsync_dir_recorder::was_fsynced(backup_path.parent().expect("backup must have a parent")),
|
||||
"strict inline overwrite must persist the rollback backup directory entry"
|
||||
);
|
||||
assert_eq!(
|
||||
os::fsync_dir_recorder::was_limited(backup_path.parent().expect("backup must have a parent")),
|
||||
cfg!(unix),
|
||||
"only Unix rollback backup fsyncs should use the disk file-sync limit"
|
||||
);
|
||||
assert!(
|
||||
os::fsync_dir_recorder::was_fsynced(&dst_object_dir),
|
||||
"strict inline overwrite must persist the committed xl.meta directory entry"
|
||||
);
|
||||
assert_eq!(
|
||||
os::fsync_dir_recorder::was_limited(&dst_object_dir),
|
||||
cfg!(unix),
|
||||
"only Unix inline commit fsyncs should use the disk file-sync limit"
|
||||
);
|
||||
// The rollback backup must contain the previous metadata bytes verbatim so
|
||||
// that undo_write can restore the prior committed object; guards the inline
|
||||
// backup write against truncation/corruption regressions.
|
||||
@@ -14712,12 +14331,10 @@ mod test {
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn windows_and_unix_cancelled_inline_preparation_serializes_newer_commit() {
|
||||
use std::sync::mpsc;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Strict);
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
@@ -14756,11 +14373,6 @@ mod test {
|
||||
.await
|
||||
.expect("preparation waiter should run")
|
||||
.expect("preparation must reach the backup hook");
|
||||
assert_eq!(
|
||||
disk.file_sync_permits.available_permits(),
|
||||
os::MAX_PARALLEL_FILE_SYNCS - 1,
|
||||
"strict inline preparation must hold one disk file-sync permit"
|
||||
);
|
||||
cancelled.abort();
|
||||
assert!(cancelled.await.expect_err("operation should be cancelled").is_cancelled());
|
||||
|
||||
@@ -14834,56 +14446,6 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn relaxed_inline_preparation_does_not_use_file_sync_limit() {
|
||||
use std::sync::mpsc;
|
||||
use tempfile::tempdir;
|
||||
|
||||
let _mode = durability_mode_override::set(DurabilityMode::Relaxed);
|
||||
let dir = tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be utf8")).expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
let bucket = "relaxed-inline-preparation";
|
||||
let object = "inline-object";
|
||||
ensure_test_volume(&disk, bucket).await;
|
||||
ensure_test_volume(&disk, RUSTFS_META_TMP_BUCKET).await;
|
||||
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
set_inline_preparation_before_backup(object, move || {
|
||||
entered_tx.send(()).expect("signal blocked preparation");
|
||||
release_rx.recv().expect("wait for permit assertion");
|
||||
});
|
||||
let rename_disk = Arc::clone(&disk);
|
||||
let rename = tokio::spawn(async move {
|
||||
rename_disk
|
||||
.rename_data(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"relaxed-inline-stage",
|
||||
test_file_info(object, Uuid::new_v4(), None, Some(Bytes::from_static(b"payload"))),
|
||||
bucket,
|
||||
object,
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(10)))
|
||||
.await
|
||||
.expect("preparation waiter should run")
|
||||
.expect("preparation must reach the hook");
|
||||
assert_eq!(
|
||||
disk.file_sync_permits.available_permits(),
|
||||
os::MAX_PARALLEL_FILE_SYNCS,
|
||||
"relaxed inline preparation must not consume strict sync capacity"
|
||||
);
|
||||
|
||||
release_tx.send(()).expect("release inline preparation");
|
||||
rename
|
||||
.await
|
||||
.expect("rename task should join")
|
||||
.expect("relaxed inline rename should commit");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_purge_pending_payload_stays_object_and_cleans_local_backup() {
|
||||
use tempfile::tempdir;
|
||||
@@ -19832,133 +19394,6 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
/// Same heal hazard as the io_uring test, but exercised through the default
|
||||
/// `StdBackend` read path (rustfs/backlog#1801): a cached descriptor keeps
|
||||
/// serving the pre-heal inode until `invalidate_cached_fds_under` drops it.
|
||||
/// `StdBackend` reads via mmap/`try_clone`, so this proves the dup-based hit
|
||||
/// path also defers to invalidation rather than masking a healed shard.
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn std_fd_cache_hides_a_healed_shard_until_invalidated() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
let root_dir = tempdir().expect("operation should succeed");
|
||||
let root = root_dir.path().to_path_buf();
|
||||
let backend = temp_env::with_vars([(ENV_RUSTFS_LOCAL_FD_CACHE, Some("true"))], || StdBackend::new(root.clone()));
|
||||
if backend.fd_cache.is_none() {
|
||||
// RLIMIT_NOFILE too low for 512 fds/disk (rustfs/backlog#1178): the
|
||||
// cache is off, so there is nothing to exercise. Do not vacuously pass.
|
||||
eprintln!(
|
||||
"std_fd_cache_hides_a_healed_shard_until_invalidated: skipped \
|
||||
(RLIMIT_NOFILE too low for the std fd cache)"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let volume = "bucket";
|
||||
let object = "obj/0d1e2f/part.1";
|
||||
let dir = root.join(volume).join("obj/0d1e2f");
|
||||
std::fs::create_dir_all(&dir).expect("operation should succeed");
|
||||
let part = root.join(volume).join(object);
|
||||
std::fs::write(&part, b"corrupt-shard").expect("operation should succeed");
|
||||
|
||||
let before = backend
|
||||
.pread_bytes(volume, object, 0, b"corrupt-shard".len(), None)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
assert_eq!(before, Bytes::from_static(b"corrupt-shard"));
|
||||
|
||||
// Heal: rename rebuilt content onto the same part path — inode swap, path
|
||||
// unchanged. A cached descriptor would keep reading the old inode.
|
||||
let rebuilt = dir.join("part.1.rebuilt");
|
||||
std::fs::write(&rebuilt, b"healed--shard").expect("operation should succeed");
|
||||
std::fs::rename(&rebuilt, &part).expect("operation should succeed");
|
||||
|
||||
let stale = backend
|
||||
.pread_bytes(volume, object, 0, b"healed--shard".len(), None)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
assert_eq!(
|
||||
stale,
|
||||
Bytes::from_static(b"corrupt-shard"),
|
||||
"a cached descriptor is expected to still see the pre-heal inode — this is the \
|
||||
hazard invalidate_cached_fds exists to close, and the assertion proves the cache is live"
|
||||
);
|
||||
|
||||
backend.invalidate_cached_fds_under(volume, "obj/0d1e2f");
|
||||
let healed = backend
|
||||
.pread_bytes(volume, object, 0, b"healed--shard".len(), None)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
assert_eq!(healed, Bytes::from_static(b"healed--shard"));
|
||||
}
|
||||
|
||||
/// A repeated read of the same shard must (a) return correct bytes both times
|
||||
/// and (b) actually populate the descriptor cache, so the second read can skip
|
||||
/// `File::open` (rustfs/backlog#1801).
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn std_fd_cache_serves_repeated_reads_and_caches_descriptor() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
let root_dir = tempdir().expect("operation should succeed");
|
||||
let root = root_dir.path().to_path_buf();
|
||||
let backend = temp_env::with_vars([(ENV_RUSTFS_LOCAL_FD_CACHE, Some("true"))], || StdBackend::new(root.clone()));
|
||||
let cache = match backend.fd_cache.as_ref() {
|
||||
Some(c) => c,
|
||||
None => {
|
||||
eprintln!(
|
||||
"std_fd_cache_serves_repeated_reads_and_caches_descriptor: skipped \
|
||||
(RLIMIT_NOFILE too low for the std fd cache)"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let volume = "bucket";
|
||||
let object = "obj/abc/part.1";
|
||||
std::fs::create_dir_all(root.join(volume).join("obj/abc")).expect("operation should succeed");
|
||||
let payload = b"hello-small-shard-payload";
|
||||
std::fs::write(root.join(volume).join(object), payload).expect("operation should succeed");
|
||||
|
||||
let first = backend
|
||||
.pread_bytes(volume, object, 0, payload.len(), None)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
assert_eq!(first, Bytes::from_static(payload));
|
||||
|
||||
// After the first miss the freshly opened descriptor is indexed; a second
|
||||
// read of the same path is a cache hit.
|
||||
assert_eq!(cache.entry_count().await, 1, "the first read should have cached exactly one descriptor");
|
||||
|
||||
let second = backend
|
||||
.pread_bytes(volume, object, 0, payload.len(), None)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
assert_eq!(second, Bytes::from_static(payload));
|
||||
|
||||
// Invalidating by the object prefix drops the cached descriptor.
|
||||
backend.invalidate_cached_fds_under(volume, "obj/abc");
|
||||
assert_eq!(cache.entry_count().await, 0, "prefix invalidation must drop the cached descriptor");
|
||||
}
|
||||
|
||||
/// `StdBackend::new_without_fd_cache` must not build a descriptor cache.
|
||||
/// `UringBackend` wraps a `StdBackend` and owns the only cache for the disk,
|
||||
/// so an inner cache would be populated by fallback reads
|
||||
/// (`UringBackend::pread_bytes` delegates inward) yet never invalidated —
|
||||
/// the stale-inode hazard `FdCache` exists to close (backlog#1176/#1801).
|
||||
/// This pins the contract so a future constructor change cannot regress it.
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn new_without_fd_cache_builds_no_descriptor_cache() {
|
||||
let root_dir = tempfile::tempdir().expect("operation should succeed");
|
||||
let backend = StdBackend::new_without_fd_cache(root_dir.path().to_path_buf());
|
||||
assert!(
|
||||
backend.fd_cache.is_none(),
|
||||
"new_without_fd_cache must not build a descriptor cache — UringBackend owns the only cache for the disk"
|
||||
);
|
||||
}
|
||||
|
||||
/// The mutation paths on `LocalDisk` must actually call
|
||||
/// `invalidate_cached_fds`, not merely have it available (backlog#1145).
|
||||
/// `rename_file` replaces the inode at a path a reader has already cached;
|
||||
|
||||
@@ -1251,7 +1251,7 @@ pub struct VolumeInfo {
|
||||
pub created: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Debug, Default, Clone, Copy)]
|
||||
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
|
||||
pub struct ReadOptions {
|
||||
pub incl_free_versions: bool,
|
||||
pub read_data: bool,
|
||||
|
||||
+12
-190
@@ -78,59 +78,28 @@ pub fn check_path_length(path_name: &str) -> Result<()> {
|
||||
/// their own unique tempdir to stay robust against parallel test execution.
|
||||
#[cfg(test)]
|
||||
pub(crate) mod fsync_dir_recorder {
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
type Hook = Box<dyn FnOnce() + Send>;
|
||||
|
||||
static RECORDED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
static LIMITED: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
static BEFORE_LIMITED: std::sync::LazyLock<Mutex<HashMap<PathBuf, Hook>>> =
|
||||
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
fn record_path(paths: &Mutex<Vec<PathBuf>>, path: &Path, description: &str) {
|
||||
let mut paths = paths.lock().expect(description);
|
||||
paths.push(path.to_path_buf());
|
||||
if let Ok(canonical) = path.canonicalize()
|
||||
&& canonical != path
|
||||
{
|
||||
paths.push(canonical);
|
||||
}
|
||||
}
|
||||
|
||||
fn contains_path(paths: &[PathBuf], path: &Path) -> bool {
|
||||
let canonical = path.canonicalize().ok();
|
||||
paths
|
||||
.iter()
|
||||
.any(|recorded| recorded == path || canonical.as_ref().is_some_and(|canonical| recorded == canonical))
|
||||
}
|
||||
|
||||
pub(crate) fn record(dir: &Path) {
|
||||
record_path(&RECORDED, dir, "fsync dir recorder");
|
||||
let mut recorded = RECORDED.lock().expect("fsync dir recorder poisoned");
|
||||
recorded.push(dir.to_path_buf());
|
||||
if let Ok(canonical) = dir.canonicalize()
|
||||
&& canonical != dir
|
||||
{
|
||||
recorded.push(canonical);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn was_fsynced(dir: &Path) -> bool {
|
||||
contains_path(&RECORDED.lock().expect("fsync dir recorder poisoned"), dir)
|
||||
}
|
||||
|
||||
pub(crate) fn record_limited(dir: &Path) {
|
||||
record_path(&LIMITED, dir, "limited fsync dir recorder");
|
||||
let hook = BEFORE_LIMITED.lock().expect("limited fsync hook poisoned").remove(dir);
|
||||
if let Some(hook) = hook {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn was_limited(dir: &Path) -> bool {
|
||||
contains_path(&LIMITED.lock().expect("limited fsync dir recorder poisoned"), dir)
|
||||
}
|
||||
|
||||
pub(crate) fn set_before_limited(dir: &Path, hook: impl FnOnce() + Send + 'static) {
|
||||
BEFORE_LIMITED
|
||||
let canonical = dir.canonicalize().ok();
|
||||
RECORDED
|
||||
.lock()
|
||||
.expect("limited fsync hook poisoned")
|
||||
.insert(dir.to_path_buf(), Box::new(hook));
|
||||
.expect("fsync dir recorder poisoned")
|
||||
.iter()
|
||||
.any(|p| p == dir || canonical.as_ref().is_some_and(|canonical| p == canonical))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1151,79 +1120,6 @@ pub(crate) async fn run_blocking_namespace_operation<T: Send + 'static>(
|
||||
.map_err(|err| io::Error::other(format!("blocking namespace operation failed: {err}")))?
|
||||
}
|
||||
|
||||
/// Admit one strict inline commit under the disk sync limit. The caller already
|
||||
/// owns the namespace lease, establishing namespace -> disk ordering. Holding
|
||||
/// admission across adjacent durability barriers prevents one transaction from
|
||||
/// repeatedly joining the disk semaphore tail.
|
||||
pub(crate) struct FileSyncAdmission {
|
||||
disk_permit: Arc<OwnedSemaphorePermit>,
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_file_sync_admission(disk_permits: Arc<Semaphore>) -> io::Result<FileSyncAdmission> {
|
||||
let disk_permit = disk_permits
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|_| io::Error::other("disk file sync concurrency limiter closed"))?;
|
||||
Ok(FileSyncAdmission {
|
||||
disk_permit: Arc::new(disk_permit),
|
||||
})
|
||||
}
|
||||
|
||||
/// Keep the disk admission and namespace lease with the blocking syscall if
|
||||
/// the async waiter is cancelled. The process-wide admission remains with the
|
||||
/// waiter so cancellation cannot starve healthy disks.
|
||||
pub(crate) async fn run_blocking_namespace_file_sync_operation<T: Send + 'static>(
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
admission: &FileSyncAdmission,
|
||||
operation: impl FnOnce() -> io::Result<T> + Send + 'static,
|
||||
) -> io::Result<T> {
|
||||
run_blocking_namespace_file_sync_operation_with_global(lease, admission, &FILE_SYNC_PERMITS, operation).await
|
||||
}
|
||||
|
||||
async fn run_blocking_namespace_file_sync_operation_with_global<T: Send + 'static>(
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
admission: &FileSyncAdmission,
|
||||
global_permits: &Semaphore,
|
||||
operation: impl FnOnce() -> io::Result<T> + Send + 'static,
|
||||
) -> io::Result<T> {
|
||||
let global_permit = global_permits
|
||||
.acquire()
|
||||
.await
|
||||
.map_err(|_| io::Error::other("global file sync concurrency limiter closed"))?;
|
||||
let disk_permit = admission.disk_permit.clone();
|
||||
let result = tokio::task::spawn_blocking(move || {
|
||||
let _lease = lease;
|
||||
let _disk_permit = disk_permit;
|
||||
operation()
|
||||
})
|
||||
.await;
|
||||
drop(global_permit);
|
||||
result.map_err(|err| io::Error::other(format!("blocking namespace file sync operation failed: {err}")))?
|
||||
}
|
||||
|
||||
pub(crate) async fn fsync_dir_with_namespace_file_sync_limit(
|
||||
dir: impl AsRef<Path>,
|
||||
lease: Arc<NamespaceMutationLease>,
|
||||
admission: &FileSyncAdmission,
|
||||
) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let dir = dir.as_ref().to_path_buf();
|
||||
run_blocking_namespace_file_sync_operation(lease, admission, move || {
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::record_limited(&dir);
|
||||
fsync_dir_std(dir)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = (lease, admission);
|
||||
fsync_dir_std(dir)
|
||||
}
|
||||
}
|
||||
|
||||
struct RenamePreparation {
|
||||
parent_guard: Option<ExistingBaseDirectoryGuard>,
|
||||
#[cfg(windows)]
|
||||
@@ -2888,7 +2784,6 @@ pub fn is_dir_not_empty_error(err: &io::Error) -> bool {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
use tempfile::tempdir;
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
@@ -4658,79 +4553,6 @@ mod tests {
|
||||
fsync_dir(temp_dir.path()).await.expect("fsync dir must succeed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn file_sync_admission_is_reused_across_commit_barriers() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let limiter = Arc::new(Semaphore::new(1));
|
||||
let lease = acquire_namespace_mutation_lease(temp_dir.path()).await;
|
||||
let admission = acquire_file_sync_admission(limiter.clone())
|
||||
.await
|
||||
.expect("first commit should acquire admission");
|
||||
|
||||
run_blocking_namespace_file_sync_operation(lease.clone(), &admission, || Ok(()))
|
||||
.await
|
||||
.expect("first barrier should complete under the admission");
|
||||
let mut waiting = Box::pin(acquire_file_sync_admission(limiter));
|
||||
assert!(
|
||||
futures::poll!(&mut waiting).is_pending(),
|
||||
"another commit must remain queued between durability barriers"
|
||||
);
|
||||
run_blocking_namespace_file_sync_operation(lease, &admission, || Ok(()))
|
||||
.await
|
||||
.expect("later barrier should reuse admission without requeuing");
|
||||
|
||||
drop(admission);
|
||||
tokio::time::timeout(Duration::from_secs(30), waiting)
|
||||
.await
|
||||
.expect("queued commit should acquire admission after release")
|
||||
.expect("queued commit should acquire admission");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn cancelled_file_sync_waiter_keeps_disk_admission_until_blocking_work_finishes() {
|
||||
use std::sync::mpsc;
|
||||
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let limiter = Arc::new(Semaphore::new(1));
|
||||
let global_permits = Arc::new(Semaphore::new(1));
|
||||
let lease = acquire_namespace_mutation_lease(temp_dir.path()).await;
|
||||
let admission = acquire_file_sync_admission(limiter.clone())
|
||||
.await
|
||||
.expect("file sync admission should be acquired");
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_tx, release_rx) = mpsc::channel();
|
||||
let waiter_global_permits = global_permits.clone();
|
||||
let waiter = tokio::spawn(async move {
|
||||
run_blocking_namespace_file_sync_operation_with_global(lease, &admission, waiter_global_permits.as_ref(), move || {
|
||||
entered_tx.send(()).expect("signal blocking work");
|
||||
release_rx.recv().expect("wait for blocking work release");
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("blocking work waiter should run")
|
||||
.expect("blocking work should start");
|
||||
waiter.abort();
|
||||
assert!(waiter.await.expect_err("waiter should be cancelled").is_cancelled());
|
||||
let returned_global_permit = global_permits
|
||||
.try_acquire()
|
||||
.expect("cancelled waiter must return global capacity for healthy disks");
|
||||
assert!(
|
||||
limiter.clone().try_acquire_owned().is_err(),
|
||||
"cancelled waiter must not return disk capacity while blocking work is active"
|
||||
);
|
||||
|
||||
release_tx.send(()).expect("release blocking work");
|
||||
let _returned_permit = tokio::time::timeout(Duration::from_secs(30), limiter.acquire_owned())
|
||||
.await
|
||||
.expect("disk capacity should return after blocking work finishes")
|
||||
.expect("disk limiter should remain open");
|
||||
drop(returned_global_permit);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
async fn sync_dir_files_syncs_regular_files_and_dir() {
|
||||
|
||||
@@ -18,11 +18,7 @@ use std::io::IoSlice;
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
||||
use tracing::error;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_ERASURE: &str = "erasure";
|
||||
const EVENT_BITROT_SHORT_SHARD_READ: &str = "bitrot_short_shard_read";
|
||||
const EVENT_BITROT_HASH_MISMATCH: &str = "bitrot_hash_mismatch";
|
||||
use uuid::Uuid;
|
||||
|
||||
/// A shard source that may already hold its bytes in memory.
|
||||
///
|
||||
@@ -77,6 +73,7 @@ pin_project! {
|
||||
buf: Vec<u8>,
|
||||
skip_verify: bool,
|
||||
last_verify_duration: Duration,
|
||||
id: Uuid,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,6 +90,7 @@ where
|
||||
buf: Vec::new(),
|
||||
skip_verify,
|
||||
last_verify_duration: Duration::ZERO,
|
||||
id: Uuid::new_v4(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -120,7 +118,7 @@ where
|
||||
|
||||
let need = self.hash_algo.size() + want;
|
||||
self.read_scratch_block(need, want).await?;
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need])?;
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need], &self.id)?;
|
||||
out.copy_from_slice(data);
|
||||
self.last_verify_duration = verify;
|
||||
Ok(want)
|
||||
@@ -159,7 +157,7 @@ where
|
||||
}
|
||||
let filled = fill(&mut self.inner, &mut self.buf[..need]).await?;
|
||||
if filled < need {
|
||||
return Err(short_shard_read(filled.saturating_sub(self.hash_algo.size()), want));
|
||||
return Err(short_shard_read(&self.id, filled.saturating_sub(self.hash_algo.size()), want));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -168,23 +166,15 @@ where
|
||||
/// buffer returns its length, a short read is UnexpectedEof (backlog#799 B2).
|
||||
fn finish_len(&self, data_len: usize, want: usize) -> std::io::Result<usize> {
|
||||
if data_len < want {
|
||||
return Err(short_shard_read(data_len, want));
|
||||
return Err(short_shard_read(&self.id, data_len, want));
|
||||
}
|
||||
Ok(data_len)
|
||||
}
|
||||
}
|
||||
|
||||
/// A truncated shard is `UnexpectedEof`, not a short success (backlog#799 B2).
|
||||
fn short_shard_read(got: usize, want: usize) -> std::io::Error {
|
||||
error!(
|
||||
event = EVENT_BITROT_SHORT_SHARD_READ,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE,
|
||||
state = "failed",
|
||||
got,
|
||||
want,
|
||||
"short shard read: got {got} of {want} bytes"
|
||||
);
|
||||
fn short_shard_read(id: &Uuid, got: usize, want: usize) -> std::io::Error {
|
||||
error!("bitrot reader short shard read: id={id} got {got} of {want} bytes");
|
||||
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, format!("short shard read: got {got} of {want} bytes"))
|
||||
}
|
||||
|
||||
@@ -194,7 +184,12 @@ fn short_shard_read(got: usize, want: usize) -> std::io::Error {
|
||||
/// hash never reaches the caller's buffer. The verify duration is returned
|
||||
/// rather than stored so this stays a free function usable while `self` is
|
||||
/// borrowed for the block.
|
||||
fn split_and_verify<'a>(hash_algo: &HashAlgorithm, skip_verify: bool, block: &'a [u8]) -> std::io::Result<(&'a [u8], Duration)> {
|
||||
fn split_and_verify<'a>(
|
||||
hash_algo: &HashAlgorithm,
|
||||
skip_verify: bool,
|
||||
block: &'a [u8],
|
||||
id: &Uuid,
|
||||
) -> std::io::Result<(&'a [u8], Duration)> {
|
||||
let (hash, data) = block.split_at(hash_algo.size());
|
||||
if skip_verify {
|
||||
return Ok((data, Duration::ZERO));
|
||||
@@ -203,14 +198,7 @@ fn split_and_verify<'a>(hash_algo: &HashAlgorithm, skip_verify: bool, block: &'a
|
||||
let actual_hash = hash_algo.hash_encode(data);
|
||||
let verify = verify_start.elapsed();
|
||||
if actual_hash.as_ref() != hash {
|
||||
error!(
|
||||
event = EVENT_BITROT_HASH_MISMATCH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE,
|
||||
state = "failed",
|
||||
data_len = data.len(),
|
||||
"bitrot hash mismatch"
|
||||
);
|
||||
error!("bitrot reader hash mismatch, id={id} data_len={}", data.len());
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
|
||||
}
|
||||
Ok((data, verify))
|
||||
@@ -266,7 +254,7 @@ where
|
||||
// `need` bytes returns `None` and falls through to the scratch path,
|
||||
// keeping the short-read contract.
|
||||
if let Some(block) = self.inner.try_take_block(need) {
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &block)?;
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &block, &self.id)?;
|
||||
out.extend_from_slice(data);
|
||||
self.last_verify_duration = verify;
|
||||
return Ok(want);
|
||||
@@ -276,7 +264,7 @@ where
|
||||
// the sink differs (`extend_from_slice` into `out` instead of
|
||||
// `copy_from_slice` into a pre-zeroed buffer).
|
||||
self.read_scratch_block(need, want).await?;
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need])?;
|
||||
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need], &self.id)?;
|
||||
out.extend_from_slice(data);
|
||||
self.last_verify_duration = verify;
|
||||
Ok(want)
|
||||
|
||||
@@ -29,7 +29,6 @@ use crate::set_disk::shard_source::{ShardReadCost, ShardStripeSource, StripeRead
|
||||
use futures::FutureExt;
|
||||
use futures::stream::{FuturesUnordered, StreamExt};
|
||||
use pin_project_lite::pin_project;
|
||||
use smallvec::{SmallVec, smallvec};
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::io::ErrorKind;
|
||||
@@ -41,15 +40,9 @@ use tracing::{debug, error, warn};
|
||||
|
||||
type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, bool)> + Send + 'a>>;
|
||||
|
||||
const INLINE_SHARD_SLOTS: usize = 32;
|
||||
type ShardBuffers = SmallVec<[Option<Vec<u8>>; INLINE_SHARD_SLOTS]>;
|
||||
type ShardErrors = SmallVec<[Option<Error>; INLINE_SHARD_SLOTS]>;
|
||||
type ShardIndexes = SmallVec<[usize; INLINE_SHARD_SLOTS]>;
|
||||
type ActiveReaders = SmallVec<[bool; INLINE_SHARD_SLOTS]>;
|
||||
|
||||
/// One stripe's worth of shard buffers plus the per-shard read errors, as
|
||||
/// returned by `ParallelReader::read` / `read_stripe_timed`.
|
||||
type StripeReadOutput = (ShardBuffers, ShardErrors);
|
||||
type StripeReadOutput = (Vec<Option<Vec<u8>>>, Vec<Option<Error>>);
|
||||
|
||||
const ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING: &str = "RUSTFS_SHARD_LOCALITY_SCHEDULING";
|
||||
const ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE: &str = "RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE";
|
||||
@@ -397,7 +390,7 @@ pub(crate) struct ParallelReader<R> {
|
||||
// start, parity slots only once a data shard is missing/dead. Unengaged
|
||||
// parity stays an unopened deferred reader; `deferred_handles[i]` realigns
|
||||
// it to the current stripe when it is engaged mid-object (backlog#923).
|
||||
engaged: SmallVec<[bool; INLINE_SHARD_SLOTS]>,
|
||||
engaged: Vec<bool>,
|
||||
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
|
||||
stripe_index: usize,
|
||||
}
|
||||
@@ -580,7 +573,7 @@ where
|
||||
// behavior. With the gate on, only data slots start engaged; parity is
|
||||
// engaged on demand, stripe-aligned through its deferred handle.
|
||||
let data_shards_only = get_lockstep_data_shards_only_enabled();
|
||||
let engaged: SmallVec<_> = (0..readers.len())
|
||||
let engaged = (0..readers.len())
|
||||
.map(|index| !data_shards_only || index < e.data_shards)
|
||||
.collect();
|
||||
ParallelReader {
|
||||
@@ -619,7 +612,7 @@ where
|
||||
fn record_shard_read_result(
|
||||
shards: &mut [Option<Vec<u8>>],
|
||||
errs: &mut [Option<Error>],
|
||||
retire_readers: &mut ShardIndexes,
|
||||
retire_readers: &mut Vec<usize>,
|
||||
success: &mut usize,
|
||||
successful_costs: &mut ShardReadCostCounts,
|
||||
i: usize,
|
||||
@@ -644,7 +637,7 @@ fn record_shard_read_result(
|
||||
}
|
||||
}
|
||||
|
||||
fn retire_abandoned_readers(errs: &mut [Option<Error>], retire_readers: &mut ShardIndexes, active_readers: &[bool]) {
|
||||
fn retire_abandoned_readers(errs: &mut [Option<Error>], retire_readers: &mut Vec<usize>, active_readers: &[bool]) {
|
||||
for (i, active) in active_readers.iter().enumerate() {
|
||||
if !*active {
|
||||
continue;
|
||||
@@ -699,7 +692,7 @@ where
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
#[hotpath::measure(impl_type = "ParallelReader")]
|
||||
pub async fn read(&mut self) -> StripeReadOutput {
|
||||
pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
|
||||
// On the reconstruction-verifying GET path, read every live shard reader
|
||||
// in lockstep so all readers advance one block per stripe and stay
|
||||
// mutually aligned. The adaptive data-first path below only reads
|
||||
@@ -723,7 +716,7 @@ where
|
||||
};
|
||||
|
||||
if shard_size == 0 {
|
||||
return (smallvec![None; num_readers], smallvec![None; num_readers]);
|
||||
return (vec![None; num_readers], vec![None; num_readers]);
|
||||
}
|
||||
|
||||
// Advance to the next stripe so the following read() computes the correct
|
||||
@@ -734,8 +727,8 @@ where
|
||||
// is only read above to derive `shard_size`, so advancing here is safe.
|
||||
self.offset += shard_size;
|
||||
|
||||
let mut shards: ShardBuffers = smallvec![None; num_readers];
|
||||
let mut errs: ShardErrors = smallvec![None; num_readers];
|
||||
let mut shards: Vec<Option<Vec<u8>>> = vec![None; num_readers];
|
||||
let mut errs = vec![None; num_readers];
|
||||
let read_costs = self.read_costs.as_slice();
|
||||
let locality_preference_enabled = self.locality_preference_enabled;
|
||||
let low_cost_available = self
|
||||
@@ -766,11 +759,11 @@ where
|
||||
|
||||
self.buffers.ensure_slots(num_readers);
|
||||
|
||||
let mut retire_readers = ShardIndexes::new();
|
||||
let mut retire_readers = Vec::new();
|
||||
if num_readers >= self.data_shards {
|
||||
let mut reader_iter = ReaderLaunchIter::new(&mut self.readers, read_costs, locality_preference_enabled);
|
||||
let mut sets = FuturesUnordered::new();
|
||||
let mut active_readers: ActiveReaders = smallvec![false; num_readers];
|
||||
let mut active_readers = vec![false; num_readers];
|
||||
let stripe_read_start = self.metrics_path.map(|_| Instant::now());
|
||||
let mut scheduled = 0usize;
|
||||
for _ in 0..self.data_shards {
|
||||
@@ -1030,7 +1023,7 @@ where
|
||||
/// stripe would reintroduce the desync. A parity reader that cannot be
|
||||
/// realigned (no pending deferred handle) is likewise retired instead of
|
||||
/// being read out of position.
|
||||
async fn read_lockstep(&mut self) -> StripeReadOutput {
|
||||
async fn read_lockstep(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
|
||||
let num_readers = self.readers.len();
|
||||
let shard_size = if self.offset + self.shard_size > self.shard_file_size {
|
||||
self.shard_file_size - self.offset
|
||||
@@ -1038,8 +1031,8 @@ where
|
||||
self.shard_size
|
||||
};
|
||||
|
||||
let mut shards: ShardBuffers = smallvec![None; num_readers];
|
||||
let mut errs: ShardErrors = smallvec![None; num_readers];
|
||||
let mut shards: Vec<Option<Vec<u8>>> = vec![None; num_readers];
|
||||
let mut errs: Vec<Option<Error>> = vec![None; num_readers];
|
||||
if shard_size == 0 {
|
||||
return (shards, errs);
|
||||
}
|
||||
@@ -1078,7 +1071,7 @@ where
|
||||
// Pre-claim per-slot buffers so the `self.readers` borrow below stays
|
||||
// disjoint from `self.buffers`; `Some(buffer)` also records which slots
|
||||
// participate, avoiding a per-stripe sidecar allocation.
|
||||
let mut bufs: ShardBuffers = SmallVec::with_capacity(num_readers);
|
||||
let mut bufs: Vec<Option<Vec<u8>>> = Vec::with_capacity(num_readers);
|
||||
for i in 0..num_readers {
|
||||
bufs.push(if self.engaged[i] && self.readers[i].is_some() {
|
||||
Some(self.buffers.take(i, shard_size))
|
||||
@@ -1093,7 +1086,7 @@ where
|
||||
let locality_preference_enabled = self.locality_preference_enabled;
|
||||
let stripe_read_start = metrics_path.map(|_| Instant::now());
|
||||
|
||||
let mut retire_readers = ShardIndexes::new();
|
||||
let mut retire_readers = Vec::new();
|
||||
let mut scheduled = 0usize;
|
||||
let mut success = 0usize;
|
||||
let mut completed = 0usize;
|
||||
@@ -1358,7 +1351,10 @@ fn get_data_block_len(shards: &[Option<Vec<u8>>], data_blocks: usize) -> usize {
|
||||
/// stripe-read stage timer. Factored out so the depth-1 prefetch loop and the
|
||||
/// serial loop time reads identically. A free `async fn` (rather than a closure)
|
||||
/// so the returned future's borrow of `reader` is correctly tied to the call.
|
||||
async fn read_stripe_timed<R>(reader: &mut ParallelReader<R>, stage_metrics_enabled: bool) -> StripeReadOutput
|
||||
async fn read_stripe_timed<R>(
|
||||
reader: &mut ParallelReader<R>,
|
||||
stage_metrics_enabled: bool,
|
||||
) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>)
|
||||
where
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
@@ -1971,32 +1967,6 @@ mod tests {
|
||||
|
||||
type BoxedShardReader = crate::io_support::bitrot::ShardReader;
|
||||
|
||||
#[test]
|
||||
fn shard_scratch_stays_inline_through_the_common_limit_and_spills_safely() {
|
||||
let inline: ShardBuffers = smallvec![None; INLINE_SHARD_SLOTS];
|
||||
assert!(!inline.spilled(), "the common shard-count boundary must not allocate");
|
||||
|
||||
let spilled: ShardBuffers = smallvec![None; INLINE_SHARD_SLOTS + 1];
|
||||
assert!(spilled.spilled(), "larger supported shard counts must fall back to the heap");
|
||||
assert_eq!(spilled.len(), INLINE_SHARD_SLOTS + 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parallel_reader_preserves_slot_count_above_inline_capacity() {
|
||||
const DATA_SHARDS: usize = INLINE_SHARD_SLOTS;
|
||||
const TOTAL_SHARDS: usize = INLINE_SHARD_SLOTS + 1;
|
||||
let readers = std::iter::repeat_with(|| None).take(TOTAL_SHARDS).collect();
|
||||
let erasure = Erasure::new(DATA_SHARDS, 1, DATA_SHARDS);
|
||||
let mut reader: ParallelReader<Cursor<Vec<u8>>> = ParallelReader::new(readers, erasure, 0, DATA_SHARDS);
|
||||
|
||||
let (shards, errors) = reader.read().await;
|
||||
|
||||
assert!(shards.spilled());
|
||||
assert!(errors.spilled());
|
||||
assert_eq!(shards.len(), TOTAL_SHARDS);
|
||||
assert_eq!(errors.len(), TOTAL_SHARDS);
|
||||
}
|
||||
|
||||
/// Counts the raw bytes pulled from a shard stream, to prove which shards
|
||||
/// a decode path actually touches (backlog#923 call-count evidence).
|
||||
struct CountingShardReader {
|
||||
@@ -2373,19 +2343,6 @@ mod tests {
|
||||
assert_eq!(err.expect("range beyond total length should fail").kind(), ErrorKind::InvalidInput);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_erasure_decode_zero_length_does_not_read_or_emit() {
|
||||
let erasure = Erasure::new(2, 1, 64);
|
||||
let readers: Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> = vec![None, None, None];
|
||||
let mut output = Vec::new();
|
||||
|
||||
let (written, err) = erasure.decode(&mut output, readers, 0, 0, 0).await;
|
||||
|
||||
assert_eq!(written, 0);
|
||||
assert!(err.is_none());
|
||||
assert!(output.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_erasure_decode_with_read_costs_restores_missing_data_shard_range() {
|
||||
const DATA_SHARDS: usize = 2;
|
||||
|
||||
@@ -91,11 +91,6 @@ fn use_bytesmut_ingest() -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
fn small_ingest_capacity(erasure: &Erasure, size_hint: usize) -> usize {
|
||||
let data_len = size_hint.min(erasure.block_size);
|
||||
erasure.encoded_capacity_for_data_len(data_len).min(erasure.block_size)
|
||||
}
|
||||
|
||||
/// Keeps the encoder producer scoped to its parent future. Tokio detaches a
|
||||
/// task when its `JoinHandle` is dropped, so the producer must be aborted when
|
||||
/// an upload is cancelled before the encode pipeline finishes.
|
||||
@@ -545,14 +540,13 @@ impl Erasure {
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
require_single_block: bool,
|
||||
size_hint: usize,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let mut buf = Vec::with_capacity(small_ingest_capacity(&self, size_hint));
|
||||
let mut buf = Vec::with_capacity(self.block_size);
|
||||
let total = if require_single_block {
|
||||
let read_limit = self
|
||||
.block_size
|
||||
@@ -886,24 +880,7 @@ impl Erasure {
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
let size_hint = self.block_size;
|
||||
self.encode_small_direct(reader, writers, quorum, false, size_hint).await
|
||||
}
|
||||
|
||||
/// Size-aware inline fast path. `size_hint` only controls the bounded initial
|
||||
/// allocation; reads remain authoritative.
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub async fn encode_inline_small_with_size_hint<R>(
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
size_hint: usize,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
self.encode_small_direct(reader, writers, quorum, false, size_hint).await
|
||||
self.encode_small_direct(reader, writers, quorum, false).await
|
||||
}
|
||||
|
||||
/// Fast path for single-block non-inline objects: avoids the producer/consumer
|
||||
@@ -918,24 +895,7 @@ impl Erasure {
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
let size_hint = self.block_size;
|
||||
self.encode_small_direct(reader, writers, quorum, true, size_hint).await
|
||||
}
|
||||
|
||||
/// Size-aware single-block fast path. `size_hint` only controls the bounded
|
||||
/// initial allocation; reads remain authoritative.
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub async fn encode_single_block_non_inline_with_size_hint<R>(
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
writers: &mut [Option<BitrotWriterWrapper>],
|
||||
quorum: usize,
|
||||
size_hint: usize,
|
||||
) -> std::io::Result<(R, usize)>
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin,
|
||||
{
|
||||
self.encode_small_direct(reader, writers, quorum, true, size_hint).await
|
||||
self.encode_small_direct(reader, writers, quorum, true).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2333,10 +2293,7 @@ mod tests {
|
||||
|
||||
let erasure = Arc::new(Erasure::new(1, 0, 16));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(Vec::<u8>::new()));
|
||||
let (_reader, total) = erasure
|
||||
.encode_inline_small_with_size_hint(reader, &mut writers, 1, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
let (_reader, total) = erasure.encode_inline_small(reader, &mut writers, 1).await.unwrap();
|
||||
|
||||
assert_eq!(total, 0);
|
||||
// No shutdown was called, so nothing should be committed
|
||||
@@ -2368,10 +2325,7 @@ mod tests {
|
||||
let payload = b"hello inline small";
|
||||
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(payload.to_vec()));
|
||||
let (_reader, total) = erasure
|
||||
.encode_inline_small_with_size_hint(reader, &mut writers, DATA_SHARDS, 1)
|
||||
.await
|
||||
.unwrap();
|
||||
let (_reader, total) = erasure.encode_inline_small(reader, &mut writers, DATA_SHARDS).await.unwrap();
|
||||
|
||||
assert_eq!(total, payload.len());
|
||||
// All shards must have received data (shutdown flushed the bitrot header + shard bytes)
|
||||
@@ -2438,7 +2392,7 @@ mod tests {
|
||||
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
|
||||
let reader = tokio::io::BufReader::new(Cursor::new(payload));
|
||||
let err = erasure
|
||||
.encode_single_block_non_inline_with_size_hint(reader, &mut writers, DATA_SHARDS, BLOCK_SIZE)
|
||||
.encode_single_block_non_inline(reader, &mut writers, DATA_SHARDS)
|
||||
.await
|
||||
.expect_err("single-block fast path must reject oversized readers");
|
||||
|
||||
@@ -2449,21 +2403,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn small_ingest_capacity_uses_bounded_size_hint() {
|
||||
let erasure = Erasure::new(4, 2, 1024 * 1024);
|
||||
assert_eq!(small_ingest_capacity(&erasure, 0), 0);
|
||||
assert_eq!(small_ingest_capacity(&erasure, 4 * 1024), 6 * 1024);
|
||||
assert_eq!(small_ingest_capacity(&erasure, 16 * 1024), 24 * 1024);
|
||||
assert_eq!(small_ingest_capacity(&erasure, usize::MAX), 1024 * 1024);
|
||||
|
||||
let legacy = Erasure::new_with_options(4, 2, 1024 * 1024, true);
|
||||
assert_eq!(small_ingest_capacity(&legacy, 4 * 1024), 6 * 1024);
|
||||
|
||||
let high_parity = Erasure::new(4, 12, 1024 * 1024);
|
||||
assert_eq!(small_ingest_capacity(&high_parity, usize::MAX), 1024 * 1024);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_full_buf_or_eof_returns_none_on_empty_reader() {
|
||||
let mut reader = Cursor::new(Vec::<u8>::new());
|
||||
|
||||
@@ -968,15 +968,6 @@ impl Erasure {
|
||||
self.data_shards + self.parity_shards
|
||||
}
|
||||
|
||||
pub(crate) fn encoded_capacity_for_data_len(&self, data_len: usize) -> usize {
|
||||
let shard_size_fn = if self.uses_legacy {
|
||||
calc_shard_size_legacy
|
||||
} else {
|
||||
calc_shard_size
|
||||
};
|
||||
shard_size_fn(data_len, self.data_shards).saturating_mul(self.total_shard_count())
|
||||
}
|
||||
|
||||
/// Whether the erasure dimensions are safe for the shard/offset arithmetic.
|
||||
///
|
||||
/// `block_size` and `data_shards` come straight from on-disk metadata; a
|
||||
|
||||
@@ -204,8 +204,6 @@ pub enum StorageError {
|
||||
required: usize,
|
||||
achieved: usize,
|
||||
},
|
||||
#[error("Bucket quota exceeded. Current usage: {current} bytes, limit: {limit} bytes")]
|
||||
QuotaExceeded { current: u64, limit: u64 },
|
||||
|
||||
// ── Generic ──────────────────────────────────────────────────────
|
||||
#[error("Unexpected error")]
|
||||
@@ -358,13 +356,6 @@ impl From<StorageError> for DiskError {
|
||||
StorageError::VolumeNotFound => DiskError::VolumeNotFound,
|
||||
StorageError::VolumeExists => DiskError::VolumeExists,
|
||||
StorageError::FileNameTooLong => DiskError::FileNameTooLong,
|
||||
StorageError::FaultyRemoteDisk => DiskError::FaultyRemoteDisk,
|
||||
StorageError::DiskAccessDenied => DiskError::DiskAccessDenied,
|
||||
StorageError::DriveIsRoot => DiskError::DriveIsRoot,
|
||||
StorageError::IsNotRegular => DiskError::IsNotRegular,
|
||||
StorageError::VolumeNotEmpty => DiskError::VolumeNotEmpty,
|
||||
StorageError::VolumeAccessDenied => DiskError::VolumeAccessDenied,
|
||||
StorageError::FileAccessDenied => DiskError::FileAccessDenied,
|
||||
_ => DiskError::other(val),
|
||||
}
|
||||
}
|
||||
@@ -549,10 +540,6 @@ impl Clone for StorageError {
|
||||
required: *required,
|
||||
achieved: *achieved,
|
||||
},
|
||||
StorageError::QuotaExceeded { current, limit } => StorageError::QuotaExceeded {
|
||||
current: *current,
|
||||
limit: *limit,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -640,7 +627,6 @@ impl StorageError {
|
||||
StorageError::NotModified => StorageErrorCode::NotModified,
|
||||
StorageError::InvalidPartNumber(_) => StorageErrorCode::InvalidPartNumber,
|
||||
StorageError::NamespaceLockQuorumUnavailable { .. } => StorageErrorCode::NamespaceLockQuorumUnavailable,
|
||||
StorageError::QuotaExceeded { .. } => StorageErrorCode::QuotaExceeded,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -766,10 +752,6 @@ impl StorageError {
|
||||
required: Default::default(),
|
||||
achieved: Default::default(),
|
||||
}),
|
||||
StorageErrorCode::QuotaExceeded => Some(StorageError::QuotaExceeded {
|
||||
current: Default::default(),
|
||||
limit: Default::default(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1319,7 +1301,6 @@ mod tests {
|
||||
.to_u32(),
|
||||
0x42
|
||||
);
|
||||
assert_eq!(StorageError::QuotaExceeded { current: 1, limit: 2 }.to_u32(), 0x53);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1338,10 +1319,6 @@ mod tests {
|
||||
StorageError::from_u32(0x42),
|
||||
Some(StorageError::NamespaceLockQuorumUnavailable { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
StorageError::from_u32(0x53),
|
||||
Some(StorageError::QuotaExceeded { current: 0, limit: 0 })
|
||||
));
|
||||
|
||||
// Test invalid code returns None
|
||||
assert!(StorageError::from_u32(0xFF).is_none());
|
||||
@@ -1499,49 +1476,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
// Every DiskError variant must survive DiskError -> StorageError -> DiskError
|
||||
// unchanged. A variant that degrades to `DiskError::Io` on the way back loses
|
||||
// its identity for quorum aggregation (`reduce_errs` classifies by variant
|
||||
// equality), so ignore-list entries such as FaultyRemoteDisk and
|
||||
// DiskAccessDenied would silently stop matching.
|
||||
#[test]
|
||||
fn test_disk_error_storage_error_round_trip_identity_all_variants() {
|
||||
// DiskError codes are contiguous from 0x01, so enumerating via from_u32
|
||||
// covers every variant and picks up newly appended ones automatically.
|
||||
let all_variants: Vec<DiskError> = (1u32..).map_while(DiskError::from_u32).collect();
|
||||
assert!(
|
||||
all_variants.len() >= 42,
|
||||
"DiskError variant enumeration shrank: got {}, expected at least 42",
|
||||
all_variants.len()
|
||||
);
|
||||
|
||||
for original in all_variants {
|
||||
let storage_error: StorageError = original.clone().into();
|
||||
let round_tripped: DiskError = storage_error.into();
|
||||
|
||||
assert_eq!(
|
||||
std::mem::discriminant(&original),
|
||||
std::mem::discriminant(&round_tripped),
|
||||
"round trip changed variant: {original:?} -> {round_tripped:?}"
|
||||
);
|
||||
assert_eq!(original, round_tripped, "round trip not identical for {original:?}");
|
||||
}
|
||||
|
||||
// Io is the only payload-carrying variant: a representative kind and
|
||||
// message must both survive the round trip.
|
||||
let io_original = DiskError::Io(IoError::new(ErrorKind::PermissionDenied, "denied"));
|
||||
let storage_error: StorageError = io_original.clone().into();
|
||||
let io_round_tripped: DiskError = storage_error.into();
|
||||
assert_eq!(io_original, io_round_tripped);
|
||||
match io_round_tripped {
|
||||
DiskError::Io(inner) => {
|
||||
assert_eq!(inner.kind(), ErrorKind::PermissionDenied);
|
||||
assert_eq!(inner.to_string(), "denied");
|
||||
}
|
||||
other => panic!("expected DiskError::Io, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_storage_error_from_io_error() {
|
||||
// Test direct IO error conversion
|
||||
@@ -1615,7 +1549,6 @@ mod tests {
|
||||
StorageError::DecommissionAlreadyRunning,
|
||||
StorageError::RebalanceAlreadyRunning,
|
||||
StorageError::OperationCanceled,
|
||||
StorageError::QuotaExceeded { current: 1, limit: 2 },
|
||||
];
|
||||
|
||||
for original_error in test_errors {
|
||||
|
||||
@@ -120,41 +120,26 @@ struct BitrotReaderSource {
|
||||
|
||||
impl BitrotReaderSource {
|
||||
async fn open(self) -> disk::error::Result<Option<BoxedObjectReader>> {
|
||||
open_reader_source(
|
||||
self.inline_data,
|
||||
self.disk.as_ref(),
|
||||
&self.bucket,
|
||||
&self.path,
|
||||
self.offset,
|
||||
self.length,
|
||||
self.use_mmap_read,
|
||||
self.stage_metrics.map(|metrics| metrics.path),
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn open_reader_source(
|
||||
inline_data: Option<Bytes>,
|
||||
disk: Option<&DiskStore>,
|
||||
bucket: &str,
|
||||
path: &str,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
use_mmap_read: bool,
|
||||
metrics_path: Option<&'static str>,
|
||||
) -> disk::error::Result<Option<BoxedObjectReader>> {
|
||||
if let Some(data) = inline_data {
|
||||
let mut reader = Cursor::new(data);
|
||||
reader.set_position(u64::try_from(offset).map_err(|_| DiskError::FileCorrupt)?);
|
||||
Ok(Some(ShardReader::InMemory(reader)))
|
||||
} else if let Some(disk) = disk {
|
||||
open_disk_reader(disk, bucket, path, offset, length, use_mmap_read, metrics_path)
|
||||
if let Some(data) = self.inline_data {
|
||||
let mut rd = Cursor::new(data);
|
||||
let offset = u64::try_from(self.offset).map_err(|_| DiskError::FileCorrupt)?;
|
||||
rd.set_position(offset);
|
||||
Ok(Some(ShardReader::InMemory(rd)))
|
||||
} else if let Some(disk) = self.disk {
|
||||
open_disk_reader(
|
||||
&disk,
|
||||
&self.bucket,
|
||||
&self.path,
|
||||
self.offset,
|
||||
self.length,
|
||||
self.use_mmap_read,
|
||||
self.stage_metrics.map(|metrics| metrics.path),
|
||||
)
|
||||
.await
|
||||
.map(Some)
|
||||
} else {
|
||||
Ok(None)
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -638,22 +623,22 @@ async fn create_bitrot_reader_from_bytes_with_stage_metrics(
|
||||
|
||||
let reader_construction_start = stage_metrics_enabled.then(Instant::now);
|
||||
let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone());
|
||||
let source = BitrotReaderSource {
|
||||
inline_data,
|
||||
disk: disk.cloned(),
|
||||
bucket: bucket.to_string(),
|
||||
path: path.to_string(),
|
||||
offset,
|
||||
length,
|
||||
use_mmap_read,
|
||||
stage_metrics,
|
||||
};
|
||||
if let Some(metrics) = stage_metrics {
|
||||
record_get_stage_duration_if_enabled(metrics.path, metrics.reader_construction_stage, reader_construction_start);
|
||||
}
|
||||
|
||||
let file_open_start = stage_metrics_enabled.then(Instant::now);
|
||||
let reader = open_reader_source(
|
||||
inline_data,
|
||||
disk,
|
||||
bucket,
|
||||
path,
|
||||
offset,
|
||||
length,
|
||||
use_mmap_read,
|
||||
stage_metrics.map(|metrics| metrics.path),
|
||||
)
|
||||
.await?;
|
||||
let reader = source.open().await?;
|
||||
if let Some(metrics) = stage_metrics {
|
||||
record_get_stage_duration_if_enabled(metrics.path, metrics.file_open_stage, file_open_start);
|
||||
}
|
||||
@@ -713,12 +698,11 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
|
||||
) -> (BitrotReader<ShardReader>, DeferredReaderStripeHandle) {
|
||||
let stripe_stride = shard_size + checksum_algo.size();
|
||||
let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone());
|
||||
let inline_source = inline_data.is_some();
|
||||
let source = BitrotReaderSource {
|
||||
inline_data,
|
||||
disk,
|
||||
bucket: if inline_source { String::new() } else { bucket.to_string() },
|
||||
path: if inline_source { String::new() } else { path.to_string() },
|
||||
bucket: bucket.to_string(),
|
||||
path: path.to_string(),
|
||||
offset,
|
||||
length,
|
||||
use_mmap_read,
|
||||
|
||||
@@ -234,17 +234,11 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn test_format_v1() {
|
||||
// A freshly created format must survive a serialize -> parse roundtrip
|
||||
// unchanged (identity on every on-disk field).
|
||||
let format = FormatV3::new(1, 4);
|
||||
let serialized = serde_json::to_string(&format).expect("FormatV3 must serialize to JSON");
|
||||
let reparsed = FormatV3::try_from(serialized.as_str()).expect("serialized FormatV3 must parse back");
|
||||
assert_eq!(reparsed, format);
|
||||
|
||||
// minio-file-format-compat: this literal pins the on-disk format.json
|
||||
// shape (erasure version "1", distributionAlgo "CRCMOD"). `this` always
|
||||
// carries the disk's own UUID in real format.json files; a JSON null
|
||||
// there was never parseable and never written by MinIO or RustFS.
|
||||
let str = serde_json::to_string(&format);
|
||||
println!("{str:?}");
|
||||
|
||||
let data = r#"
|
||||
{
|
||||
"version": "1",
|
||||
@@ -252,7 +246,7 @@ mod test {
|
||||
"id": "321b3874-987d-4c15-8fa5-757c956b1243",
|
||||
"xl": {
|
||||
"version": "1",
|
||||
"this": "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
|
||||
"this": null,
|
||||
"sets": [
|
||||
[
|
||||
"8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
|
||||
@@ -265,23 +259,9 @@ mod test {
|
||||
}
|
||||
}"#;
|
||||
|
||||
let parsed = FormatV3::try_from(data).expect("pinned v1 format.json literal must keep parsing");
|
||||
let p = FormatV3::try_from(data);
|
||||
|
||||
assert_eq!(parsed.version, FormatMetaVersion::V1);
|
||||
assert_eq!(parsed.format, FormatBackend::Erasure);
|
||||
assert_eq!(
|
||||
parsed.id,
|
||||
Uuid::parse_str("321b3874-987d-4c15-8fa5-757c956b1243").expect("literal id is a valid UUID")
|
||||
);
|
||||
assert_eq!(parsed.erasure.version, FormatErasureVersion::V1);
|
||||
assert_eq!(
|
||||
parsed.erasure.this,
|
||||
Uuid::parse_str("8ab9a908-f869-4f1f-8e42-eb067ffa7eb5").expect("literal this is a valid UUID")
|
||||
);
|
||||
assert_eq!(parsed.erasure.sets.len(), 1);
|
||||
assert_eq!(parsed.erasure.sets[0].len(), 4);
|
||||
assert_eq!(parsed.erasure.sets[0][0], parsed.erasure.this);
|
||||
assert_eq!(parsed.erasure.distribution_algo, DistributionAlgoVersion::V1);
|
||||
println!("{p:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -211,26 +211,6 @@ impl ObjectLockConfigSnapshot {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct QuotaAdmission {
|
||||
current_usage: u64,
|
||||
quota_limit: u64,
|
||||
}
|
||||
|
||||
impl QuotaAdmission {
|
||||
pub(crate) fn current_usage(self) -> u64 {
|
||||
self.current_usage
|
||||
}
|
||||
|
||||
pub(crate) fn quota_limit(self) -> u64 {
|
||||
self.quota_limit
|
||||
}
|
||||
|
||||
pub(crate) fn remaining(self) -> u64 {
|
||||
self.quota_limit - self.current_usage
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct ObjectOptions {
|
||||
// Use the maximum parity (N/2), used when saving server configuration files
|
||||
@@ -295,22 +275,12 @@ pub struct ObjectOptions {
|
||||
pub want_checksum: Option<Checksum>,
|
||||
pub skip_verify_bitrot: bool,
|
||||
pub capacity_scope_token: Option<Uuid>,
|
||||
/// Server-derived bucket-quota snapshot for commit-boundary admission.
|
||||
pub quota_admission: Option<QuotaAdmission>,
|
||||
/// Storage-owned journal writer used by the atomic delete path. This is
|
||||
/// populated only by the `ECStore` wrapper that holds the namespace locks.
|
||||
pub tier_delete_journal_api: Option<Arc<crate::store::ECStore>>,
|
||||
}
|
||||
|
||||
impl ObjectOptions {
|
||||
pub fn set_quota_admission(&mut self, current_usage: u64, quota_limit: u64) -> bool {
|
||||
self.quota_admission = (current_usage <= quota_limit).then_some(QuotaAdmission {
|
||||
current_usage,
|
||||
quota_limit,
|
||||
});
|
||||
self.quota_admission.is_some()
|
||||
}
|
||||
|
||||
pub(crate) fn overwrites_existing_version(&self) -> bool {
|
||||
self.version_id.is_some() || !self.versioned || self.version_suspended
|
||||
}
|
||||
|
||||
@@ -58,7 +58,6 @@ use crate::io_support::bitrot::{
|
||||
create_deferred_bitrot_reader_with_stripe_handle, object_mmap_read_enabled, object_mmap_read_max_length,
|
||||
};
|
||||
use crate::set_disk::shard_source::ShardReadCost;
|
||||
use futures::FutureExt as _;
|
||||
use futures::stream::{FuturesUnordered, StreamExt};
|
||||
use metrics::counter;
|
||||
use std::{
|
||||
@@ -222,7 +221,7 @@ impl MetadataFanoutDiagnostics {
|
||||
self.observations.iter().filter(|observation| observation.ignored).count()
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn non_valid_responses(&self) -> usize {
|
||||
pub(in crate::set_disk) fn error_responses(&self) -> usize {
|
||||
self.total_responses().saturating_sub(self.valid_responses())
|
||||
}
|
||||
|
||||
@@ -273,7 +272,7 @@ impl MetadataFanoutDiagnostics {
|
||||
self.total_responses(),
|
||||
self.valid_responses(),
|
||||
self.ignored_responses(),
|
||||
self.non_valid_responses(),
|
||||
self.error_responses(),
|
||||
);
|
||||
for observation in &self.observations {
|
||||
rustfs_io_metrics::record_get_object_metadata_response(path, observation.outcome);
|
||||
@@ -2223,18 +2222,18 @@ impl SetDisks {
|
||||
let mut ress = Vec::with_capacity(disks.len());
|
||||
let mut errors = Vec::with_capacity(disks.len());
|
||||
let mut observations = observe.then(|| Vec::with_capacity(disks.len()));
|
||||
let opts = ReadOptions {
|
||||
let opts = Arc::new(ReadOptions {
|
||||
incl_free_versions,
|
||||
read_data,
|
||||
healing,
|
||||
};
|
||||
let org_bucket: Arc<str> = Arc::from(org_bucket);
|
||||
let bucket: Arc<str> = Arc::from(bucket);
|
||||
let object: Arc<str> = Arc::from(object);
|
||||
let version_id: Arc<str> = Arc::from(version_id);
|
||||
});
|
||||
let org_bucket = Arc::new(org_bucket.to_string());
|
||||
let bucket = Arc::new(bucket.to_string());
|
||||
let object = Arc::new(object.to_string());
|
||||
let version_id = Arc::new(version_id.to_string());
|
||||
let futures = disks.iter().enumerate().map(|(disk_index, disk)| {
|
||||
let disk = disk.clone();
|
||||
let task_opts = opts;
|
||||
let opts = opts.clone();
|
||||
let org_bucket = org_bucket.clone();
|
||||
let bucket = bucket.clone();
|
||||
let object = object.clone();
|
||||
@@ -2243,8 +2242,7 @@ impl SetDisks {
|
||||
let response_start = observe.then(Instant::now);
|
||||
let result = if let Some(disk) = disk {
|
||||
Self::record_read_version_call(&object, disk_index);
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||
.await
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &opts).await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
};
|
||||
@@ -2309,21 +2307,21 @@ impl SetDisks {
|
||||
let mut observations = Vec::with_capacity(disks.len());
|
||||
let mut accumulator =
|
||||
MetadataQuorumAccumulator::new(disks.len(), default_parity_count, true).with_requested_version_id(version_id);
|
||||
let opts = ReadOptions {
|
||||
let opts = Arc::new(ReadOptions {
|
||||
incl_free_versions,
|
||||
read_data,
|
||||
healing,
|
||||
};
|
||||
let org_bucket: Arc<str> = Arc::from(org_bucket);
|
||||
let bucket: Arc<str> = Arc::from(bucket);
|
||||
let object: Arc<str> = Arc::from(object);
|
||||
let version_id: Arc<str> = Arc::from(version_id);
|
||||
});
|
||||
let org_bucket = Arc::new(org_bucket.to_string());
|
||||
let bucket = Arc::new(bucket.to_string());
|
||||
let object = Arc::new(object.to_string());
|
||||
let version_id = Arc::new(version_id.to_string());
|
||||
let mut join_set = JoinSet::new();
|
||||
let bounded_fanout = is_get_metadata_early_stop_bounded_fanout_enabled();
|
||||
let mut next_disk_index = 0usize;
|
||||
let spawn_read_version =
|
||||
|join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| {
|
||||
let task_opts = opts;
|
||||
let opts = opts.clone();
|
||||
let org_bucket = org_bucket.clone();
|
||||
let bucket = bucket.clone();
|
||||
let object = object.clone();
|
||||
@@ -2334,8 +2332,7 @@ impl SetDisks {
|
||||
Self::record_read_version_call(&object, index);
|
||||
#[cfg(test)]
|
||||
Self::read_version_fanout_barrier(&object, index).await;
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||
.await
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &opts).await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
};
|
||||
@@ -2857,6 +2854,8 @@ impl SetDisks {
|
||||
file_info.validate_for_erasure_write()?;
|
||||
}
|
||||
}
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
@@ -2864,65 +2863,48 @@ impl SetDisks {
|
||||
let dst_bucket = Arc::new(dst_bucket.to_string());
|
||||
let dst_object = Arc::new(dst_object.to_string());
|
||||
|
||||
let disk_count = disks.len();
|
||||
let fanout_disks = disks.to_vec();
|
||||
let fanout_file_infos = file_infos.to_vec();
|
||||
let fanout_src_bucket = src_bucket.clone();
|
||||
let fanout_src_object = src_object.clone();
|
||||
let fanout_dst_bucket = dst_bucket.clone();
|
||||
let fanout_dst_object = dst_object.clone();
|
||||
// Keep one coordinator task so a cancelled caller cannot drop partially
|
||||
// completed disk mutations. Per-disk futures stay ordered in `join_all`,
|
||||
// preserving slot-indexed quorum and convergence accounting without a
|
||||
// scheduler task for every disk.
|
||||
let fanout = tokio::spawn(async move {
|
||||
let futures = fanout_disks
|
||||
.into_iter()
|
||||
.zip(fanout_file_infos)
|
||||
.enumerate()
|
||||
.map(|(i, (disk, mut file_info))| {
|
||||
let src_bucket = fanout_src_bucket.clone();
|
||||
let src_object = fanout_src_object.clone();
|
||||
let dst_object = fanout_dst_object.clone();
|
||||
let dst_bucket = fanout_dst_bucket.clone();
|
||||
for (i, (disk, file_info)) in disks.iter().zip(file_infos.iter()).enumerate() {
|
||||
let mut file_info = file_info.clone();
|
||||
let disk = disk.clone();
|
||||
let src_bucket = src_bucket.clone();
|
||||
let src_object = src_object.clone();
|
||||
let dst_object = dst_object.clone();
|
||||
let dst_bucket = dst_bucket.clone();
|
||||
|
||||
std::panic::AssertUnwindSafe(async move {
|
||||
// Test-only introspection guard: counts this operation as
|
||||
// in-flight for the whole body. Compiles to `()` in production.
|
||||
#[allow(clippy::let_unit_value)]
|
||||
let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object);
|
||||
futures.push(tokio::spawn(async move {
|
||||
// Test-only introspection guard: counts this task as in-flight for
|
||||
// the whole body. Compiles to `()` in production (no behavior).
|
||||
#[allow(clippy::let_unit_value)]
|
||||
let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object);
|
||||
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
|
||||
let is_delete_marker = file_info.is_canonical_delete_marker();
|
||||
if file_info.erasure.index == 0 {
|
||||
file_info.erasure.index = i + 1;
|
||||
}
|
||||
let is_delete_marker = file_info.is_canonical_delete_marker();
|
||||
if file_info.erasure.index == 0 {
|
||||
file_info.erasure.index = i + 1;
|
||||
}
|
||||
|
||||
if !is_delete_marker && !file_info.has_valid_erasure_geometry() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
if !is_delete_marker && !file_info.has_valid_erasure_geometry() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
|
||||
// Test-only awaitable pause point right before the disk rename.
|
||||
// A no-op immediately-ready future in production.
|
||||
Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await;
|
||||
// Test-only awaitable pause point right before the disk rename.
|
||||
// A no-op immediately-ready future in production.
|
||||
Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await;
|
||||
|
||||
disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
|
||||
.await
|
||||
})
|
||||
.catch_unwind()
|
||||
});
|
||||
join_all(futures).await
|
||||
});
|
||||
disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
|
||||
.await
|
||||
}));
|
||||
}
|
||||
|
||||
let mut disk_versions = vec![None; disk_count];
|
||||
let mut data_dirs = vec![None; disk_count];
|
||||
let mut cleanup_data_dirs = vec![None; disk_count];
|
||||
let mut old_current_sizes = vec![None; disk_count];
|
||||
let mut disk_versions = vec![None; disks.len()];
|
||||
let mut data_dirs = vec![None; disks.len()];
|
||||
let mut cleanup_data_dirs = vec![None; disks.len()];
|
||||
let mut old_current_sizes = vec![None; disks.len()];
|
||||
|
||||
let results = fanout.await.map_err(|_| DiskError::Unexpected)?;
|
||||
let results = join_all(futures).await;
|
||||
|
||||
for (idx, result) in results.iter().enumerate() {
|
||||
match result.as_ref().map_err(|_| DiskError::Unexpected)? {
|
||||
@@ -5883,51 +5865,6 @@ mod tests {
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_fanout_drains_after_caller_cancellation() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "rename-cancel-bucket";
|
||||
let object = "rename-cancel-object";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
let marker = metadata_test_delete_marker(object, Uuid::new_v4(), OffsetDateTime::now_utc());
|
||||
let file_infos = vec![marker; DISKS];
|
||||
let tracker = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
|
||||
let rename =
|
||||
tokio::spawn(
|
||||
async move { SetDisks::rename_data(&disks, bucket, object, &file_infos, bucket, object, DISKS - 1).await },
|
||||
);
|
||||
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("rename fan-out must reach the armed barrier");
|
||||
rename.abort();
|
||||
assert!(
|
||||
rename
|
||||
.await
|
||||
.expect_err("aborted caller should report cancellation")
|
||||
.is_cancelled(),
|
||||
"caller task should be cancelled, not panic"
|
||||
);
|
||||
assert!(tracker.running() >= 1, "the coordinator must retain in-flight disk mutations");
|
||||
|
||||
barrier.release();
|
||||
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
|
||||
while tracker.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cancelled caller's disk mutations must drain");
|
||||
|
||||
for (idx, dir) in dirs.iter().enumerate() {
|
||||
assert!(
|
||||
dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE).exists(),
|
||||
"disk {idx} must finish the rename after caller cancellation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Demo / regression guard for the barrier on the commit (old-data-dir)
|
||||
/// cleanup fan-out. Serves the same #1312/#1319 "no background disk write
|
||||
/// after release" shape, on the reclamation path that runs *after* a write is
|
||||
@@ -6098,7 +6035,7 @@ mod tests {
|
||||
assert_eq!(diagnostics.total_responses(), 3);
|
||||
assert_eq!(diagnostics.valid_responses(), 1);
|
||||
assert_eq!(diagnostics.ignored_responses(), 1);
|
||||
assert_eq!(diagnostics.non_valid_responses(), 2);
|
||||
assert_eq!(diagnostics.error_responses(), 2);
|
||||
assert_eq!(diagnostics.first_response_latency(), Some(Duration::from_millis(10)));
|
||||
assert_eq!(diagnostics.first_valid_response_latency(), Some(Duration::from_millis(30)));
|
||||
assert_eq!(diagnostics.slowest_response_latency(), Some(Duration::from_millis(30)));
|
||||
|
||||
@@ -584,14 +584,10 @@ fn capacity_scope_from_disks(disks: &[Option<DiskStore>]) -> CapacityScope {
|
||||
///
|
||||
/// **Deprecated**: Use `adaptive_duplex_buffer_size()` for object-size-aware sizing.
|
||||
pub fn get_duplex_buffer_size() -> usize {
|
||||
static CACHED: OnceLock<usize> = OnceLock::new();
|
||||
*CACHED.get_or_init(|| {
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_OBJECT_DUPLEX_BUFFER_SIZE,
|
||||
rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE,
|
||||
)
|
||||
.max(1)
|
||||
})
|
||||
rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_OBJECT_DUPLEX_BUFFER_SIZE,
|
||||
rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE,
|
||||
)
|
||||
}
|
||||
|
||||
/// Get adaptive duplex buffer size based on object size.
|
||||
@@ -601,15 +597,12 @@ pub fn get_duplex_buffer_size() -> usize {
|
||||
fn adaptive_duplex_buffer_size(object_size: i64) -> usize {
|
||||
const KB: usize = 1024;
|
||||
const MB: usize = 1024 * 1024;
|
||||
let target = match object_size {
|
||||
0..=131_072 => 64 * KB, // <= 128KB: 64KB
|
||||
131_073..=1_048_576 => 512 * KB, // <= 1MB: reduce duplex backpressure without a 1MB pipe per request
|
||||
match object_size {
|
||||
0..=1_048_576 => 64 * KB, // <= 1MB: 64KB
|
||||
1_048_577..=16_777_216 => MB, // <= 16MB: 1MB
|
||||
16_777_217..=268_435_456 => 4 * MB, // <= 256MB: 4MB
|
||||
_ => 8 * MB, // > 256MB: 8MB
|
||||
};
|
||||
let object_cap = usize::try_from(object_size).ok().filter(|size| *size > 0).unwrap_or(target);
|
||||
target.min(object_cap.max(64 * KB)).min(get_duplex_buffer_size())
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -639,11 +632,9 @@ const ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE: &str = "RUSTFS_GET_CODEC_STREAMING_
|
||||
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENABLE: bool = true;
|
||||
|
||||
const ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_MIN_SIZE";
|
||||
// Meet the direct-memory path at its default ceiling. Codec streaming remains
|
||||
// rollout-gated and starts where the eager small-object path ends.
|
||||
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: usize = DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD;
|
||||
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: usize = MI_B;
|
||||
const ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE";
|
||||
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: usize = DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE;
|
||||
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: usize = MI_B;
|
||||
|
||||
const ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE: &str = "RUSTFS_GET_CODEC_STREAMING_ENGINE";
|
||||
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENGINE: &str = GET_CODEC_STREAMING_ENGINE_LEGACY;
|
||||
@@ -673,13 +664,7 @@ const ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE: &str = "RUSTFS_
|
||||
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE: usize = 512 * 1024;
|
||||
|
||||
const ENV_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY: &str = "RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY";
|
||||
// On by default (rustfs/backlog#1802): a small object whose data shards are
|
||||
// inlined in xl.meta is reassembled straight from the already-resolved
|
||||
// metadata, skipping the Erasure reconstruct pipeline. The path has a complete
|
||||
// fallback — if the inline reassembly returns None, the GET proceeds through
|
||||
// the normal shard-read pipeline, so a miss is correctness-neutral. Set to
|
||||
// `false` to force the legacy path (kill switch).
|
||||
const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY: bool = true;
|
||||
const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY: bool = false;
|
||||
const ENV_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD: &str = "RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD";
|
||||
const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD: usize = 128 * 1024;
|
||||
|
||||
@@ -735,41 +720,10 @@ mod transition_matrix_tests;
|
||||
|
||||
pub use ops::heal_walk::HealWalkVersion;
|
||||
|
||||
pub(in crate::set_disk) enum GetObjectMetadata<T> {
|
||||
Owned(T),
|
||||
Shared(Arc<T>),
|
||||
}
|
||||
|
||||
impl<T> std::ops::Deref for GetObjectMetadata<T> {
|
||||
type Target = T;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
match self {
|
||||
Self::Owned(value) => value,
|
||||
Self::Shared(value) => value,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Clone> GetObjectMetadata<T> {
|
||||
fn into_owned(self) -> T {
|
||||
match self {
|
||||
Self::Owned(value) => value,
|
||||
Self::Shared(value) => Arc::try_unwrap(value).unwrap_or_else(|value| (*value).clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type GetObjectFileInfo = (
|
||||
GetObjectMetadata<FileInfo>,
|
||||
GetObjectMetadata<Vec<FileInfo>>,
|
||||
GetObjectMetadata<Vec<Option<DiskStore>>>,
|
||||
);
|
||||
|
||||
pub(crate) struct PreparedGetObjectMetadata {
|
||||
fi: GetObjectMetadata<FileInfo>,
|
||||
files: GetObjectMetadata<Vec<FileInfo>>,
|
||||
disks: GetObjectMetadata<Vec<Option<DiskStore>>>,
|
||||
fi: FileInfo,
|
||||
files: Vec<FileInfo>,
|
||||
disks: Vec<Option<DiskStore>>,
|
||||
object_info: Option<ObjectInfo>,
|
||||
}
|
||||
|
||||
@@ -838,9 +792,9 @@ mod prepared_get_object_metadata_tests {
|
||||
#[tokio::test]
|
||||
async fn prepared_metadata_is_consumed_exactly_once() {
|
||||
let metadata = PreparedGetObjectMetadata {
|
||||
fi: GetObjectMetadata::Owned(FileInfo::default()),
|
||||
files: GetObjectMetadata::Owned(Vec::new()),
|
||||
disks: GetObjectMetadata::Owned(Vec::new()),
|
||||
fi: FileInfo::default(),
|
||||
files: Vec::new(),
|
||||
disks: Vec::new(),
|
||||
object_info: None,
|
||||
};
|
||||
|
||||
@@ -1653,6 +1607,8 @@ enum GetDirectMemoryFallbackReason {
|
||||
Range,
|
||||
PartNumber,
|
||||
VersionId,
|
||||
Versioned,
|
||||
VersionSuspended,
|
||||
InclFreeVersions,
|
||||
SkipFreeVersion,
|
||||
DataMovement,
|
||||
@@ -1678,6 +1634,8 @@ impl GetDirectMemoryFallbackReason {
|
||||
Self::Range => "range",
|
||||
Self::PartNumber => "part_number",
|
||||
Self::VersionId => "version_id",
|
||||
Self::Versioned => "versioned",
|
||||
Self::VersionSuspended => "version_suspended",
|
||||
Self::InclFreeVersions => "incl_free_versions",
|
||||
Self::SkipFreeVersion => "skip_free_version",
|
||||
Self::DataMovement => "data_movement",
|
||||
@@ -1801,11 +1759,12 @@ fn get_small_object_direct_memory_decision_with_threshold(
|
||||
if opts.version_id.is_some() {
|
||||
return GetDirectMemoryDecision::Fallback(GetDirectMemoryFallbackReason::VersionId);
|
||||
}
|
||||
// Bucket-level versioning no longer blocks the inline path (rustfs/backlog#1802):
|
||||
// `fi` here is the already-resolved target version, so reassembling its inlined
|
||||
// data shards is correct whether the bucket is versioned or not. This direct-memory
|
||||
// decision still falls back for an explicit versionId (the `version_id` check above);
|
||||
// a delete-marker latest is rejected below.
|
||||
if opts.versioned {
|
||||
return GetDirectMemoryDecision::Fallback(GetDirectMemoryFallbackReason::Versioned);
|
||||
}
|
||||
if opts.version_suspended {
|
||||
return GetDirectMemoryDecision::Fallback(GetDirectMemoryFallbackReason::VersionSuspended);
|
||||
}
|
||||
if opts.incl_free_versions {
|
||||
return GetDirectMemoryDecision::Fallback(GetDirectMemoryFallbackReason::InclFreeVersions);
|
||||
}
|
||||
@@ -2367,8 +2326,6 @@ pub struct SetDisks {
|
||||
pub default_parity_count: usize,
|
||||
pub set_index: usize,
|
||||
pub pool_index: usize,
|
||||
/// Stable namespace shared by every object lock created for this set.
|
||||
set_lock_namespace: Arc<str>,
|
||||
pub format: FormatV3,
|
||||
disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
|
||||
get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>,
|
||||
@@ -2498,13 +2455,13 @@ impl Hash for GetObjectMetadataCacheKey {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[derive(Clone, Debug)]
|
||||
struct GetObjectMetadataCacheEntry {
|
||||
#[allow(dead_code)] // Kept for debugging; moka handles TTL internally
|
||||
created_at: Instant,
|
||||
fi: Arc<FileInfo>,
|
||||
parts_metadata: Arc<Vec<FileInfo>>,
|
||||
online_disks: Arc<Vec<Option<DiskStore>>>,
|
||||
fi: FileInfo,
|
||||
parts_metadata: Vec<FileInfo>,
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
read_quorum: usize,
|
||||
}
|
||||
|
||||
@@ -2770,7 +2727,6 @@ impl SetDisks {
|
||||
instance_ctx: Arc<InstanceContext>,
|
||||
) -> Arc<Self> {
|
||||
let ctx = instance_ctx;
|
||||
let set_lock_namespace: Arc<str> = format!("set-{pool_index}-{set_index}").into();
|
||||
Arc::new(SetDisks {
|
||||
locker_owner,
|
||||
disks,
|
||||
@@ -2778,7 +2734,6 @@ impl SetDisks {
|
||||
default_parity_count,
|
||||
set_index,
|
||||
pool_index,
|
||||
set_lock_namespace,
|
||||
format,
|
||||
set_endpoints,
|
||||
disk_health_cache: Arc::new(RwLock::new(Vec::new())),
|
||||
@@ -3227,28 +3182,23 @@ async fn try_read_inline_data_shards_direct(
|
||||
return None;
|
||||
}
|
||||
|
||||
let shards_needed = object_size.div_ceil(read_length);
|
||||
if shards_needed > data_shards {
|
||||
return None;
|
||||
}
|
||||
let encoded_capacity = read_length.checked_mul(shards_needed)?;
|
||||
let mut body = Vec::with_capacity(encoded_capacity);
|
||||
for reader in readers.iter_mut().take(shards_needed) {
|
||||
let mut body = Vec::with_capacity(object_size);
|
||||
let mut remaining = object_size;
|
||||
for reader in readers.iter_mut().take(data_shards) {
|
||||
let reader = reader.as_mut()?;
|
||||
let Ok(read) = reader.read_appending(&mut body, read_length).await else {
|
||||
let mut shard = vec![0u8; read_length];
|
||||
let Ok(read) = reader.read(&mut shard).await else {
|
||||
return None;
|
||||
};
|
||||
if read != read_length {
|
||||
return None;
|
||||
}
|
||||
|
||||
if body.len() >= object_size {
|
||||
let body = Bytes::from(body);
|
||||
return Some(if body.len() == object_size {
|
||||
body
|
||||
} else {
|
||||
body.slice(..object_size)
|
||||
});
|
||||
let take = remaining.min(shard.len());
|
||||
body.extend_from_slice(&shard[..take]);
|
||||
remaining -= take;
|
||||
if remaining == 0 {
|
||||
return Some(Bytes::from(body));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4938,28 +4888,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_ns_lock_reuses_the_set_namespace_allocation() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
|
||||
assert_eq!(&*set.set_lock_namespace, "set-0-0");
|
||||
let before = Arc::strong_count(&set.set_lock_namespace);
|
||||
let lock = set
|
||||
.new_ns_lock("bucket", "object")
|
||||
.await
|
||||
.expect("namespace lock should be created");
|
||||
|
||||
assert_eq!(
|
||||
Arc::strong_count(&set.set_lock_namespace),
|
||||
before + 1,
|
||||
"each lock should share the set namespace instead of formatting a new String"
|
||||
);
|
||||
drop(lock);
|
||||
assert_eq!(Arc::strong_count(&set.set_lock_namespace), before);
|
||||
}
|
||||
|
||||
struct SetupTypeGuard {
|
||||
previous: SetupType,
|
||||
}
|
||||
@@ -8746,11 +8674,9 @@ mod tests {
|
||||
128 * 1024
|
||||
));
|
||||
|
||||
// Bucket-level versioning no longer blocks the inline path (rustfs/backlog#1802):
|
||||
// a latest-version read on a versioned bucket is eligible.
|
||||
let mut versioned_opts = opts.clone();
|
||||
versioned_opts.versioned = true;
|
||||
assert!(is_get_small_object_direct_memory_eligible_with_threshold(
|
||||
assert!(!is_get_small_object_direct_memory_eligible_with_threshold(
|
||||
&None,
|
||||
&object_info,
|
||||
&fi,
|
||||
@@ -8811,13 +8737,11 @@ mod tests {
|
||||
GetDirectMemoryDecision::Fallback(GetDirectMemoryFallbackReason::Range)
|
||||
);
|
||||
|
||||
// Bucket-level versioning no longer falls back (rustfs/backlog#1802): the
|
||||
// latest version on a versioned bucket is served inline like any other.
|
||||
let mut versioned_opts = opts.clone();
|
||||
versioned_opts.versioned = true;
|
||||
assert_eq!(
|
||||
get_small_object_direct_memory_decision_with_threshold(&None, &object_info, &fi, &versioned_opts, true, 128 * 1024),
|
||||
GetDirectMemoryDecision::Use { object_size: 1024 }
|
||||
GetDirectMemoryDecision::Fallback(GetDirectMemoryFallbackReason::Versioned)
|
||||
);
|
||||
|
||||
let mut encrypted = object_info.clone();
|
||||
@@ -8865,6 +8789,8 @@ mod tests {
|
||||
assert_eq!(GetDirectMemoryFallbackReason::Range.as_str(), "range");
|
||||
assert_eq!(GetDirectMemoryFallbackReason::PartNumber.as_str(), "part_number");
|
||||
assert_eq!(GetDirectMemoryFallbackReason::VersionId.as_str(), "version_id");
|
||||
assert_eq!(GetDirectMemoryFallbackReason::Versioned.as_str(), "versioned");
|
||||
assert_eq!(GetDirectMemoryFallbackReason::VersionSuspended.as_str(), "version_suspended");
|
||||
assert_eq!(GetDirectMemoryFallbackReason::InclFreeVersions.as_str(), "incl_free_versions");
|
||||
assert_eq!(GetDirectMemoryFallbackReason::SkipFreeVersion.as_str(), "skip_free_version");
|
||||
assert_eq!(GetDirectMemoryFallbackReason::DataMovement.as_str(), "data_movement");
|
||||
@@ -8911,17 +8837,10 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
async fn inline_bitrot_files_for_payload_with_mode(
|
||||
payload: &[u8],
|
||||
uses_legacy: bool,
|
||||
) -> (coding::Erasure, Vec<FileInfo>, usize, HashAlgorithm) {
|
||||
let erasure = coding::Erasure::new_with_options(4, 2, 1024 * 1024, uses_legacy);
|
||||
async fn inline_bitrot_files_for_payload(payload: &[u8]) -> (coding::Erasure, Vec<FileInfo>, usize, HashAlgorithm) {
|
||||
let erasure = coding::Erasure::new(4, 2, 1024 * 1024);
|
||||
let read_length = erasure.shard_file_offset(0, payload.len(), payload.len());
|
||||
let checksum_algo = if uses_legacy {
|
||||
HashAlgorithm::HighwayHash256SLegacy
|
||||
} else {
|
||||
HashAlgorithm::HighwayHash256S
|
||||
};
|
||||
let checksum_algo = HashAlgorithm::HighwayHash256S;
|
||||
let shards = erasure.encode_data(payload).expect("payload should encode");
|
||||
let mut files = Vec::with_capacity(shards.len());
|
||||
|
||||
@@ -8943,10 +8862,6 @@ mod tests {
|
||||
(erasure, files, read_length, checksum_algo)
|
||||
}
|
||||
|
||||
async fn inline_bitrot_files_for_payload(payload: &[u8]) -> (coding::Erasure, Vec<FileInfo>, usize, HashAlgorithm) {
|
||||
inline_bitrot_files_for_payload_with_mode(payload, false).await
|
||||
}
|
||||
|
||||
fn inline_data_shard_fileinfo(
|
||||
name: &str,
|
||||
data_blocks: usize,
|
||||
@@ -9026,41 +8941,15 @@ mod tests {
|
||||
assert_eq!(body.as_ref(), payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inline_data_shards_direct_read_reassembles_legacy_payload_with_padding() {
|
||||
let payload = b"legacy inline payload whose size is not divisible by the data shard count";
|
||||
let (erasure, files, read_length, checksum_algo) = inline_bitrot_files_for_payload_with_mode(payload, true).await;
|
||||
assert_ne!(payload.len() % erasure.data_shards, 0, "test payload must exercise EC padding");
|
||||
let mut readers = build_inline_bitrot_readers(
|
||||
&files,
|
||||
erasure.data_shards,
|
||||
"bucket",
|
||||
"object",
|
||||
read_length,
|
||||
erasure.shard_size(),
|
||||
&checksum_algo,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("legacy inline bitrot readers should build");
|
||||
|
||||
let body = try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, payload.len())
|
||||
.await
|
||||
.expect("legacy data shard direct read should succeed");
|
||||
|
||||
assert_eq!(body.len(), payload.len());
|
||||
assert_eq!(body.as_ref(), payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inline_data_shards_direct_read_rejects_corrupt_shard() {
|
||||
let payload = b"small inline object payload that will be corrupted";
|
||||
let (erasure, mut files, read_length, checksum_algo) = inline_bitrot_files_for_payload(payload).await;
|
||||
let second = files[1].data.as_mut().expect("second shard should exist");
|
||||
let mut corrupted = second.to_vec();
|
||||
let first = files[0].data.as_mut().expect("first shard should exist");
|
||||
let mut corrupted = first.to_vec();
|
||||
let last = corrupted.last_mut().expect("encoded shard should not be empty");
|
||||
*last ^= 0xff;
|
||||
*second = Bytes::from(corrupted);
|
||||
*first = Bytes::from(corrupted);
|
||||
|
||||
let mut readers = build_inline_bitrot_readers(
|
||||
&files,
|
||||
@@ -9077,7 +8966,7 @@ mod tests {
|
||||
|
||||
let body = try_read_inline_data_shards_direct(&mut readers, 4, read_length, payload.len()).await;
|
||||
|
||||
assert!(body.is_none(), "a later corrupt shard must discard the already-appended body prefix");
|
||||
assert!(body.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -9153,71 +9042,6 @@ mod tests {
|
||||
assert_eq!(body.as_ref(), payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_memory_versioned_bucket_uses_inline_data_shards_for_latest() {
|
||||
let tempdir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let endpoint =
|
||||
Endpoint::try_from(tempdir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("disk should be created");
|
||||
|
||||
let payload = vec![b'v'; 64 * 1024];
|
||||
let payload_size = i64::try_from(payload.len()).expect("test payload size should fit i64");
|
||||
let (erasure, files, _read_length, _checksum_algo) = inline_bitrot_files_for_payload(&payload).await;
|
||||
let mut fi = FileInfo::new("bucket/object", erasure.data_shards, erasure.parity_shards);
|
||||
fi.size = payload_size;
|
||||
fi.data = files[0].data.clone();
|
||||
fi.add_object_part(1, String::new(), payload.len(), None, payload_size, None, None);
|
||||
|
||||
let mut object_info = ObjectInfo {
|
||||
size: payload_size,
|
||||
actual_size: payload_size,
|
||||
parts: Arc::new(vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: payload.len(),
|
||||
actual_size: payload_size,
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
object_info.inlined = true;
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
let metrics_size_bucket = rustfs_io_metrics::get_object_size_bucket(fi.size);
|
||||
|
||||
assert_eq!(
|
||||
get_small_object_direct_memory_decision_with_threshold(&None, &object_info, &fi, &opts, true, 128 * 1024),
|
||||
GetDirectMemoryDecision::Use {
|
||||
object_size: payload.len()
|
||||
}
|
||||
);
|
||||
|
||||
let body = SetDisks::try_get_object_direct_data_shards_with_fileinfo(
|
||||
"bucket",
|
||||
"object",
|
||||
&fi,
|
||||
&files,
|
||||
&vec![Some(disk); erasure.total_shard_count()],
|
||||
true,
|
||||
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART,
|
||||
metrics_size_bucket,
|
||||
)
|
||||
.await
|
||||
.expect("versioned latest direct-memory read should not fail")
|
||||
.expect("versioned latest should use inline data shards");
|
||||
|
||||
assert_eq!(body.as_ref(), payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_memory_data_shards_direct_read_reassembles_single_block_payload() {
|
||||
use uuid::Uuid;
|
||||
@@ -11305,13 +11129,4 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adaptive_duplex_buffer_size_raises_mid_sized_gets_without_penalizing_tiny_objects() {
|
||||
assert_eq!(adaptive_duplex_buffer_size(64 * 1024), 64 * 1024);
|
||||
assert_eq!(adaptive_duplex_buffer_size(128 * 1024), 64 * 1024);
|
||||
assert_eq!(adaptive_duplex_buffer_size(256 * 1024), 256 * 1024);
|
||||
assert_eq!(adaptive_duplex_buffer_size(1024 * 1024), 512 * 1024);
|
||||
assert_eq!(adaptive_duplex_buffer_size(2 * 1024 * 1024), 1024 * 1024);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -362,9 +362,9 @@ impl SetDisks {
|
||||
healing: true,
|
||||
};
|
||||
let checks = target_disks.into_iter().map(|disk| {
|
||||
let task_read_options = read_options;
|
||||
let read_options = read_options.clone();
|
||||
async move {
|
||||
let file_info = match disk.read_version("", bucket, object, version_id, &task_read_options).await {
|
||||
let file_info = match disk.read_version("", bucket, object, version_id, &read_options).await {
|
||||
Ok(file_info) => file_info,
|
||||
Err(
|
||||
DiskError::DiskNotFound
|
||||
|
||||
@@ -39,9 +39,16 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
|
||||
// Calculate quorum based on lockers count (majority)
|
||||
let lockers_count = self.lockers.len();
|
||||
let write_quorum = if lockers_count > 1 { (lockers_count / 2) + 1 } else { 1 };
|
||||
NamespaceLock::with_clients_and_quorum_shared(self.set_lock_namespace.clone(), self.lockers.clone(), write_quorum)
|
||||
NamespaceLock::with_clients_and_quorum(
|
||||
format!("set-{}-{}", self.pool_index, self.set_index),
|
||||
self.lockers.clone(),
|
||||
write_quorum,
|
||||
)
|
||||
} else {
|
||||
NamespaceLock::with_local_manager_shared(self.set_lock_namespace.clone(), self.local_lock_manager.clone())
|
||||
NamespaceLock::Local(LocalLock::new(
|
||||
format!("set-{}-{}", self.pool_index, self.set_index),
|
||||
self.local_lock_manager.clone(),
|
||||
))
|
||||
};
|
||||
|
||||
let resource = ObjectKey {
|
||||
|
||||
@@ -162,22 +162,6 @@ fn map_upload_id_metadata_error(bucket: &str, object: &str, upload_id: &str, err
|
||||
err.into()
|
||||
}
|
||||
|
||||
/// Abort a multipart commit when the guard's refresh heartbeat has observed a
|
||||
/// refresh-quorum loss (backlog#899 Phase 2): a stale holder must not race a
|
||||
/// concurrent committer past its fenced commit point.
|
||||
fn fence_commit_on_lock_loss(guard: Option<&ObjectLockDiagGuard>, mode: &'static str, lock_path: &str) -> Result<()> {
|
||||
if guard.is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode,
|
||||
bucket: RUSTFS_META_MULTIPART_BUCKET.to_string(),
|
||||
object: lock_path.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn multipart_bucket_incarnation_id(metadata: &HashMap<String, String>) -> Result<Option<Uuid>> {
|
||||
let Some(value) = rustfs_utils::http::metadata_compat::get_consistent_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) else {
|
||||
if rustfs_utils::http::metadata_compat::contains_key_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) {
|
||||
@@ -970,17 +954,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
let write_path = classify_multipart_part_write_path(multipart_part_size, fi.erasure.block_size);
|
||||
rustfs_io_metrics::record_put_object_path(write_path.multipart_metric_label());
|
||||
let small_size_hint = if matches!(write_path, SmallWritePath::SingleBlockNonInline) {
|
||||
usize::try_from(multipart_part_size).map_err(Error::other)?
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let encode_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
|
||||
let (reader, w_size) = match write_path {
|
||||
SmallWritePath::SingleBlockNonInline => {
|
||||
Arc::clone(&erasure)
|
||||
.encode_single_block_non_inline_with_size_hint(stream, &mut writers, write_quorum, small_size_hint)
|
||||
.encode_single_block_non_inline(stream, &mut writers, write_quorum)
|
||||
.await?
|
||||
}
|
||||
SmallWritePath::PipelineBatchedLarge => {
|
||||
@@ -1087,38 +1066,29 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
|
||||
let part_path = format!("{}/{}/{}", upload_id_path, fi.data_dir.unwrap_or_default(), part_suffix);
|
||||
let part_lock_path = format!("{upload_id_path}/{part_suffix}");
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockAcquire).await;
|
||||
// Serialize only same-part commits (rename_part), not the whole upload.
|
||||
// Each concurrent stream writes to its own unique temp dir (see
|
||||
// `tmp_part` above), so the encode/stream phase never conflicts and must
|
||||
// stay lock-free — holding a lock across it would serialize slow
|
||||
// re-transmits of the same part and defeat the S3 "last finisher wins"
|
||||
// semantics. The mixed-generation hazard is confined to rename_part,
|
||||
// where two temp parts are moved cross-disk onto the SAME final
|
||||
// part_path: interleaving there can leave shards from two generations,
|
||||
// each individually bitrot-valid, that only surface as silent corruption
|
||||
// at read time (backlog#853). A write lock scoped to this part number
|
||||
// makes each same-part commit atomic across disks, so the last committer
|
||||
// wins consistently, while different part numbers commit onto disjoint
|
||||
// part paths and stay concurrent (issue#5961 — an uploadId-wide write
|
||||
// lock serialized them into 503 lock-acquire timeouts). The shared
|
||||
// uploadId read lock keeps completion/abort (which take the uploadId
|
||||
// write lock) from racing any in-flight part commit; a guarded
|
||||
// completion takes the object lock before the upload lock to preserve
|
||||
// global ordering.
|
||||
let (_upload_commit_guard, _part_commit_guard) = if opts.no_lock {
|
||||
(None, None)
|
||||
// Serialize only the commit (rename_part), not the whole upload. Each
|
||||
// concurrent stream writes to its own unique temp dir (see `tmp_part`
|
||||
// above), so the encode/stream phase never conflicts and must stay
|
||||
// lock-free — holding a lock across it would serialize slow re-transmits
|
||||
// of the same part and defeat the S3 "last finisher wins" semantics
|
||||
// (it also caused UploadPart lock-acquire timeouts). The mixed-generation
|
||||
// hazard is confined to rename_part, where two temp parts are moved
|
||||
// cross-disk onto the SAME final part_path: interleaving there can leave
|
||||
// shards from two generations, each individually bitrot-valid, that only
|
||||
// surface as silent corruption at read time (backlog#853). A write lock
|
||||
// scoped to the uploadId namespace makes each commit atomic across disks,
|
||||
// so the last committer wins consistently. A guarded completion takes
|
||||
// the object lock before this upload lock to preserve global ordering.
|
||||
let _upload_commit_guard = if opts.no_lock {
|
||||
None
|
||||
} else {
|
||||
let upload_guard = self
|
||||
.acquire_read_lock_diag("put_object_part_commit", RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
|
||||
.await?;
|
||||
let part_guard = self
|
||||
.acquire_write_lock_diag("put_object_part_commit", RUSTFS_META_MULTIPART_BUCKET, &part_lock_path)
|
||||
.await?;
|
||||
(Some(upload_guard), Some(part_guard))
|
||||
Some(
|
||||
self.acquire_write_lock_diag("put_object_part_commit", RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
|
||||
.await?,
|
||||
)
|
||||
};
|
||||
let (commit_fi, _) = self.check_upload_id_exists(bucket, object, upload_id, false).await?;
|
||||
ensure_multipart_bucket_incarnation(
|
||||
@@ -1132,8 +1102,15 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.await?;
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost).await;
|
||||
fence_commit_on_lock_loss(_upload_commit_guard.as_ref(), "put_object_part_commit", &upload_id_path)?;
|
||||
fence_commit_on_lock_loss(_part_commit_guard.as_ref(), "put_object_part_commit", &part_lock_path)?;
|
||||
if _upload_commit_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_commit",
|
||||
bucket: RUSTFS_META_MULTIPART_BUCKET.to_string(),
|
||||
object: upload_id_path.clone(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
|
||||
let _ = self
|
||||
@@ -1157,7 +1134,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterRename).await;
|
||||
drop(_part_commit_guard);
|
||||
drop(_upload_commit_guard);
|
||||
|
||||
let ret: PartInfo = PartInfo {
|
||||
@@ -1884,12 +1860,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
|
||||
object_size += ext_part.size;
|
||||
if opts.quota_admission.is_some() && ext_part.actual_size < 0 {
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
object_actual_size = object_actual_size
|
||||
.checked_add(ext_part.actual_size)
|
||||
.ok_or(Error::PartMissingOrCorrupt)?;
|
||||
object_actual_size += ext_part.actual_size;
|
||||
|
||||
fi.parts.push(completed_multipart_object_part(p.part_num, ext_part));
|
||||
}
|
||||
@@ -1918,15 +1889,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(admission) = opts.quota_admission {
|
||||
let quota_operation_size = u64::try_from(object_actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
if quota_operation_size > admission.remaining() {
|
||||
return Err(Error::QuotaExceeded {
|
||||
current: admission.current_usage(),
|
||||
limit: admission.quota_limit(),
|
||||
});
|
||||
}
|
||||
}
|
||||
if let Some(rc_crc) = get_header_map(&opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC) {
|
||||
if let Ok(rc_crc_bytes) = base64_simd::STANDARD.decode_to_vec(&rc_crc) {
|
||||
fi.checksum = Some(Bytes::from(rc_crc_bytes));
|
||||
@@ -2589,157 +2551,29 @@ mod tests {
|
||||
.new_multipart_upload(bucket, object, create_opts)
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let part = put_test_part(set_disks, bucket, object, &upload.upload_id, 1, content, content.len() as i64).await;
|
||||
(upload.upload_id, vec![part])
|
||||
}
|
||||
|
||||
async fn put_test_part(
|
||||
set_disks: &Arc<SetDisks>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
part_number: usize,
|
||||
content: &[u8],
|
||||
actual_size: i64,
|
||||
) -> CompletePart {
|
||||
let mut reader = PutObjReader::new(
|
||||
HashReader::from_stream(Cursor::new(content.to_vec()), content.len() as i64, actual_size, None, None, false)
|
||||
.expect("hash reader should be constructed"),
|
||||
HashReader::from_stream(
|
||||
Cursor::new(content.to_vec()),
|
||||
content.len() as i64,
|
||||
content.len() as i64,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.expect("hash reader should be constructed"),
|
||||
);
|
||||
let part = set_disks
|
||||
.put_object_part(bucket, object, upload_id, part_number, &mut reader, &ObjectOptions::default())
|
||||
.put_object_part(bucket, object, &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("uploading the part should succeed");
|
||||
CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_multipart_quota_rejection_preserves_destination_and_upload() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-quota-admission-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
let existing_payload = b"existing object";
|
||||
let mut existing_reader = PutObjReader::from_vec(existing_payload.to_vec());
|
||||
let existing = set_disks
|
||||
.put_object(bucket, object, &mut existing_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("existing object should be stored");
|
||||
|
||||
let payload = vec![0x51; 4096];
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &payload, &ObjectOptions::default()).await;
|
||||
let mut denied_opts = ObjectOptions::default();
|
||||
assert!(denied_opts.set_quota_admission(100, 4195));
|
||||
|
||||
let err = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts.clone(), &denied_opts)
|
||||
.await
|
||||
.expect_err("completion larger than the remaining quota must be rejected");
|
||||
assert!(matches!(
|
||||
err,
|
||||
StorageError::QuotaExceeded {
|
||||
current: 100,
|
||||
limit: 4195
|
||||
}
|
||||
));
|
||||
|
||||
let current = set_disks
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("quota rejection must preserve the existing destination");
|
||||
assert_eq!(current.etag, existing.etag);
|
||||
assert!(
|
||||
set_disks
|
||||
.check_upload_id_exists(bucket, object, &upload_id, false)
|
||||
.await
|
||||
.is_ok(),
|
||||
"quota rejection must leave the multipart upload retryable"
|
||||
);
|
||||
|
||||
let mut allowed_opts = ObjectOptions::default();
|
||||
assert!(allowed_opts.set_quota_admission(100, 4196));
|
||||
let completed = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &allowed_opts)
|
||||
.await
|
||||
.expect("completion at the exact remaining-quota boundary should succeed");
|
||||
assert_eq!(completed.get_actual_size().expect("completed logical size should resolve"), 4096);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_multipart_quota_uses_compressed_logical_size() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-compressed-quota-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
let mut create_opts = ObjectOptions::default();
|
||||
insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string());
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &create_opts)
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let part = put_test_part(&set_disks, bucket, object, &upload.upload_id, 1, &[0x52; 128], 8192).await;
|
||||
let mut complete_opts = ObjectOptions::default();
|
||||
assert!(complete_opts.set_quota_admission(0, 4096));
|
||||
|
||||
let err = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload.upload_id, vec![part], &complete_opts)
|
||||
.await
|
||||
.expect_err("logical size above the remaining quota must be rejected");
|
||||
assert!(matches!(err, StorageError::QuotaExceeded { current: 0, limit: 4096 }));
|
||||
assert!(
|
||||
set_disks
|
||||
.check_upload_id_exists(bucket, object, &upload.upload_id, false)
|
||||
.await
|
||||
.is_ok(),
|
||||
"quota rejection must leave compressed parts retryable"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complete_multipart_quota_rejects_invalid_logical_sizes() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-invalid-logical-size-bucket";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
let mut create_opts = ObjectOptions::default();
|
||||
insert_str(&mut create_opts.user_defined, SUFFIX_COMPRESSION, "S2".to_string());
|
||||
let mut complete_opts = ObjectOptions::default();
|
||||
assert!(complete_opts.set_quota_admission(0, u64::MAX));
|
||||
|
||||
let negative_upload = set_disks
|
||||
.new_multipart_upload(bucket, "negative", &create_opts)
|
||||
.await
|
||||
.expect("negative-size upload should be created");
|
||||
let negative_part = put_test_part(&set_disks, bucket, "negative", &negative_upload.upload_id, 1, &[0x53], -1).await;
|
||||
let negative_err = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, "negative", &negative_upload.upload_id, vec![negative_part], &complete_opts)
|
||||
.await
|
||||
.expect_err("negative logical size must fail closed");
|
||||
assert!(matches!(negative_err, StorageError::PartMissingOrCorrupt));
|
||||
|
||||
let overflow_upload = set_disks
|
||||
.new_multipart_upload(bucket, "overflow", &create_opts)
|
||||
.await
|
||||
.expect("overflow upload should be created");
|
||||
let first = put_test_part(&set_disks, bucket, "overflow", &overflow_upload.upload_id, 1, &[0x54], i64::MAX).await;
|
||||
let second = put_test_part(&set_disks, bucket, "overflow", &overflow_upload.upload_id, 2, &[0x55], 1).await;
|
||||
let overflow_err = set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, "overflow", &overflow_upload.upload_id, vec![first, second], &complete_opts)
|
||||
.await
|
||||
.expect_err("overflowing logical size must fail closed");
|
||||
assert!(matches!(overflow_err, StorageError::PartMissingOrCorrupt));
|
||||
(
|
||||
upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
)
|
||||
}
|
||||
|
||||
async fn assert_complete_first_linearizes(bucket: &'static str, object: &'static str, create_opts: ObjectOptions) {
|
||||
@@ -3532,268 +3366,6 @@ mod tests {
|
||||
.expect("abort should delete the upload after UploadPart releases the lock");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn put_object_part_different_part_numbers_commit_concurrently() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
const PART1_SIZE: usize = 5 * 1024 * 1024; // non-final parts must be >= 5MiB to complete
|
||||
const PART2_SIZE: usize = 4096;
|
||||
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let locker: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager));
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, vec![locker]).await;
|
||||
let bucket = "multipart-concurrent-part-numbers-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let upload_id = upload.upload_id;
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
// issue#5961: the barrier releases only once BOTH commits are paused
|
||||
// inside their commit sections, so reaching wait_until_paused proves the
|
||||
// two part numbers held their commit locks concurrently. Under an
|
||||
// uploadId-wide exclusive commit lock the second put errors at the 5s
|
||||
// lock-acquire timeout instead of arriving, and wait_until_paused fails
|
||||
// deterministically. No wall-clock bound on the success path.
|
||||
let barrier = MultipartCommitBarrier::install_for_arrivals(bucket, object, MultipartCommitPause::PutPartAfterRename, 2);
|
||||
|
||||
let put1_store = set_disks.clone();
|
||||
let put1_upload_id = upload_id.clone();
|
||||
let put1 = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x51; PART1_SIZE]);
|
||||
put1_store
|
||||
.put_object_part(bucket, object, &put1_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
let put2_store = set_disks.clone();
|
||||
let put2_upload_id = upload_id.clone();
|
||||
let put2 = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x52; PART2_SIZE]);
|
||||
put2_store
|
||||
.put_object_part(bucket, object, &put2_upload_id, 2, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
barrier.release();
|
||||
let part1 = put1
|
||||
.await
|
||||
.expect("part 1 task should not panic")
|
||||
.expect("part 1 should commit after the barrier is released");
|
||||
let part2 = put2
|
||||
.await
|
||||
.expect("part 2 task should not panic")
|
||||
.expect("part 2 should commit after the barrier is released");
|
||||
assert_eq!(part1.part_num, 1);
|
||||
assert_eq!(part2.part_num, 2);
|
||||
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
vec![
|
||||
CompletePart {
|
||||
part_num: part1.part_num,
|
||||
etag: part1.etag.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
CompletePart {
|
||||
part_num: part2.part_num,
|
||||
etag: part2.etag.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("completion should succeed with both concurrently committed parts");
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed object should open");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("completed object should stream fully");
|
||||
assert_eq!(body.len(), PART1_SIZE + PART2_SIZE);
|
||||
assert!(body[..PART1_SIZE].iter().all(|b| *b == 0x51), "part 1 bytes must round-trip");
|
||||
assert!(body[PART1_SIZE..].iter().all(|b| *b == 0x52), "part 2 bytes must round-trip");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn put_object_part_same_part_retries_serialize_on_part_lock() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-same-part-retry-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let part_lock_path = format!("{upload_id_path}/part.1");
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartAfterRename);
|
||||
|
||||
let first_store = set_disks.clone();
|
||||
let first_upload_id = upload_id.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x53; 4096]);
|
||||
first_store
|
||||
.put_object_part(bucket, object, &first_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
|
||||
// The paused commit must hold its part lock EXCLUSIVELY: even a shared
|
||||
// probe on the part key has to time out. This pins the write-ness of the
|
||||
// part lock — a shared part lock would let two same-part rename_part
|
||||
// calls interleave into mixed-generation shards (backlog#853).
|
||||
let probe = set_disks
|
||||
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &part_lock_path)
|
||||
.await
|
||||
.expect("part namespace lock should be created")
|
||||
.get_read_lock(Duration::from_secs(1))
|
||||
.await;
|
||||
assert!(
|
||||
probe.is_err(),
|
||||
"the in-flight part commit must hold an exclusive write lock on its part key"
|
||||
);
|
||||
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, part_lock_path));
|
||||
let retry_store = set_disks.clone();
|
||||
let retry_upload_id = upload_id.clone();
|
||||
let retry = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x54; 4096]);
|
||||
retry_store
|
||||
.put_object_part(bucket, object, &retry_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(1).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(
|
||||
!retry.is_finished(),
|
||||
"a retry of the same part number must wait for the in-flight commit (backlog#853)"
|
||||
);
|
||||
|
||||
barrier.release();
|
||||
first
|
||||
.await
|
||||
.expect("first attempt task should not panic")
|
||||
.expect("first attempt should commit after the barrier is released");
|
||||
let retry_part = retry
|
||||
.await
|
||||
.expect("retry task should not panic")
|
||||
.expect("the retry should commit after the first attempt releases the part lock");
|
||||
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: retry_part.part_num,
|
||||
etag: retry_part.etag.clone(),
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("the last committed retry must win the final part generation");
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("completed object should open");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("completed object should stream fully");
|
||||
assert_eq!(body, vec![0x54; 4096], "the retry's generation must be the one served");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn put_object_part_fences_part_lock_loss_before_rename() {
|
||||
let target = Arc::new(std::sync::RwLock::new(None));
|
||||
let refresh_calls = Arc::new(AtomicUsize::new(0));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = (0..4)
|
||||
.map(|_| {
|
||||
Arc::new(SelectiveLockLossClient::new(Arc::clone(&target), Arc::clone(&refresh_calls))) as Arc<dyn LockClient>
|
||||
})
|
||||
.collect();
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-put-part-part-lock-loss-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
let part_lock_path = format!("{upload_id_path}/part.1");
|
||||
*target.write().expect("lock-loss target should be writable") =
|
||||
Some(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, part_lock_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
let put_upload_id = upload_id.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x47; 4096]);
|
||||
put_store
|
||||
.put_object_part(bucket, object, &put_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
tokio::time::advance(Duration::from_secs(11)).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert!(
|
||||
refresh_calls.load(Ordering::Acquire) > 0,
|
||||
"part lock heartbeat should reach the test client"
|
||||
);
|
||||
barrier.release();
|
||||
|
||||
let err = put
|
||||
.await
|
||||
.expect("UploadPart task should not panic")
|
||||
.expect_err("UploadPart must fail after losing the part lock");
|
||||
match err {
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
bucket: lock_bucket,
|
||||
object: lock_object,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(lock_bucket, RUSTFS_META_MULTIPART_BUCKET);
|
||||
assert_eq!(lock_object, part_lock_path);
|
||||
}
|
||||
other => panic!("unexpected lock-loss error: {other:?}"),
|
||||
}
|
||||
let listed = set_disks
|
||||
.list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("part lock loss before rename must leave the upload readable");
|
||||
assert!(listed.parts.is_empty(), "part lock loss before rename must not publish the part");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn put_object_part_fences_upload_lock_loss_before_rename() {
|
||||
|
||||
@@ -56,22 +56,6 @@ use http::HeaderValue;
|
||||
use rustfs_utils::path::decode_dir_object;
|
||||
use std::future::Future;
|
||||
|
||||
#[inline]
|
||||
fn duration_millis_f64(duration: std::time::Duration) -> f64 {
|
||||
duration.as_secs_f64() * 1000.0
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod duration_metrics_tests {
|
||||
use super::duration_millis_f64;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
fn duration_millis_preserves_sub_millisecond_precision() {
|
||||
assert_eq!(duration_millis_f64(Duration::from_micros(125)), 0.125);
|
||||
}
|
||||
}
|
||||
|
||||
fn is_restore_control_metadata(key: &str) -> bool {
|
||||
key.eq_ignore_ascii_case(X_AMZ_RESTORE.as_str())
|
||||
|| key.eq_ignore_ascii_case(rustfs_utils::http::headers::AMZ_RESTORE_EXPIRY_DAYS)
|
||||
@@ -750,8 +734,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
0,
|
||||
object_info.size,
|
||||
&mut output,
|
||||
fi.into_owned(),
|
||||
files.into_owned(),
|
||||
fi,
|
||||
files,
|
||||
&disks,
|
||||
self.set_index,
|
||||
self.pool_index,
|
||||
@@ -867,8 +851,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
offset,
|
||||
length,
|
||||
&mut writer,
|
||||
fi.into_owned(),
|
||||
files.into_owned(),
|
||||
fi,
|
||||
files,
|
||||
&disks,
|
||||
set_index,
|
||||
pool_index,
|
||||
@@ -1123,12 +1107,8 @@ impl SetDisks {
|
||||
writers.push(w);
|
||||
errors.push(e);
|
||||
}
|
||||
let writer_setup_elapsed = writer_setup_stage_start.elapsed();
|
||||
let writer_setup_ms = writer_setup_elapsed.as_millis() as u64;
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
"set_disk_writer_setup",
|
||||
duration_millis_f64(writer_setup_elapsed),
|
||||
);
|
||||
let writer_setup_ms = writer_setup_stage_start.elapsed().as_millis() as u64;
|
||||
rustfs_io_metrics::record_put_object_stage_duration("set_disk_writer_setup", writer_setup_ms as f64);
|
||||
|
||||
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
|
||||
if nil_count < write_quorum {
|
||||
@@ -1158,16 +1138,11 @@ impl SetDisks {
|
||||
|
||||
let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size);
|
||||
rustfs_io_metrics::record_put_object_path(write_path.metric_label());
|
||||
let small_size_hint = if matches!(write_path, SmallWritePath::Inline | SmallWritePath::SingleBlockNonInline) {
|
||||
usize::try_from(put_object_size).map_err(Error::other)?
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let encode_stage_start = Instant::now();
|
||||
let (reader, w_size) = match write_path {
|
||||
SmallWritePath::Inline => match Arc::clone(&erasure)
|
||||
.encode_inline_small_with_size_hint(stream, &mut writers, write_quorum, small_size_hint)
|
||||
.encode_inline_small(stream, &mut writers, write_quorum)
|
||||
.await
|
||||
{
|
||||
Ok((r, w)) => (r, w),
|
||||
@@ -1177,7 +1152,7 @@ impl SetDisks {
|
||||
}
|
||||
},
|
||||
SmallWritePath::SingleBlockNonInline => match Arc::clone(&erasure)
|
||||
.encode_single_block_non_inline_with_size_hint(stream, &mut writers, write_quorum, small_size_hint)
|
||||
.encode_single_block_non_inline(stream, &mut writers, write_quorum)
|
||||
.await
|
||||
{
|
||||
Ok((r, w)) => (r, w),
|
||||
@@ -1203,9 +1178,8 @@ impl SetDisks {
|
||||
}
|
||||
},
|
||||
};
|
||||
let encode_elapsed = encode_stage_start.elapsed();
|
||||
let encode_ms = encode_elapsed.as_millis() as u64;
|
||||
rustfs_io_metrics::record_put_object_stage_duration("set_disk_encode", duration_millis_f64(encode_elapsed));
|
||||
let encode_ms = encode_stage_start.elapsed().as_millis() as u64;
|
||||
rustfs_io_metrics::record_put_object_stage_duration("set_disk_encode", encode_ms as f64);
|
||||
|
||||
let _ = mem::replace(&mut data.stream, reader);
|
||||
// if let Err(err) = close_bitrot_writers(&mut writers).await {
|
||||
@@ -1523,18 +1497,8 @@ impl SetDisks {
|
||||
let _ = rustfs_common::heal_channel::send_heal_request(request).await;
|
||||
});
|
||||
}
|
||||
|
||||
let rename_stage_elapsed = rename_stage_start.elapsed();
|
||||
let rename_stage_ms = rename_stage_elapsed.as_millis() as u64;
|
||||
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
|
||||
// `rename_data` has completed the authoritative quorum commit. The
|
||||
// exact old-data-dir reclamation below is best-effort space cleanup;
|
||||
// it must not serialize the next operation on this object.
|
||||
drop(object_lock_guard);
|
||||
|
||||
rustfs_io_metrics::record_put_object_stage_duration("set_disk_rename", duration_millis_f64(rename_stage_elapsed));
|
||||
let rename_stage_ms = rename_stage_start.elapsed().as_millis() as u64;
|
||||
rustfs_io_metrics::record_put_object_stage_duration("set_disk_rename", rename_stage_ms as f64);
|
||||
if (rename_stage_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS {
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_COMMIT_TAIL_SLOW,
|
||||
@@ -1563,13 +1527,9 @@ impl SetDisks {
|
||||
let cleanup = self
|
||||
.commit_rename_data_dir(&cleanup_disks, bucket, object, &old_dir.to_string(), &committed_dir, write_quorum)
|
||||
.await;
|
||||
let cleanup_elapsed = cleanup_stage_start.elapsed();
|
||||
let cleanup_ms = cleanup_elapsed.as_millis() as u64;
|
||||
let cleanup_ms = cleanup_stage_start.elapsed().as_millis() as u64;
|
||||
cleanup_stage_ms = Some(cleanup_ms);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
"set_disk_old_data_cleanup",
|
||||
duration_millis_f64(cleanup_elapsed),
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration("set_disk_old_data_cleanup", cleanup_ms as f64);
|
||||
self.report_old_data_dir_cleanup(bucket, object, &old_dir.to_string(), &cleanup)
|
||||
.await;
|
||||
if (cleanup_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS {
|
||||
@@ -1590,6 +1550,8 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
drop(object_lock_guard); // drop object lock guard to release the lock
|
||||
|
||||
for (i, op_disk) in online_disks.iter().enumerate() {
|
||||
if let Some(disk) = op_disk
|
||||
&& disk.is_online().await
|
||||
@@ -1688,6 +1650,10 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
if result.is_ok() {
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
}
|
||||
|
||||
if issue3031_diag_enabled() {
|
||||
warn!(
|
||||
target: "rustfs_ecstore::set_disk",
|
||||
@@ -3201,8 +3167,7 @@ impl SetDisks {
|
||||
// Force the full quorum fanout (allow_early_stop=false): `disks` is the
|
||||
// write target below, and an early-stop subset would only carry read
|
||||
// quorum, failing write quorum on update_object_meta (backlog#872).
|
||||
let (fi, _, disks) = self.get_object_fileinfo_gated(bucket, object, opts, false, false).await?;
|
||||
let mut fi = fi.into_owned();
|
||||
let (mut fi, _, disks) = self.get_object_fileinfo_gated(bucket, object, opts, false, false).await?;
|
||||
|
||||
fi.metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_owned());
|
||||
if let Some(eval_metadata) = &opts.eval_metadata {
|
||||
@@ -3225,7 +3190,7 @@ impl SetDisks {
|
||||
});
|
||||
}
|
||||
|
||||
self.update_object_meta(bucket, object, fi.clone(), &disks).await?;
|
||||
self.update_object_meta(bucket, object, fi.clone(), disks.as_slice()).await?;
|
||||
|
||||
Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended))
|
||||
}
|
||||
@@ -4629,8 +4594,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
// _lock_guard = guard_opt;
|
||||
// }
|
||||
|
||||
let (fi, meta_arr, online_disks) = self.get_object_fileinfo(bucket, object, opts, true, false).await?;
|
||||
let mut fi = fi.into_owned();
|
||||
let (mut fi, meta_arr, online_disks) = self.get_object_fileinfo(bucket, object, opts, true, false).await?;
|
||||
/*if err != nil {
|
||||
return Err(to_object_err(err, vec![bucket, object]));
|
||||
}*/
|
||||
@@ -4744,7 +4708,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
cloned_fi.size,
|
||||
&mut writer,
|
||||
cloned_fi,
|
||||
meta_arr.into_owned(),
|
||||
meta_arr,
|
||||
&online_disks,
|
||||
set_index,
|
||||
pool_index,
|
||||
@@ -4868,7 +4832,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
};
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
let current = self.get_object_fileinfo(bucket, object, &commit_opts, true, false).await;
|
||||
let (current_fi, _, _) = match current {
|
||||
let (mut current_fi, _, _) = match current {
|
||||
Ok(current) => current,
|
||||
Err(err) => {
|
||||
drop(transition_lock_guard);
|
||||
@@ -4879,7 +4843,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let mut current_fi = current_fi.into_owned();
|
||||
let source_matches = current_fi.version_id == fi.version_id
|
||||
&& current_fi.data_dir == fi.data_dir
|
||||
&& current_fi.mod_time == fi.mod_time
|
||||
@@ -5590,7 +5553,7 @@ mod get_object_downstream_close_accounting_tests {
|
||||
let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
|
||||
let (decode_failures, emit_failures, legacy_fanout, internal_fanout) = metrics::with_local_recorder(&recorder, || {
|
||||
let (decode_failures, emit_failures) = metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "get-downstream-close-accounting";
|
||||
@@ -5658,14 +5621,6 @@ mod get_object_downstream_close_accounting_tests {
|
||||
("reason", GetObjectFailureReason::DownstreamClosed.as_str()),
|
||||
],
|
||||
),
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_total_responses",
|
||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)],
|
||||
),
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_total_responses",
|
||||
&[("path", GET_OBJECT_PATH_INTERNAL_META)],
|
||||
),
|
||||
)
|
||||
})
|
||||
});
|
||||
@@ -5673,11 +5628,6 @@ mod get_object_downstream_close_accounting_tests {
|
||||
|
||||
assert!(decode_failures > 0, "the producer must expose the downstream close at decode");
|
||||
assert_eq!(emit_failures, 0, "downstream closure must not be counted as an emit failure");
|
||||
assert_eq!(legacy_fanout, vec![4.0], "ordinary object fanout must retain the legacy_duplex path");
|
||||
assert!(
|
||||
internal_fanout.is_empty(),
|
||||
"ordinary object fanout must not be attributed to internal_meta"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -5691,7 +5641,7 @@ mod get_object_downstream_close_accounting_tests {
|
||||
let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
|
||||
let (internal_missing, legacy_unknown, internal_fanout, legacy_fanout) = metrics::with_local_recorder(&recorder, || {
|
||||
let (internal_missing, legacy_unknown) = metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let options = ObjectOptions {
|
||||
@@ -5727,14 +5677,6 @@ mod get_object_downstream_close_accounting_tests {
|
||||
("reason", GetObjectFailureReason::Unknown.as_str()),
|
||||
],
|
||||
),
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_error_responses",
|
||||
&[("path", GET_OBJECT_PATH_INTERNAL_META)],
|
||||
),
|
||||
recorder.histogram_values(
|
||||
"rustfs_io_get_object_metadata_fanout_error_responses",
|
||||
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)],
|
||||
),
|
||||
)
|
||||
})
|
||||
});
|
||||
@@ -5745,8 +5687,6 @@ mod get_object_downstream_close_accounting_tests {
|
||||
legacy_unknown, 0,
|
||||
"internal metadata miss must not be attributed to legacy_duplex/unknown"
|
||||
);
|
||||
assert_eq!(internal_fanout, vec![4.0], "internal metadata fanout must retain its path label");
|
||||
assert!(legacy_fanout.is_empty(), "internal metadata fanout must not leak into legacy_duplex");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6238,9 +6178,9 @@ mod transition_commit_failure_tests {
|
||||
cache_key.clone(),
|
||||
Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi: Arc::new((*fi).clone()),
|
||||
parts_metadata: Arc::new(parts_metadata.into_owned()),
|
||||
online_disks: Arc::new(online_disks.into_owned()),
|
||||
fi: fi.clone(),
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
read_quorum: 2,
|
||||
}),
|
||||
)
|
||||
@@ -8671,10 +8611,8 @@ mod put_object_tmp_cleanup_tests {
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
||||
use super::*;
|
||||
use crate::disk::DiskAPI as _;
|
||||
use crate::set_disk::core::io_primitives::rename_fanout_barrier;
|
||||
use std::time::Duration;
|
||||
use tempfile::TempDir;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
/// Large enough that the erasure shards are written as real tmp files
|
||||
/// (never inlined into xl.meta), so both tests exercise actual cleanup.
|
||||
@@ -8759,168 +8697,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_put_releases_namespace_lock_before_old_data_cleanup() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-commit-lock-window";
|
||||
let object = "commit-lock-window-object";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let mut initial_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]);
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut initial_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initial object should be committed");
|
||||
let mut initial = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initial object should populate the metadata cache");
|
||||
let mut initial_body = Vec::new();
|
||||
initial
|
||||
.stream
|
||||
.read_to_end(&mut initial_body)
|
||||
.await
|
||||
.expect("initial body should drain");
|
||||
assert_eq!(initial_body, vec![b'0'; TEST_OBJECT_SIZE]);
|
||||
|
||||
let cleanup_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_CLEANUP);
|
||||
let first_store = Arc::clone(&set_disks);
|
||||
let first = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
first_store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), cleanup_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("first overwrite should reach old-data cleanup");
|
||||
|
||||
let mut committed = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
|
||||
)
|
||||
.await
|
||||
.expect("GET should not wait for old-data cleanup")
|
||||
.expect("committed overwrite should be readable during old-data cleanup");
|
||||
let mut committed_body = Vec::new();
|
||||
committed
|
||||
.stream
|
||||
.read_to_end(&mut committed_body)
|
||||
.await
|
||||
.expect("committed overwrite body should drain");
|
||||
assert_eq!(committed_body, vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
|
||||
let second_commit_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
|
||||
let second_store = Arc::clone(&set_disks);
|
||||
let second = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
second_store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), second_commit_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("second overwrite should acquire the namespace lock during cleanup");
|
||||
|
||||
cleanup_barrier.release();
|
||||
first
|
||||
.await
|
||||
.expect("first overwrite task should join")
|
||||
.expect("first overwrite should remain successful after cleanup");
|
||||
drop(cleanup_barrier);
|
||||
second_commit_barrier.release();
|
||||
second
|
||||
.await
|
||||
.expect("second overwrite task should join")
|
||||
.expect("second overwrite should commit after acquiring the released namespace lock");
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the latest overwrite should be readable");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("latest body should drain");
|
||||
assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_post_commit_cleanup_does_not_retain_namespace_lock() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-commit-lock-cancelled-cleanup";
|
||||
let object = "commit-lock-cancelled-cleanup-object";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let mut initial_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]);
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut initial_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initial object should be committed");
|
||||
|
||||
let cleanup_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let cleanup_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_CLEANUP);
|
||||
let first_store = Arc::clone(&set_disks);
|
||||
let first = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
first_store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), cleanup_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("first overwrite should reach old-data cleanup");
|
||||
|
||||
let second_commit_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
|
||||
let second_store = Arc::clone(&set_disks);
|
||||
let second = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
second_store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), second_commit_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("second overwrite should acquire the namespace lock before cancellation");
|
||||
|
||||
first.abort();
|
||||
assert!(
|
||||
first
|
||||
.await
|
||||
.expect_err("the first request should be cancelled during cleanup")
|
||||
.is_cancelled()
|
||||
);
|
||||
assert!(
|
||||
cleanup_tasks.running() >= 1,
|
||||
"cancelled cleanup must remain observable until its disk task drains"
|
||||
);
|
||||
cleanup_barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while cleanup_tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cancelled cleanup disk tasks should drain");
|
||||
drop(cleanup_barrier);
|
||||
|
||||
second_commit_barrier.release();
|
||||
second
|
||||
.await
|
||||
.expect("second overwrite task should join")
|
||||
.expect("second overwrite should survive the earlier request cancellation");
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the latest overwrite should be readable");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("latest body should drain");
|
||||
assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_object_no_lock_aborts_after_outer_namespace_lock_loss() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
|
||||
@@ -30,12 +30,12 @@ use crate::diagnostics::get::{
|
||||
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
|
||||
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
|
||||
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY,
|
||||
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE,
|
||||
GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
|
||||
GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
|
||||
GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
|
||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, mark_get_object_downstream_closed,
|
||||
record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP,
|
||||
GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, GET_STAGE_READER_SETUP_DROP_PENDING,
|
||||
GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT,
|
||||
GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, GetObjectFailureReason, classify_disk_error,
|
||||
get_stage_timer_if_enabled, mark_get_object_downstream_closed, record_get_object_pipeline_failure,
|
||||
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use crate::io_support::bitrot::{
|
||||
@@ -116,9 +116,9 @@ impl SetDisks {
|
||||
.then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE)
|
||||
}
|
||||
|
||||
async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option<Arc<GetObjectMetadataCacheEntry>> {
|
||||
async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option<GetObjectMetadataCacheEntry> {
|
||||
match self.lookup_cached_get_object_fileinfo(bucket, object).await {
|
||||
MetadataCacheLookup::Hit(entry) => Some(entry),
|
||||
MetadataCacheLookup::Hit(entry) => Some((*entry).clone()),
|
||||
MetadataCacheLookup::Miss | MetadataCacheLookup::RejectedInsufficientQuorum => None,
|
||||
}
|
||||
}
|
||||
@@ -180,9 +180,9 @@ impl SetDisks {
|
||||
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
|
||||
let entry = Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi: Arc::new(fi.clone()),
|
||||
parts_metadata: Arc::new(parts_metadata.to_vec()),
|
||||
online_disks: Arc::new(online_disks.to_vec()),
|
||||
fi: fi.clone(),
|
||||
parts_metadata: parts_metadata.to_vec(),
|
||||
online_disks: online_disks.to_vec(),
|
||||
read_quorum,
|
||||
});
|
||||
self.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {})
|
||||
@@ -221,10 +221,10 @@ impl SetDisks {
|
||||
let disks = self.disks.read().await.clone();
|
||||
let required_reads = self.default_read_quorum();
|
||||
|
||||
let bucket: Arc<str> = Arc::from(bucket);
|
||||
let object: Arc<str> = Arc::from(object);
|
||||
let version_id: Arc<str> = Arc::from(version_id);
|
||||
let opts = *opts;
|
||||
let bucket = bucket.to_string();
|
||||
let object = object.to_string();
|
||||
let version_id = version_id.to_string();
|
||||
let opts = opts.clone();
|
||||
|
||||
let processor = runtime_sources::batch_processors().read_processor();
|
||||
let tasks: Vec<_> = disks
|
||||
@@ -235,9 +235,9 @@ impl SetDisks {
|
||||
let bucket = bucket.clone();
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
let task_opts = opts;
|
||||
let opts = opts.clone();
|
||||
|
||||
async move { disk.read_version(&bucket, &bucket, &object, &version_id, &task_opts).await }
|
||||
async move { disk.read_version(&bucket, &bucket, &object, &version_id, &opts).await }
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
@@ -257,7 +257,7 @@ impl SetDisks {
|
||||
opts: &ObjectOptions,
|
||||
read_data: bool,
|
||||
caller_allows_early_stop: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
) -> Result<(FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>)> {
|
||||
self.get_object_fileinfo_gated(bucket, object, opts, read_data, caller_allows_early_stop)
|
||||
.await
|
||||
}
|
||||
@@ -274,7 +274,7 @@ impl SetDisks {
|
||||
opts: &ObjectOptions,
|
||||
read_data: bool,
|
||||
allow_early_stop: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
) -> Result<(FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>)> {
|
||||
let vid = opts.version_id.clone().unwrap_or_default();
|
||||
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
|
||||
@@ -300,11 +300,7 @@ impl SetDisks {
|
||||
GET_STAGE_METADATA_CACHE_LOOKUP,
|
||||
metadata_cache_lookup_start,
|
||||
);
|
||||
return Ok((
|
||||
GetObjectMetadata::Shared(Arc::clone(&cached.fi)),
|
||||
GetObjectMetadata::Shared(Arc::clone(&cached.parts_metadata)),
|
||||
GetObjectMetadata::Shared(Arc::clone(&cached.online_disks)),
|
||||
));
|
||||
return Ok((cached.fi.clone(), cached.parts_metadata.clone(), cached.online_disks.clone()));
|
||||
}
|
||||
MetadataCacheLookup::Miss => {
|
||||
rustfs_io_metrics::record_get_object_metadata_cache_decision(
|
||||
@@ -353,12 +349,7 @@ impl SetDisks {
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await?;
|
||||
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
GET_OBJECT_PATH_INTERNAL_META
|
||||
} else {
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX
|
||||
};
|
||||
metadata_fanout_diagnostics.record(metadata_metrics_path);
|
||||
metadata_fanout_diagnostics.record(GET_OBJECT_PATH_LEGACY_DUPLEX);
|
||||
let metadata_fanout_complete = metadata_fanout_diagnostics.total_responses() >= disks.len();
|
||||
// warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata);
|
||||
// warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs);
|
||||
@@ -396,7 +387,7 @@ impl SetDisks {
|
||||
|
||||
let (op_online_disks, fi, fileinfo_selection_quorum) =
|
||||
Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum)?;
|
||||
metadata_fanout_diagnostics.record_quorum_candidate_latency(metadata_metrics_path, fileinfo_selection_quorum);
|
||||
metadata_fanout_diagnostics.record_quorum_candidate_latency(GET_OBJECT_PATH_LEGACY_DUPLEX, fileinfo_selection_quorum);
|
||||
if errs.iter().any(|err| err.is_some()) {
|
||||
let version_id = resolved_read_repair_version_id(&fi, opts.version_id.as_deref());
|
||||
submit_read_repair_heal(
|
||||
@@ -427,11 +418,7 @@ impl SetDisks {
|
||||
|
||||
// let online_disks: Vec<Option<DiskStore>> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect();
|
||||
|
||||
Ok((
|
||||
GetObjectMetadata::Owned(fi),
|
||||
GetObjectMetadata::Owned(parts_metadata),
|
||||
GetObjectMetadata::Owned(op_online_disks),
|
||||
))
|
||||
Ok((fi, parts_metadata, op_online_disks))
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
@@ -2687,39 +2674,6 @@ mod metadata_cache_tests {
|
||||
assert_eq!(cached.read_quorum, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_fileinfo_cache_hit_shares_cached_metadata() {
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
let fi = valid_test_fileinfo("object");
|
||||
let parts_metadata = vec![fi.clone()];
|
||||
let online_disks = Vec::new();
|
||||
let generation = set.get_object_metadata_cache_generation("bucket", "object");
|
||||
set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, &parts_metadata, &online_disks, 0)
|
||||
.await;
|
||||
let cached = set
|
||||
.cached_get_object_fileinfo("bucket", "object")
|
||||
.await
|
||||
.expect("fresh cache entry should be returned");
|
||||
|
||||
let (returned_fi, returned_parts_metadata, returned_online_disks) = set
|
||||
.get_object_fileinfo("bucket", "object", &ObjectOptions::default(), true, false)
|
||||
.await
|
||||
.expect("cache-backed metadata lookup should succeed");
|
||||
|
||||
assert!(
|
||||
matches!(returned_fi, GetObjectMetadata::Shared(ref value) if Arc::ptr_eq(value, &cached.fi)),
|
||||
"cache hits must share FileInfo ownership"
|
||||
);
|
||||
assert!(
|
||||
matches!(returned_parts_metadata, GetObjectMetadata::Shared(ref value) if Arc::ptr_eq(value, &cached.parts_metadata)),
|
||||
"cache hits must share the metadata vector"
|
||||
);
|
||||
assert!(
|
||||
matches!(returned_online_disks, GetObjectMetadata::Shared(ref value) if Arc::ptr_eq(value, &cached.online_disks)),
|
||||
"cache hits must share the online-disk vector"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_metadata_cache_rejects_deleted_and_invalid_fileinfo() {
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
@@ -2759,9 +2713,9 @@ mod metadata_cache_tests {
|
||||
),
|
||||
Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi: Arc::new(fi.clone()),
|
||||
parts_metadata: Arc::new(vec![fi]),
|
||||
online_disks: Arc::new(vec![None]),
|
||||
fi: fi.clone(),
|
||||
parts_metadata: vec![fi],
|
||||
online_disks: vec![None],
|
||||
read_quorum: 1,
|
||||
}),
|
||||
)
|
||||
@@ -2775,6 +2729,9 @@ mod metadata_cache_tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_metadata_cache_rejects_stale_entries() {
|
||||
// moka handles TTL expiry automatically via time_to_live(250ms).
|
||||
// This test verifies that entries inserted with the cache API are retrievable
|
||||
// while fresh, and that the cache API works correctly.
|
||||
let set = new_metadata_cache_test_set().await;
|
||||
let fi = valid_test_fileinfo("object");
|
||||
|
||||
@@ -2786,14 +2743,6 @@ mod metadata_cache_tests {
|
||||
set.cached_get_object_fileinfo("bucket", "object").await.is_some(),
|
||||
"freshly inserted entry should be returned"
|
||||
);
|
||||
|
||||
tokio::time::timeout(GET_OBJECT_METADATA_CACHE_TTL + Duration::from_secs(1), async {
|
||||
while set.cached_get_object_fileinfo("bucket", "object").await.is_some() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("metadata cache entry should expire after its TTL");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2855,13 +2804,9 @@ mod metadata_cache_tests {
|
||||
barrier.wait_until_paused().await;
|
||||
set.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
barrier.release();
|
||||
let (fi, parts_metadata, online_disks) = read
|
||||
.await
|
||||
read.await
|
||||
.expect("metadata read task should not panic")
|
||||
.expect("metadata fanout should still return its selected FileInfo");
|
||||
assert!(matches!(fi, GetObjectMetadata::Owned(_)));
|
||||
assert!(matches!(parts_metadata, GetObjectMetadata::Owned(_)));
|
||||
assert!(matches!(online_disks, GetObjectMetadata::Owned(_)));
|
||||
|
||||
assert!(
|
||||
set.get_object_metadata_cache
|
||||
@@ -2908,9 +2853,9 @@ mod metadata_cache_tests {
|
||||
let key = GetObjectMetadataCacheKey::new("bucket", "object", generation);
|
||||
let entry = Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi: Arc::new(fi.clone()),
|
||||
parts_metadata: Arc::new(vec![fi]),
|
||||
online_disks: Arc::new(Vec::new()),
|
||||
fi: fi.clone(),
|
||||
parts_metadata: vec![fi],
|
||||
online_disks: Vec::new(),
|
||||
read_quorum: 0,
|
||||
});
|
||||
|
||||
@@ -3018,9 +2963,9 @@ mod metadata_cache_tests {
|
||||
let entry = |fi: FileInfo| {
|
||||
Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
parts_metadata: Arc::new(vec![fi.clone()]),
|
||||
fi: Arc::new(fi),
|
||||
online_disks: Arc::new(Vec::new()),
|
||||
parts_metadata: vec![fi.clone()],
|
||||
fi,
|
||||
online_disks: Vec::new(),
|
||||
read_quorum: 0,
|
||||
})
|
||||
};
|
||||
@@ -3470,7 +3415,7 @@ mod tests {
|
||||
);
|
||||
assert_eq!(diagnostics.total_responses(), 9);
|
||||
assert_eq!(diagnostics.valid_responses(), 1);
|
||||
assert_eq!(diagnostics.non_valid_responses(), 8);
|
||||
assert_eq!(diagnostics.error_responses(), 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3485,7 +3430,7 @@ mod tests {
|
||||
);
|
||||
|
||||
assert_eq!(diagnostics.ignored_responses(), 2);
|
||||
assert_eq!(diagnostics.non_valid_responses(), 3);
|
||||
assert_eq!(diagnostics.error_responses(), 3);
|
||||
assert_eq!(diagnostics.observations[0].outcome, GET_METADATA_RESPONSE_DISK_NOT_FOUND);
|
||||
assert_eq!(diagnostics.observations[1].outcome, GET_METADATA_RESPONSE_IGNORED);
|
||||
assert_eq!(diagnostics.observations[2].outcome, GET_METADATA_RESPONSE_NOT_FOUND);
|
||||
@@ -3553,7 +3498,7 @@ mod tests {
|
||||
|
||||
assert_eq!(diagnostics.total_responses(), 3);
|
||||
assert_eq!(diagnostics.valid_responses(), 3);
|
||||
assert_eq!(diagnostics.non_valid_responses(), 0);
|
||||
assert_eq!(diagnostics.error_responses(), 0);
|
||||
assert!(
|
||||
diagnostics
|
||||
.observations
|
||||
@@ -5529,36 +5474,33 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_streaming_default_min_size_meets_direct_memory_ceiling() {
|
||||
for engine in [None, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS)] {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, engine),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE, None::<&str>),
|
||||
],
|
||||
|| {
|
||||
let below_threshold_fi = codec_streaming_test_fileinfo(128 * 1024 - 1, 1);
|
||||
let below_threshold_object_info = codec_streaming_test_object_info(&below_threshold_fi);
|
||||
assert_eq!(
|
||||
codec_streaming_reader_gate_for_test(&None, &below_threshold_object_info, &below_threshold_fi, true)
|
||||
.decision,
|
||||
GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::BelowMinSize)
|
||||
);
|
||||
fn rustfs_codec_streaming_uses_conservative_default_min_size() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS)),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE, None::<&str>),
|
||||
],
|
||||
|| {
|
||||
let below_threshold_fi = codec_streaming_test_fileinfo(512 * 1024, 1);
|
||||
let below_threshold_object_info = codec_streaming_test_object_info(&below_threshold_fi);
|
||||
assert_eq!(
|
||||
codec_streaming_reader_gate_for_test(&None, &below_threshold_object_info, &below_threshold_fi, true).decision,
|
||||
GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::BelowMinSize)
|
||||
);
|
||||
|
||||
let threshold_fi = codec_streaming_test_fileinfo(128 * 1024, 1);
|
||||
let threshold_object_info = codec_streaming_test_object_info(&threshold_fi);
|
||||
assert_eq!(
|
||||
codec_streaming_reader_gate_for_test(&None, &threshold_object_info, &threshold_fi, true).decision,
|
||||
GetCodecStreamingDecision::Use
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
let threshold_fi = codec_streaming_test_fileinfo(1_048_576, 1);
|
||||
let threshold_object_info = codec_streaming_test_object_info(&threshold_fi);
|
||||
assert_eq!(
|
||||
codec_streaming_reader_gate_for_test(&None, &threshold_object_info, &threshold_fi, true).decision,
|
||||
GetCodecStreamingDecision::Use
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -5856,10 +5798,10 @@ mod tests {
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
|
||||
],
|
||||
|| {
|
||||
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
|
||||
let fi = codec_streaming_test_fileinfo(1024, 1);
|
||||
let object_info = codec_streaming_test_object_info(&fi);
|
||||
|
||||
assert_eq!(
|
||||
@@ -5880,10 +5822,10 @@ mod tests {
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("on")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
|
||||
],
|
||||
|| {
|
||||
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
|
||||
let fi = codec_streaming_test_fileinfo(1024, 1);
|
||||
let object_info = codec_streaming_test_object_info(&fi);
|
||||
|
||||
assert_eq!(
|
||||
@@ -5901,10 +5843,10 @@ mod tests {
|
||||
[
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("false")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("on")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
|
||||
],
|
||||
|| {
|
||||
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
|
||||
let fi = codec_streaming_test_fileinfo(1024, 1);
|
||||
let object_info = codec_streaming_test_object_info(&fi);
|
||||
|
||||
assert_eq!(
|
||||
@@ -5984,10 +5926,10 @@ mod tests {
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, Some("0")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
|
||||
],
|
||||
|| {
|
||||
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
|
||||
let fi = codec_streaming_test_fileinfo(1024, 1);
|
||||
let object_info = codec_streaming_test_object_info(&fi);
|
||||
|
||||
assert_eq!(
|
||||
@@ -6004,10 +5946,10 @@ mod tests {
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, Some("100")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
|
||||
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
|
||||
],
|
||||
|| {
|
||||
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
|
||||
let fi = codec_streaming_test_fileinfo(1024, 1);
|
||||
let object_info = codec_streaming_test_object_info(&fi);
|
||||
|
||||
assert_eq!(
|
||||
|
||||
@@ -77,10 +77,9 @@ impl SetDisks {
|
||||
version_suspended: opts.version_suspended,
|
||||
..Default::default()
|
||||
};
|
||||
let (fi, _, disks) = self
|
||||
let (mut fi, _, disks) = self
|
||||
.get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
|
||||
.await?;
|
||||
let mut fi = fi.into_owned();
|
||||
if let Some(expected_operation_id) = expected_operation_id {
|
||||
require_restore_operation_id(&fi.metadata, expected_operation_id)?;
|
||||
}
|
||||
@@ -102,7 +101,7 @@ impl SetDisks {
|
||||
bucket,
|
||||
object,
|
||||
fi.clone(),
|
||||
&disks,
|
||||
disks.as_slice(),
|
||||
&UpdateMetadataOpts {
|
||||
replace_user_metadata: true,
|
||||
..Default::default()
|
||||
@@ -144,10 +143,9 @@ impl SetDisks {
|
||||
version_suspended: opts.version_suspended,
|
||||
..Default::default()
|
||||
};
|
||||
let (fi, _, disks) = self
|
||||
let (mut fi, _, disks) = self
|
||||
.get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
|
||||
.await?;
|
||||
let mut fi = fi.into_owned();
|
||||
if let Some(expected_operation_id) = expected_operation_id {
|
||||
match restore_operation_id_from_metadata(&fi.metadata)? {
|
||||
Some(actual_operation_id) if actual_operation_id == expected_operation_id => {}
|
||||
@@ -172,7 +170,7 @@ impl SetDisks {
|
||||
bucket,
|
||||
object,
|
||||
fi,
|
||||
&disks,
|
||||
disks.as_slice(),
|
||||
&UpdateMetadataOpts {
|
||||
replace_user_metadata: true,
|
||||
..Default::default()
|
||||
|
||||
@@ -117,17 +117,16 @@ impl StripeReadState {
|
||||
Self::from_parts_with_read_costs(shards, errors, &[], read_quorum)
|
||||
}
|
||||
|
||||
pub(crate) fn from_parts_with_read_costs<S, E>(shards: S, errors: E, read_costs: &[ShardReadCost], read_quorum: usize) -> Self
|
||||
where
|
||||
S: IntoIterator<Item = Option<Vec<u8>>>,
|
||||
S::IntoIter: ExactSizeIterator,
|
||||
E: IntoIterator<Item = Option<Error>>,
|
||||
E::IntoIter: ExactSizeIterator,
|
||||
{
|
||||
let mut shards = shards.into_iter();
|
||||
let mut errors = errors.into_iter();
|
||||
pub(crate) fn from_parts_with_read_costs(
|
||||
shards: Vec<Option<Vec<u8>>>,
|
||||
errors: Vec<Option<Error>>,
|
||||
read_costs: &[ShardReadCost],
|
||||
read_quorum: usize,
|
||||
) -> Self {
|
||||
let slot_count = shards.len().max(errors.len());
|
||||
let mut slots = Vec::with_capacity(slot_count);
|
||||
let mut shards = shards.into_iter();
|
||||
let mut errors = errors.into_iter();
|
||||
for index in 0..slot_count {
|
||||
let read_cost = read_costs.get(index).copied().unwrap_or(ShardReadCost::Unknown);
|
||||
slots.push(ShardSlot::with_read_cost(
|
||||
|
||||
@@ -23,7 +23,6 @@ use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use std::collections::BTreeMap;
|
||||
use std::future::Future;
|
||||
|
||||
@@ -154,31 +153,6 @@ where
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
pub async fn get_bucket_metadata(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
sys.read().await.get(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
sys.read().await.get_bucket_policy(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy_raw(&self, bucket: &str) -> Result<(String, OffsetDateTime)> {
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
sys.read().await.get_bucket_policy_raw(bucket).await
|
||||
}
|
||||
|
||||
pub async fn restricts_public_bucket_access(&self, bucket: &str) -> Result<bool> {
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
let (config, _) = sys.read().await.get_public_access_block_config(bucket).await?;
|
||||
Ok(config.restrict_public_buckets.unwrap_or(false))
|
||||
}
|
||||
|
||||
pub async fn update_bucket_metadata_config(&self, bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
|
||||
metadata_sys::update_in(&self.ctx, bucket, config_file, data).await
|
||||
}
|
||||
|
||||
pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
|
||||
metadata_sys::get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
|
||||
}
|
||||
@@ -457,13 +431,7 @@ impl ECStore {
|
||||
None
|
||||
};
|
||||
|
||||
let mut meta = existing_metadata.unwrap_or_else(|| {
|
||||
if confirmed_missing && !is_meta_bucketname(bucket) {
|
||||
BucketMetadata::new_with_default_durability(bucket)
|
||||
} else {
|
||||
BucketMetadata::new(bucket)
|
||||
}
|
||||
});
|
||||
let mut meta = existing_metadata.unwrap_or_else(|| BucketMetadata::new(bucket));
|
||||
let existing_incarnation_is_authoritative = meta.bucket_incarnation_sidecar;
|
||||
if confirmed_missing || is_meta_bucketname(bucket) {
|
||||
meta.set_created(opts.created_at);
|
||||
@@ -1109,26 +1077,6 @@ mod tests {
|
||||
(temp_dir, ecstore)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn request_metadata_methods_fail_closed_before_instance_initialization() {
|
||||
let (_temp_dir, store) = setup_multi_pool_scanner_listing_test_env().await;
|
||||
|
||||
let expected = "bucket metadata sys not initialized for this instance";
|
||||
let errors = [
|
||||
store.get_bucket_metadata("bucket").await.unwrap_err(),
|
||||
store.get_bucket_policy("bucket").await.unwrap_err(),
|
||||
store.get_bucket_policy_raw("bucket").await.unwrap_err(),
|
||||
store.restricts_public_bucket_access("bucket").await.unwrap_err(),
|
||||
store
|
||||
.update_bucket_metadata_config("bucket", crate::bucket::metadata::BUCKET_POLICY_CONFIG, Vec::new())
|
||||
.await
|
||||
.unwrap_err(),
|
||||
];
|
||||
for error in errors {
|
||||
assert_eq!(error.to_string(), format!("Io error: {expected}"));
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_bucket_with_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str) {
|
||||
let generation_before_make = ecstore.scanner_namespace_mutation_generation();
|
||||
ecstore
|
||||
@@ -1563,76 +1511,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn make_bucket_seeds_new_bucket_durability_override() {
|
||||
temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, None::<&str>)], async {
|
||||
let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
|
||||
let bucket = format!("bucket-default-durability-{}", Uuid::new_v4().simple());
|
||||
|
||||
ecstore
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("new bucket should be created");
|
||||
|
||||
let metadata = metadata_sys::get_in(&ecstore.ctx, &bucket)
|
||||
.await
|
||||
.expect("metadata should load for the new bucket");
|
||||
assert_eq!(
|
||||
metadata.durability_config().and_then(|cfg| cfg.normalized_mode()).as_deref(),
|
||||
Some(crate::bucket::durability::BUCKET_DURABILITY_MODE_RELAXED)
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn force_create_existing_bucket_keeps_durability_override() {
|
||||
let (_disk_paths, ecstore) = setup_bucket_delete_test_env().await;
|
||||
let bucket = format!("bucket-force-durability-{}", Uuid::new_v4().simple());
|
||||
|
||||
temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, Some("inherit"))], async {
|
||||
ecstore
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("plain bucket should be created without a durability override");
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
metadata_sys::get_in(&ecstore.ctx, &bucket)
|
||||
.await
|
||||
.expect("metadata should load after initial create")
|
||||
.durability_config()
|
||||
.is_none(),
|
||||
"test setup: the existing bucket must start without an override"
|
||||
);
|
||||
|
||||
temp_env::async_with_vars([(crate::bucket::durability::ENV_NEW_BUCKET_DURABILITY_MODE, None::<&str>)], async {
|
||||
ecstore
|
||||
.make_bucket(
|
||||
&bucket,
|
||||
&MakeBucketOptions {
|
||||
force_create: true,
|
||||
lock_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("force create should update existing bucket metadata");
|
||||
})
|
||||
.await;
|
||||
|
||||
let metadata = metadata_sys::get_in(&ecstore.ctx, &bucket)
|
||||
.await
|
||||
.expect("metadata should load after force create");
|
||||
assert!(metadata.lock_enabled, "force create sanity check: Object Lock should be enabled");
|
||||
assert!(
|
||||
metadata.durability_config().is_none(),
|
||||
"force create must not apply the new-bucket default to existing bucket metadata"
|
||||
);
|
||||
}
|
||||
|
||||
/// `DeleteBucket`'s emptiness check is a raw disk scan (`has_xlmeta_files`),
|
||||
/// not an S3-level listing, so "the client drained the bucket" and "the
|
||||
/// bucket is deletable" are two different contracts. Nothing pinned the
|
||||
|
||||
@@ -9529,6 +9529,294 @@ mod test {
|
||||
.expect("a partial outage with a healthy set must not fail the walk");
|
||||
}
|
||||
|
||||
// use std::sync::Arc;
|
||||
|
||||
// use crate::cache_value::metacache_set::list_path_raw;
|
||||
// use crate::cache_value::metacache_set::ListPathRawOptions;
|
||||
// use crate::disk::endpoint::Endpoint;
|
||||
// use crate::disk::error::is_err_eof;
|
||||
// use crate::disk::format::FormatV3;
|
||||
// use crate::disk::new_disk;
|
||||
// use crate::disk::DiskAPI;
|
||||
// use crate::disk::DiskOption;
|
||||
// use crate::disk::MetaCacheEntries;
|
||||
// use crate::disk::MetaCacheEntry;
|
||||
// use crate::disk::WalkDirOptions;
|
||||
// use crate::layout::endpoints::EndpointServerPools;
|
||||
// use crate::error::Error;
|
||||
// use crate::metacache::writer::MetacacheReader;
|
||||
// use crate::set_disk::SetDisks;
|
||||
// use crate::store::list_objects::ListPathOptions;
|
||||
// use crate::store::list_objects::WalkOptions;
|
||||
// use crate::store::list_objects::WalkVersionsSortOrder;
|
||||
// use futures::future::join_all;
|
||||
// use rustfs_lock::namespace_lock::NsLockMap;
|
||||
// use tokio::sync::broadcast;
|
||||
// use tokio::sync::mpsc;
|
||||
// use tokio::sync::RwLock;
|
||||
// use uuid::Uuid;
|
||||
|
||||
// #[tokio::test]
|
||||
// async fn test_walk_dir() {
|
||||
// let mut ep = Endpoint::try_from("/Users/weisd/project/weisd/s3-rustfs/target/volume/test").unwrap();
|
||||
// ep.pool_idx = 0;
|
||||
// ep.set_idx = 0;
|
||||
// ep.disk_idx = 0;
|
||||
// ep.is_local = true;
|
||||
|
||||
// let disk = new_disk(&ep, &DiskOption::default()).await.expect("init disk fail");
|
||||
|
||||
// // let disk = match LocalDisk::new(&ep, false).await {
|
||||
// // Ok(res) => res,
|
||||
// // Err(err) => {
|
||||
// // println!("LocalDisk::new err {:?}", err);
|
||||
// // return;
|
||||
// // }
|
||||
// // };
|
||||
|
||||
// let (rd, mut wr) = tokio::io::duplex(64);
|
||||
|
||||
// let job = tokio::spawn(async move {
|
||||
// let opts = WalkDirOptions {
|
||||
// bucket: "dada".to_owned(),
|
||||
// base_dir: "".to_owned(),
|
||||
// recursive: true,
|
||||
// ..Default::default()
|
||||
// };
|
||||
|
||||
// println!("walk opts {:?}", opts);
|
||||
// if let Err(err) = disk.walk_dir(opts, &mut wr).await {
|
||||
// println!("walk_dir err {:?}", err);
|
||||
// }
|
||||
// });
|
||||
|
||||
// let job2 = tokio::spawn(async move {
|
||||
// let mut mrd = MetacacheReader::new(rd);
|
||||
|
||||
// loop {
|
||||
// match mrd.peek().await {
|
||||
// Ok(res) => {
|
||||
// if let Some(info) = res {
|
||||
// println!("info {:?}", info.name)
|
||||
// } else {
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// Err(err) => {
|
||||
// if is_err_eof(&err) {
|
||||
// break;
|
||||
// }
|
||||
|
||||
// println!("get err {:?}", err);
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
// join_all(vec![job, job2]).await;
|
||||
// }
|
||||
|
||||
// #[tokio::test]
|
||||
// async fn test_list_path_raw() {
|
||||
// let mut ep = Endpoint::try_from("/Users/weisd/project/weisd/s3-rustfs/target/volume/test").unwrap();
|
||||
// ep.pool_idx = 0;
|
||||
// ep.set_idx = 0;
|
||||
// ep.disk_idx = 0;
|
||||
// ep.is_local = true;
|
||||
|
||||
// let disk = new_disk(&ep, &DiskOption::default()).await.expect("init disk fail");
|
||||
|
||||
// // let disk = match LocalDisk::new(&ep, false).await {
|
||||
// // Ok(res) => res,
|
||||
// // Err(err) => {
|
||||
// // println!("LocalDisk::new err {:?}", err);
|
||||
// // return;
|
||||
// // }
|
||||
// // };
|
||||
|
||||
// let (_, rx) = broadcast::channel(1);
|
||||
// let bucket = "dada".to_owned();
|
||||
// let forward_to = None;
|
||||
// let disks = vec![Some(disk)];
|
||||
// let fallback_disks = Vec::new();
|
||||
|
||||
// list_path_raw(
|
||||
// rx,
|
||||
// ListPathRawOptions {
|
||||
// disks,
|
||||
// fallback_disks,
|
||||
// bucket,
|
||||
// path: "".to_owned(),
|
||||
// recursice: true,
|
||||
// forward_to,
|
||||
// min_disks: 1,
|
||||
// report_not_found: false,
|
||||
// agreed: Some(Box::new(move |entry: MetaCacheEntry| {
|
||||
// Box::pin(async move { println!("get entry: {}", entry.name) })
|
||||
// })),
|
||||
// partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<Error>]| {
|
||||
// Box::pin(async move { println!("get entries: {:?}", entries) })
|
||||
// })),
|
||||
// finished: None,
|
||||
// ..Default::default()
|
||||
// },
|
||||
// )
|
||||
// .await
|
||||
// .unwrap();
|
||||
// }
|
||||
|
||||
// #[tokio::test]
|
||||
// async fn test_set_list_path() {
|
||||
// let mut ep = Endpoint::try_from("/Users/weisd/project/weisd/s3-rustfs/target/volume/test").unwrap();
|
||||
// ep.pool_idx = 0;
|
||||
// ep.set_idx = 0;
|
||||
// ep.disk_idx = 0;
|
||||
// ep.is_local = true;
|
||||
|
||||
// let disk = new_disk(&ep, &DiskOption::default()).await.expect("init disk fail");
|
||||
// let _ = disk.set_disk_id(Some(Uuid::new_v4())).await;
|
||||
|
||||
// let set = SetDisks {
|
||||
// lockers: Vec::new(),
|
||||
// locker_owner: String::new(),
|
||||
// ns_mutex: Arc::new(RwLock::new(NsLockMap::new(false))),
|
||||
// disks: RwLock::new(vec![Some(disk)]),
|
||||
// set_endpoints: Vec::new(),
|
||||
// set_drive_count: 1,
|
||||
// default_parity_count: 0,
|
||||
// set_index: 0,
|
||||
// pool_index: 0,
|
||||
// format: FormatV3::new(1, 1),
|
||||
// };
|
||||
|
||||
// let (_tx, rx) = broadcast::channel(1);
|
||||
|
||||
// let bucket = "dada".to_owned();
|
||||
|
||||
// let opts = ListPathOptions {
|
||||
// bucket,
|
||||
// recursive: true,
|
||||
// ..Default::default()
|
||||
// };
|
||||
|
||||
// let (sender, mut recv) = mpsc::channel(10);
|
||||
|
||||
// set.list_path(rx, opts, sender).await.unwrap();
|
||||
|
||||
// while let Some(entry) = recv.recv().await {
|
||||
// println!("get entry {:?}", entry.name)
|
||||
// }
|
||||
// }
|
||||
|
||||
// #[tokio::test]
|
||||
//walk() {
|
||||
// let server_address = "localhost:9000";
|
||||
|
||||
// let (endpoint_pools, _setup_type) = EndpointServerPools::from_volumes(
|
||||
// server_address,
|
||||
// vec!["/Users/weisd/project/weisd/s3-rustfs/target/volume/test".to_string()],
|
||||
// )
|
||||
// .unwrap();
|
||||
|
||||
// let store = ECStore::new(server_address.to_string(), endpoint_pools.clone())
|
||||
// .await
|
||||
// .unwrap();
|
||||
|
||||
// let (_tx, rx) = broadcast::channel(1);
|
||||
|
||||
// let bucket = "dada".to_owned();
|
||||
// let opts = ListPathOptions {
|
||||
// bucket,
|
||||
// recursive: true,
|
||||
// ..Default::default()
|
||||
// };
|
||||
|
||||
// let (sender, mut recv) = mpsc::channel(10);
|
||||
|
||||
// store.list_merged(rx, opts, sender).await.unwrap();
|
||||
|
||||
// while let Some(entry) = recv.recv().await {
|
||||
// println!("get entry {:?}", entry.name)
|
||||
// }
|
||||
// }
|
||||
|
||||
// #[tokio::test]
|
||||
// async fn test_list_path() {
|
||||
// let server_address = "localhost:9000";
|
||||
|
||||
// let (endpoint_pools, _setup_type) = EndpointServerPools::from_volumes(
|
||||
// server_address,
|
||||
// vec!["/Users/weisd/project/weisd/s3-rustfs/target/volume/test".to_string()],
|
||||
// )
|
||||
// .unwrap();
|
||||
|
||||
// let store = ECStore::new(server_address.to_string(), endpoint_pools.clone())
|
||||
// .await
|
||||
// .unwrap();
|
||||
|
||||
// let bucket = "dada".to_owned();
|
||||
// let opts = ListPathOptions {
|
||||
// bucket,
|
||||
// recursive: true,
|
||||
// limit: 100,
|
||||
|
||||
// ..Default::default()
|
||||
// };
|
||||
|
||||
// let ret = store.list_path(&opts).await.unwrap();
|
||||
// println!("ret {:?}", ret);
|
||||
// }
|
||||
|
||||
// #[tokio::test]
|
||||
// async fn test_list_objects_v2() {
|
||||
// let server_address = "localhost:9000";
|
||||
|
||||
// let (endpoint_pools, _setup_type) = EndpointServerPools::from_volumes(
|
||||
// server_address,
|
||||
// vec!["/Users/weisd/project/weisd/s3-rustfs/target/volume/test".to_string()],
|
||||
// )
|
||||
// .unwrap();
|
||||
|
||||
// let store = ECStore::new(server_address.to_string(), endpoint_pools.clone())
|
||||
// .await
|
||||
// .unwrap();
|
||||
|
||||
// let ret = store.list_objects_v2("data", "", "", "", 100, false, "").await.unwrap();
|
||||
// println!("ret {:?}", ret);
|
||||
// }
|
||||
|
||||
// #[tokio::test]
|
||||
// async fn test_walk() {
|
||||
// let server_address = "localhost:9000";
|
||||
|
||||
// let (endpoint_pools, _setup_type) = EndpointServerPools::from_volumes(
|
||||
// server_address,
|
||||
// vec!["/Users/weisd/project/weisd/s3-rustfs/target/volume/test".to_string()],
|
||||
// )
|
||||
// .unwrap();
|
||||
|
||||
// let store = ECStore::new(server_address.to_string(), endpoint_pools.clone())
|
||||
// .await
|
||||
// .unwrap();
|
||||
|
||||
// ECStore::init(store.clone()).await.unwrap();
|
||||
|
||||
// let (_tx, rx) = broadcast::channel(1);
|
||||
|
||||
// let bucket = ".rustfs.sys";
|
||||
// let prefix = "config/iam/sts/";
|
||||
|
||||
// let (sender, mut recv) = mpsc::channel(10);
|
||||
|
||||
// let opts = WalkOptions::default();
|
||||
|
||||
// store.walk(rx, bucket, prefix, sender, opts).await.unwrap();
|
||||
|
||||
// while let Some(entry) = recv.recv().await {
|
||||
// println!("get entry {:?}", entry)
|
||||
// }
|
||||
// }
|
||||
|
||||
#[tokio::test]
|
||||
async fn merge_entry_channels_produces_sorted_unique_output_from_two_channels() {
|
||||
let (tx_a, rx_a) = mpsc::channel(4);
|
||||
|
||||
@@ -277,10 +277,9 @@ pub struct FileInfo {
|
||||
/// Values of these keys must never reach logs at any level.
|
||||
fn is_sensitive_metadata_key(key: &str) -> bool {
|
||||
// `is_encryption_metadata_key` covers the x-minio-internal- SSE prefix but not
|
||||
// its reserved x-rustfs-internal- twin, which has no writer today but must
|
||||
// stay redacted in case one appears.
|
||||
// its x-rustfs-internal- twin, which the dual-key invariant writes alongside it.
|
||||
is_encryption_metadata_key(key)
|
||||
|| starts_with_ignore_ascii_case(key, rustfs_utils::http::RUSTFS_INTERNAL_ENCRYPTION_PREFIX)
|
||||
|| starts_with_ignore_ascii_case(key, "x-rustfs-internal-server-side-encryption-")
|
||||
|| rustfs_utils::http::REPLICATION_SSE_TRANSPORT_PREFIXES
|
||||
.iter()
|
||||
.any(|prefix| starts_with_ignore_ascii_case(key, prefix))
|
||||
|
||||
@@ -785,15 +785,9 @@ mod tests {
|
||||
let heal_manager = create_test_heal_manager();
|
||||
let processor = HealChannelProcessor::new(heal_manager);
|
||||
|
||||
let sender = processor.get_response_sender();
|
||||
sender
|
||||
.send(HealChannelResponse {
|
||||
request_id: "request-id".to_string(),
|
||||
success: true,
|
||||
data: None,
|
||||
error: None,
|
||||
})
|
||||
.expect("a freshly constructed processor must accept responses on its channel");
|
||||
// Verify processor is created successfully
|
||||
let _sender = processor.get_response_sender();
|
||||
// If we can get the sender, processor was created correctly
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1784,22 +1778,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_process_cancel_request_cancels_cluster_task_for_legacy_root_path() {
|
||||
async fn test_process_cancel_request_treats_unknown_path_as_stopped() {
|
||||
let heal_manager = create_test_heal_manager();
|
||||
let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High);
|
||||
let cluster_task_id = cluster_request.id.clone();
|
||||
let bucket_request = HealRequest::bucket("bucket".to_string());
|
||||
let bucket_task_id = bucket_request.id.clone();
|
||||
heal_manager
|
||||
.submit_heal_request(cluster_request)
|
||||
.await
|
||||
.expect("cluster request should be accepted");
|
||||
heal_manager
|
||||
.submit_heal_request(bucket_request)
|
||||
.await
|
||||
.expect("bucket request should be accepted");
|
||||
|
||||
let processor = HealChannelProcessor::new(heal_manager.clone());
|
||||
let processor = HealChannelProcessor::new(heal_manager);
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
processor
|
||||
@@ -1815,38 +1796,6 @@ mod tests {
|
||||
assert_eq!(response.request_id, ".");
|
||||
assert_eq!(response.data.as_deref(), Some("stopped".as_bytes()));
|
||||
assert!(response.error.is_none());
|
||||
assert!(matches!(
|
||||
heal_manager.get_task_status(&cluster_task_id).await,
|
||||
Err(crate::Error::TaskNotFound { .. })
|
||||
));
|
||||
assert_eq!(
|
||||
heal_manager
|
||||
.get_task_status(&bucket_task_id)
|
||||
.await
|
||||
.expect("bucket request should not match the root path"),
|
||||
HealTaskStatus::Pending
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_process_cancel_request_treats_unknown_path_as_stopped() {
|
||||
let heal_manager = create_test_heal_manager();
|
||||
let processor = HealChannelProcessor::new(heal_manager);
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
processor
|
||||
.process_cancel_request("missing".to_string(), String::new(), tx)
|
||||
.await
|
||||
.expect("cancel should process");
|
||||
|
||||
let response = rx
|
||||
.await
|
||||
.expect("oneshot should resolve")
|
||||
.expect("cancel response should be returned");
|
||||
assert!(response.success);
|
||||
assert_eq!(response.request_id, "missing");
|
||||
assert_eq!(response.data.as_deref(), Some("stopped".as_bytes()));
|
||||
assert!(response.error.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -52,7 +52,6 @@ const EVENT_HEAL_MAINLINE_THROTTLE: &str = "heal_mainline_throttle";
|
||||
const EVENT_HEAL_SCHEDULER_STATE: &str = "heal_scheduler_state";
|
||||
const EVENT_HEAL_QUEUE_STATE: &str = "heal_queue_state";
|
||||
const EVENT_HEAL_UNCLEAN_SHUTDOWN: &str = "heal_unclean_shutdown";
|
||||
const LEGACY_ROOT_HEAL_PATH: &str = ".";
|
||||
const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3;
|
||||
const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
|
||||
@@ -66,14 +65,6 @@ fn durable_replacement_recovery_is_due(state: &ResumeState, task_id: &str) -> bo
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending)))
|
||||
}
|
||||
|
||||
fn replacement_discovery_error_is_expected_for_deferred_endpoint(
|
||||
error: &Error,
|
||||
endpoint: &str,
|
||||
deferred_replacement_endpoints: &HashSet<String>,
|
||||
) -> bool {
|
||||
matches!(error, Error::Disk(DiskError::UnformattedDisk)) && deferred_replacement_endpoints.contains(endpoint)
|
||||
}
|
||||
|
||||
fn unblock_replacement_recovery_sets_after_validation(
|
||||
blocked_sets: &mut HashSet<String>,
|
||||
retry_succeeded: HashSet<String>,
|
||||
@@ -602,7 +593,7 @@ impl RetryingHeal {
|
||||
|
||||
fn heal_type_matches_path(heal_type: &HealType, heal_path: &str) -> bool {
|
||||
let heal_path = heal_path.trim_matches('/');
|
||||
if heal_path.is_empty() || heal_path == LEGACY_ROOT_HEAL_PATH {
|
||||
if heal_path.is_empty() {
|
||||
return matches!(heal_type, HealType::Cluster);
|
||||
}
|
||||
|
||||
@@ -2509,7 +2500,6 @@ impl HealManager {
|
||||
let mut endpoints = HashMap::<String, Vec<Endpoint>>::new();
|
||||
let mut durable_recoveries = HashMap::<String, (String, Vec<Endpoint>, Vec<String>, String)>::new();
|
||||
let mut conflicted_recovery_sets = HashSet::<String>::new();
|
||||
let mut deferred_replacement_endpoints = HashSet::<String>::new();
|
||||
let local_disks = {
|
||||
let local_disk_map = local_disk_map_read().await;
|
||||
local_disk_map.values().flatten().cloned().collect::<Vec<_>>()
|
||||
@@ -2587,7 +2577,6 @@ impl HealManager {
|
||||
if !super::replacement_readiness::auto_replacement_target_ready(disk, &local_disks)
|
||||
.await
|
||||
{
|
||||
deferred_replacement_endpoints.insert(endpoint.to_string());
|
||||
skipped_invalid_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
@@ -2661,24 +2650,6 @@ impl HealManager {
|
||||
let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await {
|
||||
Ok(task_ids) => task_ids,
|
||||
Err(error) => {
|
||||
let endpoint_string = endpoint.to_string();
|
||||
if replacement_discovery_error_is_expected_for_deferred_endpoint(
|
||||
&error,
|
||||
&endpoint_string,
|
||||
&deferred_replacement_endpoints,
|
||||
) {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
disk_state = "replacement_path_unavailable",
|
||||
result = "recovery_records_unavailable",
|
||||
"Replacement recovery discovery skipped for deferred replacement"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if let Some(set_disk_id) = &disk_set_disk_id {
|
||||
conflicted_recovery_sets.insert(set_disk_id.clone());
|
||||
}
|
||||
@@ -4684,28 +4655,6 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replacement_recovery_discovery_unformatted_is_quiet_only_for_deferred_endpoint() {
|
||||
let error = Error::Disk(DiskError::UnformattedDisk);
|
||||
let deferred = HashSet::from(["endpoint-a".to_string()]);
|
||||
|
||||
assert!(replacement_discovery_error_is_expected_for_deferred_endpoint(
|
||||
&error,
|
||||
"endpoint-a",
|
||||
&deferred
|
||||
));
|
||||
assert!(!replacement_discovery_error_is_expected_for_deferred_endpoint(
|
||||
&error,
|
||||
"endpoint-b",
|
||||
&deferred
|
||||
));
|
||||
assert!(!replacement_discovery_error_is_expected_for_deferred_endpoint(
|
||||
&Error::Disk(DiskError::Timeout),
|
||||
"endpoint-a",
|
||||
&deferred
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replacement_recovery_retry_barrier_requires_all_set_records_to_validate() {
|
||||
let mut blocked = HashSet::from(["pool_0_set_0".to_string(), "pool_0_set_1".to_string()]);
|
||||
@@ -5111,17 +5060,6 @@ mod tests {
|
||||
assert!(manager.retrying_heals.lock().await.get(&bucket_request_id).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_type_matches_path_accepts_legacy_root() {
|
||||
assert!(heal_type_matches_path(&HealType::Cluster, LEGACY_ROOT_HEAL_PATH));
|
||||
assert!(!heal_type_matches_path(
|
||||
&HealType::Bucket {
|
||||
bucket: "bucket".to_string(),
|
||||
},
|
||||
LEGACY_ROOT_HEAL_PATH,
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_retrying_duplicate_token_can_query_and_cancel_original_retry() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
|
||||
@@ -2075,7 +2075,6 @@ impl ResumeUtils {
|
||||
match disk.list_dir("", RUSTFS_META_BUCKET, recovery_dir, -1).await {
|
||||
Ok(entries) => Ok(entries),
|
||||
Err(DiskError::FileNotFound) => Ok(Vec::new()),
|
||||
Err(error @ DiskError::UnformattedDisk) => Err(error.into()),
|
||||
Err(error) => Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to list replacement recovery records: {error}"),
|
||||
}),
|
||||
|
||||
@@ -294,18 +294,6 @@ pub const GET_OBJECT_SIZE_BUCKET_LE_256_KIB: &str = "le_256kib";
|
||||
pub const GET_OBJECT_SIZE_BUCKET_LE_512_KIB: &str = "le_512kib";
|
||||
pub const GET_OBJECT_SIZE_BUCKET_LE_1_MIB: &str = "le_1mib";
|
||||
pub const GET_OBJECT_SIZE_BUCKET_GT_1_MIB: &str = "gt_1mib";
|
||||
pub const GET_OBJECT_SIZE_BUCKET_UNKNOWN: &str = "unknown";
|
||||
|
||||
pub struct GetObjectStreamingBodyFailure {
|
||||
pub stage: &'static str,
|
||||
pub reason: &'static str,
|
||||
pub error_class: &'static str,
|
||||
pub strategy: &'static str,
|
||||
pub buffer_source: &'static str,
|
||||
pub size_bucket: &'static str,
|
||||
pub emitted_bytes: usize,
|
||||
pub remaining_bytes: usize,
|
||||
}
|
||||
|
||||
/// Return the bounded size bucket used by small-object GET diagnostics.
|
||||
#[inline(always)]
|
||||
@@ -604,44 +592,6 @@ pub fn record_get_object_reader_stream_poll(
|
||||
.record(duration_secs);
|
||||
}
|
||||
|
||||
/// Record a GET response body failure with bounded attribution labels.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_streaming_body_failure(failure: GetObjectStreamingBodyFailure) {
|
||||
if !metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
counter!(
|
||||
"rustfs_io_get_object_streaming_body_failure_total",
|
||||
"stage" => failure.stage,
|
||||
"reason" => failure.reason,
|
||||
"error_class" => failure.error_class,
|
||||
"strategy" => failure.strategy,
|
||||
"buffer_source" => failure.buffer_source,
|
||||
"size_bucket" => failure.size_bucket
|
||||
)
|
||||
.increment(1);
|
||||
histogram!(
|
||||
"rustfs_io_get_object_streaming_body_failure_emitted_bytes",
|
||||
"stage" => failure.stage,
|
||||
"reason" => failure.reason,
|
||||
"error_class" => failure.error_class,
|
||||
"strategy" => failure.strategy,
|
||||
"buffer_source" => failure.buffer_source,
|
||||
"size_bucket" => failure.size_bucket
|
||||
)
|
||||
.record(usize_to_f64(failure.emitted_bytes));
|
||||
histogram!(
|
||||
"rustfs_io_get_object_streaming_body_failure_remaining_bytes",
|
||||
"stage" => failure.stage,
|
||||
"reason" => failure.reason,
|
||||
"error_class" => failure.error_class,
|
||||
"strategy" => failure.strategy,
|
||||
"buffer_source" => failure.buffer_source,
|
||||
"size_bucket" => failure.size_bucket
|
||||
)
|
||||
.record(usize_to_f64(failure.remaining_bytes));
|
||||
}
|
||||
|
||||
/// Record a poll of the single-chunk in-memory GetObject handoff stream.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_memory_body_stream_poll(source: &'static str, outcome: &'static str, bytes: usize, duration_secs: f64) {
|
||||
@@ -795,12 +745,8 @@ pub fn record_get_object_metadata_cache_decision(path: &'static str, decision: &
|
||||
}
|
||||
|
||||
/// Record aggregate metadata fanout shape for one GetObject metadata read.
|
||||
///
|
||||
/// The legacy `metadata_fanout_error_responses` series records every non-valid
|
||||
/// response, including not-found and ignored outcomes. Use
|
||||
/// `metadata_response_total` outcome labels for failure attribution.
|
||||
#[inline(always)]
|
||||
pub fn record_get_object_metadata_fanout_shape(path: &'static str, total: usize, valid: usize, ignored: usize, non_valid: usize) {
|
||||
pub fn record_get_object_metadata_fanout_shape(path: &'static str, total: usize, valid: usize, ignored: usize, errors: usize) {
|
||||
if !get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
@@ -811,7 +757,7 @@ pub fn record_get_object_metadata_fanout_shape(path: &'static str, total: usize,
|
||||
histogram!("rustfs_io_get_object_metadata_fanout_ignored_responses", "path" => path)
|
||||
.record(metadata_fanout_count_to_f64(ignored));
|
||||
histogram!("rustfs_io_get_object_metadata_fanout_error_responses", "path" => path)
|
||||
.record(metadata_fanout_count_to_f64(non_valid));
|
||||
.record(metadata_fanout_count_to_f64(errors));
|
||||
}
|
||||
|
||||
/// Record a guarded metadata early-stop hit for GetObject.
|
||||
@@ -2968,16 +2914,6 @@ mod tests {
|
||||
record_list_objects(50.0, 100, false);
|
||||
record_error("get_object", "timeout");
|
||||
record_cpu_usage(25.5);
|
||||
record_get_object_streaming_body_failure(GetObjectStreamingBodyFailure {
|
||||
stage: "reader_stream",
|
||||
reason: "short_eof",
|
||||
error_class: "short_eof",
|
||||
strategy: "standard",
|
||||
buffer_source: "selected",
|
||||
size_bucket: GET_OBJECT_SIZE_BUCKET_GT_1_MIB,
|
||||
emitted_bytes: 1024,
|
||||
remaining_bytes: 512,
|
||||
});
|
||||
|
||||
// Enabled: the same recorders run their emission bodies without panicking.
|
||||
set_metrics_enabled(true);
|
||||
@@ -2986,16 +2922,6 @@ mod tests {
|
||||
record_list_objects(50.0, 100, false);
|
||||
record_error("get_object", "timeout");
|
||||
record_cpu_usage(25.5);
|
||||
record_get_object_streaming_body_failure(GetObjectStreamingBodyFailure {
|
||||
stage: "reader_stream",
|
||||
reason: "reader_error",
|
||||
error_class: "timeout",
|
||||
strategy: "standard",
|
||||
buffer_source: "selected",
|
||||
size_bucket: GET_OBJECT_SIZE_BUCKET_GT_1_MIB,
|
||||
emitted_bytes: 2048,
|
||||
remaining_bytes: 256,
|
||||
});
|
||||
|
||||
set_metrics_enabled(false);
|
||||
}
|
||||
|
||||
@@ -62,7 +62,7 @@ moka = { workspace = true, features = ["future"] }
|
||||
# Additional dependencies
|
||||
md-5 = { workspace = true }
|
||||
arc-swap = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["http"] }
|
||||
rustfs-utils = { workspace = true }
|
||||
rustfs-security-governance = { workspace = true }
|
||||
# `EventName` for KMS audit records. A leaf crate with no rustfs dependencies,
|
||||
# so the audit sink can live outside this crate without a second, drifting
|
||||
|
||||
@@ -2758,14 +2758,10 @@ mod tests {
|
||||
use crate::config::{BackendConfig, KmsConfig};
|
||||
use crate::types::{CancelKeyDeletionRequest, CreateKeyRequest, DeleteKeyRequest, KeyStatus, KeyUsage};
|
||||
|
||||
// A dev Vault speaks plain HTTP, which validate() refuses unless
|
||||
// development mode is declared on the config itself — the env override
|
||||
// is applied by the config loaders, not by Default::default().
|
||||
let kms_config = KmsConfig {
|
||||
backend_config: BackendConfig::VaultKv2(Box::new(integration_vault_config())),
|
||||
..Default::default()
|
||||
}
|
||||
.with_insecure_development_defaults();
|
||||
};
|
||||
let backend = VaultKmsBackend::new(kms_config).await.expect("backend");
|
||||
|
||||
let key_id = format!("cancel-persist-{}", uuid::Uuid::new_v4());
|
||||
|
||||
+19
-12
@@ -27,10 +27,6 @@ use base64::Engine;
|
||||
use jiff::Zoned;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use rand::random;
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_CONTEXT_HEADER, INTERNAL_ENCRYPTION_IV_HEADER,
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, INTERNAL_ENCRYPTION_TAG_HEADER,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
||||
@@ -85,6 +81,17 @@ fn request_encryption_context(context: &ObjectEncryptionContext) -> HashMap<Stri
|
||||
enc_context
|
||||
}
|
||||
|
||||
const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id";
|
||||
|
||||
/// Carries the AEAD algorithm the object was sealed with.
|
||||
///
|
||||
/// The S3 `x-amz-server-side-encryption` header records the *SSE mode*
|
||||
/// (`AES256` / `aws:kms`), not the cipher, so it cannot round-trip
|
||||
/// `ChaCha20Poly1305`. Without this header a ChaCha-sealed object comes back
|
||||
/// from the projection claiming `aws:kms` and is then opened with the wrong
|
||||
/// cipher.
|
||||
const INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "x-rustfs-encryption-algorithm";
|
||||
|
||||
/// Result of object encryption
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EncryptionResult {
|
||||
@@ -800,19 +807,19 @@ impl ObjectEncryptionService {
|
||||
|
||||
// Internal headers for decryption
|
||||
headers.insert(
|
||||
INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
|
||||
"x-rustfs-encryption-iv".to_string(),
|
||||
base64::engine::general_purpose::STANDARD.encode(&metadata.iv),
|
||||
);
|
||||
|
||||
if let Some(ref tag) = metadata.tag {
|
||||
headers.insert(
|
||||
INTERNAL_ENCRYPTION_TAG_HEADER.to_string(),
|
||||
"x-rustfs-encryption-tag".to_string(),
|
||||
base64::engine::general_purpose::STANDARD.encode(tag),
|
||||
);
|
||||
}
|
||||
|
||||
headers.insert(
|
||||
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
|
||||
"x-rustfs-encryption-key".to_string(),
|
||||
base64::engine::general_purpose::STANDARD.encode(&metadata.encrypted_data_key),
|
||||
);
|
||||
|
||||
@@ -824,7 +831,7 @@ impl ObjectEncryptionService {
|
||||
None => context_aad(&metadata.encryption_context).unwrap_or_default(),
|
||||
};
|
||||
headers.insert(
|
||||
INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(),
|
||||
"x-rustfs-encryption-context".to_string(),
|
||||
String::from_utf8_lossy(&context_bytes).into_owned(),
|
||||
);
|
||||
|
||||
@@ -869,13 +876,13 @@ impl ObjectEncryptionService {
|
||||
};
|
||||
|
||||
let iv = headers
|
||||
.get(INTERNAL_ENCRYPTION_IV_HEADER)
|
||||
.get("x-rustfs-encryption-iv")
|
||||
.ok_or_else(|| KmsError::validation_error("Missing IV header"))?;
|
||||
let iv = base64::engine::general_purpose::STANDARD
|
||||
.decode(iv)
|
||||
.map_err(|e| KmsError::validation_error(format!("Invalid IV: {e}")))?;
|
||||
|
||||
let tag = if let Some(tag_str) = headers.get(INTERNAL_ENCRYPTION_TAG_HEADER) {
|
||||
let tag = if let Some(tag_str) = headers.get("x-rustfs-encryption-tag") {
|
||||
Some(
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(tag_str)
|
||||
@@ -885,7 +892,7 @@ impl ObjectEncryptionService {
|
||||
None
|
||||
};
|
||||
|
||||
let encrypted_data_key = if let Some(key_str) = headers.get(INTERNAL_ENCRYPTION_KEY_HEADER) {
|
||||
let encrypted_data_key = if let Some(key_str) = headers.get("x-rustfs-encryption-key") {
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(key_str)
|
||||
.map_err(|e| KmsError::validation_error(format!("Invalid encrypted key: {e}")))?
|
||||
@@ -897,7 +904,7 @@ impl ObjectEncryptionService {
|
||||
// callers that inspect the context, but the bytes are carried through
|
||||
// untouched: re-serializing the parsed map is exactly how the original
|
||||
// ordering — and with it the ability to open the object — was lost.
|
||||
let (encryption_context, context_aad) = match headers.get(INTERNAL_ENCRYPTION_CONTEXT_HEADER) {
|
||||
let (encryption_context, context_aad) = match headers.get("x-rustfs-encryption-context") {
|
||||
Some(context_str) => (
|
||||
serde_json::from_str(context_str)
|
||||
.map_err(|e| KmsError::validation_error(format!("Invalid encryption context: {e}")))?,
|
||||
|
||||
@@ -32,14 +32,14 @@ use std::collections::HashMap;
|
||||
use std::fmt::Debug;
|
||||
use std::future::Future;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::Arc;
|
||||
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use rustfs_kms::backends::BackendCapabilities;
|
||||
use rustfs_kms::{
|
||||
CreateKeyRequest, DeleteKeyRequest, KeyUsage, KmsConfig, KmsError, KmsManager, KmsServiceManager, KmsServiceStatus,
|
||||
ObjectEncryptionService, Result,
|
||||
CreateKeyRequest, KeyUsage, KmsConfig, KmsError, KmsManager, KmsServiceManager, KmsServiceStatus, ObjectEncryptionService,
|
||||
Result,
|
||||
};
|
||||
use tempfile::TempDir;
|
||||
|
||||
@@ -126,12 +126,6 @@ pub struct TestKms {
|
||||
manager: Arc<KmsServiceManager>,
|
||||
kind: BackendKind,
|
||||
config: KmsConfig,
|
||||
/// Ids of the keys [`TestKms::create_key`] created, so a run against a
|
||||
/// persistent Vault can remove them afterwards instead of accumulating
|
||||
/// `behavior-*` keys forever (rustfs/backlog#1774). Shared through an Arc
|
||||
/// because the harness instance is consumed by the spec while the cleanup
|
||||
/// runs after it.
|
||||
created_keys: Arc<Mutex<Vec<String>>>,
|
||||
/// Held for the harness lifetime so the local key directory outlives a
|
||||
/// simulated process restart.
|
||||
_dir: Option<TempDir>,
|
||||
@@ -153,7 +147,6 @@ impl TestKms {
|
||||
manager,
|
||||
kind: BackendKind::Local,
|
||||
config,
|
||||
created_keys: Arc::new(Mutex::new(Vec::new())),
|
||||
_dir: Some(dir),
|
||||
}
|
||||
}
|
||||
@@ -173,7 +166,6 @@ impl TestKms {
|
||||
manager,
|
||||
kind: BackendKind::VaultKv2,
|
||||
config,
|
||||
created_keys: Arc::new(Mutex::new(Vec::new())),
|
||||
_dir: None,
|
||||
}
|
||||
}
|
||||
@@ -188,7 +180,6 @@ impl TestKms {
|
||||
manager,
|
||||
kind: BackendKind::VaultTransit,
|
||||
config,
|
||||
created_keys: Arc::new(Mutex::new(Vec::new())),
|
||||
_dir: None,
|
||||
}
|
||||
}
|
||||
@@ -201,7 +192,6 @@ impl TestKms {
|
||||
manager,
|
||||
kind: BackendKind::Static,
|
||||
config,
|
||||
created_keys: Arc::new(Mutex::new(Vec::new())),
|
||||
_dir: None,
|
||||
}
|
||||
}
|
||||
@@ -263,75 +253,8 @@ impl TestKms {
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("create_key({name}) should succeed on {}: {error:?}", self.kind.name()));
|
||||
assert_eq!(response.key_id, name, "created key id must be the requested name");
|
||||
self.created_keys
|
||||
.lock()
|
||||
.expect("created-keys lock")
|
||||
.push(response.key_id.clone());
|
||||
response.key_id
|
||||
}
|
||||
|
||||
/// Handle to the ids [`Self::create_key`] recorded, for cleanup that runs
|
||||
/// after a spec consumed the harness instance.
|
||||
pub fn created_keys_handle(&self) -> Arc<Mutex<Vec<String>>> {
|
||||
Arc::clone(&self.created_keys)
|
||||
}
|
||||
|
||||
/// Remove this instance's recorded Vault keys; see [`cleanup_vault_keys`].
|
||||
pub async fn cleanup(&self) {
|
||||
cleanup_vault_keys(self.kind, &self.config, self.created_keys_handle()).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Best-effort removal of the Vault keys a harness instance created, so a
|
||||
/// persistent dev Vault does not accumulate `behavior-*` keys across runs
|
||||
/// (rustfs/backlog#1774). A no-op for the Local and Static backends, whose
|
||||
/// state dies with the per-test temp directory.
|
||||
///
|
||||
/// The deletion runs on a fresh manager over the same configuration — the
|
||||
/// case's own manager is consumed by the spec and may have been stopped by a
|
||||
/// restart scenario — with the immediate-deletion gate enabled on the cleanup
|
||||
/// configuration only, so the configuration under test keeps the gate at its
|
||||
/// production default and specs asserting the gate's refusal stay honest.
|
||||
///
|
||||
/// Failures are reported but never panic: cleanup runs after the spec's own
|
||||
/// assertions, and a Vault hiccup here must not turn a green behavior run red.
|
||||
pub async fn cleanup_vault_keys(kind: BackendKind, config: &KmsConfig, created_keys: Arc<Mutex<Vec<String>>>) {
|
||||
if !kind.is_vault() {
|
||||
return;
|
||||
}
|
||||
let key_ids: Vec<String> = created_keys.lock().expect("created-keys lock").drain(..).collect();
|
||||
if key_ids.is_empty() {
|
||||
return;
|
||||
}
|
||||
let config = config.clone().with_immediate_deletion_allowed();
|
||||
let manager = start_manager(&config).await;
|
||||
let kms = manager.get_manager().await.expect("KMS manager should be running");
|
||||
for key_id in key_ids {
|
||||
// The Transit backend deletes in two steps (first call parks the key in
|
||||
// PendingDeletion, the next call destroys it); KV2 destroys on the
|
||||
// first call and reports KeyNotFound on the second.
|
||||
for _ in 0..2 {
|
||||
match kms
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
pending_window_in_days: None,
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: Some(key_id.clone()),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => continue,
|
||||
Err(KmsError::KeyNotFound { .. }) => break,
|
||||
Err(error) => {
|
||||
eprintln!("vault key cleanup: could not delete {key_id}: {error:?}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Err(error) = manager.stop().await {
|
||||
eprintln!("vault key cleanup: could not stop the cleanup manager: {error:?}");
|
||||
}
|
||||
}
|
||||
|
||||
async fn start_manager(config: &KmsConfig) -> Arc<KmsServiceManager> {
|
||||
@@ -409,12 +332,7 @@ where
|
||||
.chain(live_vault_backends());
|
||||
for kind in kinds {
|
||||
let case = BackendCase::new(kind).await;
|
||||
// Captured before the spec consumes the case; keys the spec creates
|
||||
// through the harness afterwards still land in the shared list.
|
||||
let config = case.kms.config().clone();
|
||||
let created_keys = case.kms.created_keys_handle();
|
||||
spec(case).await;
|
||||
cleanup_vault_keys(kind, &config, created_keys).await;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ thiserror.workspace = true
|
||||
parking_lot.workspace = true
|
||||
rand.workspace = true
|
||||
smallvec = { workspace = true, features = ["serde"] }
|
||||
compact_str.workspace = true
|
||||
smartstring.workspace = true
|
||||
crossbeam-queue = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -40,7 +40,6 @@ const UNLOCK_RETRY_ATTEMPTS: usize = 3;
|
||||
const UNLOCK_RETRY_BACKOFF: Duration = Duration::from_millis(100);
|
||||
const LOCK_ACQUIRE_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
|
||||
const LOCK_ACQUIRE_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(1);
|
||||
const LOCK_ACQUIRE_SPARE_HEDGES: usize = 1;
|
||||
const REMOTE_LOCK_RPC_FAILED_PREFIX: &str = "remote lock rpc failed:";
|
||||
const REMOTE_LOCK_RPC_TIMED_OUT_PREFIX: &str = "remote lock rpc timed out:";
|
||||
const UNRECOVERABLE_QUORUM_FAILURE_PREFIX: &str = "unrecoverable quorum failure";
|
||||
@@ -479,7 +478,7 @@ pub struct DistributedLock {
|
||||
/// Lock clients for this namespace
|
||||
clients: Vec<Arc<dyn LockClient>>,
|
||||
/// Namespace identifier
|
||||
namespace: Arc<str>,
|
||||
namespace: String,
|
||||
/// Quorum size for exclusive/write operations
|
||||
quorum: usize,
|
||||
}
|
||||
@@ -496,11 +495,6 @@ struct LockAcquireQuorumResult {
|
||||
impl DistributedLock {
|
||||
/// Create new distributed lock
|
||||
pub fn new(namespace: String, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
|
||||
Self::new_shared(namespace.into(), clients, quorum)
|
||||
}
|
||||
|
||||
/// Create a distributed lock that shares an existing namespace allocation.
|
||||
pub(crate) fn new_shared(namespace: Arc<str>, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
|
||||
let q = if clients.len() <= 1 {
|
||||
1
|
||||
} else {
|
||||
@@ -678,22 +672,13 @@ impl DistributedLock {
|
||||
self.acquire_guard(&req).await
|
||||
}
|
||||
|
||||
fn spawn_lock_requests_until(
|
||||
&self,
|
||||
pending: &mut JoinSet<LockAcquireTaskResult>,
|
||||
request: &LockRequest,
|
||||
next_client_index: &mut usize,
|
||||
target_pending: usize,
|
||||
) {
|
||||
while pending.len() < target_pending {
|
||||
let Some(client) = self.clients.get(*next_client_index).cloned() else {
|
||||
break;
|
||||
};
|
||||
let idx = *next_client_index;
|
||||
*next_client_index = (*next_client_index).saturating_add(1);
|
||||
fn spawn_lock_requests(&self, request: &LockRequest) -> JoinSet<LockAcquireTaskResult> {
|
||||
let mut pending = JoinSet::new();
|
||||
for (idx, client) in self.clients.iter().cloned().enumerate() {
|
||||
let request = request.clone();
|
||||
pending.spawn(async move { (idx, client.acquire_lock(&request).await) });
|
||||
}
|
||||
pending
|
||||
}
|
||||
|
||||
fn lock_acquire_retry_backoff(attempt: usize, rng: &mut impl rand::Rng) -> Duration {
|
||||
@@ -926,16 +911,7 @@ impl DistributedLock {
|
||||
async fn acquire_lock_quorum_once(&self, request: &LockRequest) -> Result<LockAcquireQuorumResult> {
|
||||
let required_quorum = self.required_quorum(request.lock_type);
|
||||
let attempt_started = Instant::now();
|
||||
let mut pending = JoinSet::new();
|
||||
let mut next_client_index = 0usize;
|
||||
self.spawn_lock_requests_until(
|
||||
&mut pending,
|
||||
request,
|
||||
&mut next_client_index,
|
||||
required_quorum
|
||||
.saturating_add(LOCK_ACQUIRE_SPARE_HEDGES)
|
||||
.min(self.clients.len()),
|
||||
);
|
||||
let mut pending = self.spawn_lock_requests(request);
|
||||
let mut individual_locks: Vec<HeldLockEntry> = Vec::new();
|
||||
let fallback_lock_id = request.lock_id.clone();
|
||||
let mut last_failure = None;
|
||||
@@ -1109,15 +1085,7 @@ impl DistributedLock {
|
||||
});
|
||||
}
|
||||
|
||||
let needed = required_quorum.saturating_sub(individual_locks.len());
|
||||
self.spawn_lock_requests_until(
|
||||
&mut pending,
|
||||
request,
|
||||
&mut next_client_index,
|
||||
needed.saturating_add(LOCK_ACQUIRE_SPARE_HEDGES),
|
||||
);
|
||||
let unstarted_clients = self.clients.len().saturating_sub(next_client_index);
|
||||
if individual_locks.len() + pending.len() + unstarted_clients < required_quorum {
|
||||
if individual_locks.len() + pending.len() < required_quorum {
|
||||
let rollback_count = individual_locks.len();
|
||||
Self::spawn_release_cleanup(
|
||||
individual_locks.iter().map(HeldLockEntry::release_entry).collect(),
|
||||
@@ -1939,99 +1907,6 @@ mod tests {
|
||||
assert!(!should_warn_lock_failure("Lock acquisition timeout"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shared_lock_acquire_starts_quorum_plus_one_clients() {
|
||||
let seen_ids = (0..4).map(|_| Arc::new(Mutex::new(Vec::new()))).collect::<Vec<_>>();
|
||||
let clients: Vec<Arc<dyn LockClient>> = vec![
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Success {
|
||||
delay: Duration::from_millis(20),
|
||||
}],
|
||||
seen_ids[0].clone(),
|
||||
)),
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Success {
|
||||
delay: Duration::from_millis(20),
|
||||
}],
|
||||
seen_ids[1].clone(),
|
||||
)),
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Success {
|
||||
delay: Duration::from_millis(200),
|
||||
}],
|
||||
seen_ids[2].clone(),
|
||||
)),
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Success { delay: Duration::ZERO }],
|
||||
seen_ids[3].clone(),
|
||||
)),
|
||||
];
|
||||
let lock = DistributedLock::new("test".to_string(), clients, 3);
|
||||
let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Shared, "owner")
|
||||
.with_acquire_timeout(Duration::from_secs(1));
|
||||
|
||||
let guard = lock
|
||||
.acquire_guard(&request)
|
||||
.await
|
||||
.expect("shared lock acquisition should not error")
|
||||
.expect("read quorum should be reached");
|
||||
|
||||
assert_eq!(guard.entries.len(), 2, "read quorum should track exactly two acquired entries");
|
||||
assert_eq!(
|
||||
seen_ids[3].lock().unwrap().len(),
|
||||
0usize,
|
||||
"fourth client should stay unstarted while one hedge is pending"
|
||||
);
|
||||
drop(guard);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shared_lock_acquire_replenishes_after_early_failure() {
|
||||
let seen_ids = (0..4).map(|_| Arc::new(Mutex::new(Vec::new()))).collect::<Vec<_>>();
|
||||
let clients: Vec<Arc<dyn LockClient>> = vec![
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Failure {
|
||||
error: "lock already held",
|
||||
delay: Duration::ZERO,
|
||||
}],
|
||||
seen_ids[0].clone(),
|
||||
)),
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Success {
|
||||
delay: Duration::from_millis(80),
|
||||
}],
|
||||
seen_ids[1].clone(),
|
||||
)),
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Success {
|
||||
delay: Duration::from_millis(120),
|
||||
}],
|
||||
seen_ids[2].clone(),
|
||||
)),
|
||||
Arc::new(SequencedClient::new(
|
||||
vec![AcquirePlan::Success { delay: Duration::ZERO }],
|
||||
seen_ids[3].clone(),
|
||||
)),
|
||||
];
|
||||
let lock = DistributedLock::new("test".to_string(), clients, 3);
|
||||
let request = LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Shared, "owner")
|
||||
.with_acquire_timeout(Duration::from_secs(1));
|
||||
|
||||
let guard = lock
|
||||
.acquire_guard(&request)
|
||||
.await
|
||||
.expect("shared lock acquisition should not error")
|
||||
.expect("read quorum should be reached after replenishing the failed slot");
|
||||
|
||||
assert_eq!(guard.entries.len(), 2, "read quorum should be reached");
|
||||
assert_eq!(
|
||||
seen_ids[3].lock().unwrap().len(),
|
||||
1usize,
|
||||
"fourth client should be started after an early failure"
|
||||
);
|
||||
drop(guard);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acquire_guard_returns_quorum_error_when_rpc_failures_make_quorum_impossible() {
|
||||
let clients: Vec<Arc<dyn LockClient>> = vec![
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::fast_lock::guard::FastLockGuard;
|
||||
use compact_str::CompactString;
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use smartstring::SmartString;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
@@ -143,15 +143,15 @@ impl ObjectKey {
|
||||
}
|
||||
}
|
||||
|
||||
/// Optimized object key using compact strings for better performance
|
||||
/// Optimized object key using smart strings for better performance
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OptimizedObjectKey {
|
||||
/// Bucket name - uses inline storage for small strings
|
||||
pub bucket: CompactString,
|
||||
pub bucket: SmartString<smartstring::LazyCompact>,
|
||||
/// Object name - uses inline storage for small strings
|
||||
pub object: CompactString,
|
||||
pub object: SmartString<smartstring::LazyCompact>,
|
||||
/// Version - optional for latest version semantics
|
||||
pub version: Option<CompactString>,
|
||||
pub version: Option<SmartString<smartstring::LazyCompact>>,
|
||||
/// Cached hash to avoid recomputation
|
||||
hash_cache: OnceLock<u64>,
|
||||
}
|
||||
@@ -189,7 +189,10 @@ impl Ord for OptimizedObjectKey {
|
||||
}
|
||||
|
||||
impl OptimizedObjectKey {
|
||||
pub fn new(bucket: impl Into<CompactString>, object: impl Into<CompactString>) -> Self {
|
||||
pub fn new(
|
||||
bucket: impl Into<SmartString<smartstring::LazyCompact>>,
|
||||
object: impl Into<SmartString<smartstring::LazyCompact>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
bucket: bucket.into(),
|
||||
object: object.into(),
|
||||
@@ -199,9 +202,9 @@ impl OptimizedObjectKey {
|
||||
}
|
||||
|
||||
pub fn with_version(
|
||||
bucket: impl Into<CompactString>,
|
||||
object: impl Into<CompactString>,
|
||||
version: impl Into<CompactString>,
|
||||
bucket: impl Into<SmartString<smartstring::LazyCompact>>,
|
||||
object: impl Into<SmartString<smartstring::LazyCompact>>,
|
||||
version: impl Into<SmartString<smartstring::LazyCompact>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
bucket: bucket.into(),
|
||||
@@ -229,9 +232,9 @@ impl OptimizedObjectKey {
|
||||
/// Convert from regular ObjectKey
|
||||
pub fn from_object_key(key: &ObjectKey) -> Self {
|
||||
Self {
|
||||
bucket: CompactString::from(key.bucket.as_ref()),
|
||||
object: CompactString::from(key.object.as_ref()),
|
||||
version: key.version.as_ref().map(|v| CompactString::from(v.as_ref())),
|
||||
bucket: SmartString::from(key.bucket.as_ref()),
|
||||
object: SmartString::from(key.object.as_ref()),
|
||||
version: key.version.as_ref().map(|v| SmartString::from(v.as_ref())),
|
||||
hash_cache: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,17 +28,12 @@ pub struct LocalLock {
|
||||
/// Global lock manager for fast local locks
|
||||
manager: Arc<GlobalLockManager>,
|
||||
/// Namespace identifier
|
||||
namespace: Arc<str>,
|
||||
namespace: String,
|
||||
}
|
||||
|
||||
impl LocalLock {
|
||||
/// Create new local lock
|
||||
pub fn new(namespace: String, manager: Arc<GlobalLockManager>) -> Self {
|
||||
Self::new_shared(namespace.into(), manager)
|
||||
}
|
||||
|
||||
/// Create a local lock that shares an existing namespace allocation.
|
||||
pub(crate) fn new_shared(namespace: Arc<str>, manager: Arc<GlobalLockManager>) -> Self {
|
||||
Self { namespace, manager }
|
||||
}
|
||||
|
||||
|
||||
@@ -180,11 +180,6 @@ impl NamespaceLock {
|
||||
Self::Local(LocalLock::new(namespace, manager))
|
||||
}
|
||||
|
||||
/// Create a local namespace lock that shares an existing namespace allocation.
|
||||
pub fn with_local_manager_shared(namespace: Arc<str>, manager: Arc<crate::GlobalLockManager>) -> Self {
|
||||
Self::Local(LocalLock::new_shared(namespace, manager))
|
||||
}
|
||||
|
||||
/// Create namespace lock with clients
|
||||
/// Uses DistributedLock with appropriate quorum
|
||||
pub fn with_clients(namespace: String, clients: Vec<Arc<dyn LockClient>>) -> Self {
|
||||
@@ -200,11 +195,6 @@ impl NamespaceLock {
|
||||
Self::Distributed(DistributedLock::new(namespace, clients, quorum))
|
||||
}
|
||||
|
||||
/// Create a namespace lock that shares an existing namespace allocation.
|
||||
pub fn with_clients_and_quorum_shared(namespace: Arc<str>, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
|
||||
Self::Distributed(DistributedLock::new_shared(namespace, clients, quorum))
|
||||
}
|
||||
|
||||
/// Get namespace identifier
|
||||
pub fn namespace(&self) -> &str {
|
||||
match self {
|
||||
|
||||
@@ -356,16 +356,6 @@ async fn test_namespace_lock_with_local_manager() {
|
||||
assert_eq!(lock.namespace(), "local-ns");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_lock_preserves_shared_namespace_storage() {
|
||||
let namespace: Arc<str> = Arc::from("shared-namespace");
|
||||
let namespace_ptr = Arc::as_ptr(&namespace);
|
||||
let local = LocalLock::new_shared(namespace.clone(), Arc::new(GlobalLockManager::new()));
|
||||
|
||||
assert_eq!(local.namespace(), namespace.as_ref());
|
||||
assert_eq!(local.namespace().as_ptr(), namespace_ptr.cast::<u8>());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_with_clients() {
|
||||
let clients = vec![ClientFactory::create_local(), ClientFactory::create_local()];
|
||||
|
||||
@@ -113,7 +113,7 @@ pub struct ScannerStats {
|
||||
pub current_cycle_usage_saves: u64,
|
||||
/// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan
|
||||
pub current_scan_mode: u64,
|
||||
/// Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded, 5 deferred
|
||||
/// Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded
|
||||
pub last_cycle_result: u64,
|
||||
/// Last scanner partial cycle reason: 0 unknown, 1 runtime, 2 objects, 3 directories
|
||||
pub last_cycle_partial_reason: u64,
|
||||
|
||||
@@ -460,7 +460,7 @@ pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock<MetricDescriptor> = LazyLock::
|
||||
pub static SCANNER_LAST_CYCLE_RESULT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ScannerLastCycleResult,
|
||||
"Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded, 5 deferred.",
|
||||
"Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded.",
|
||||
&[],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
|
||||
@@ -355,6 +355,37 @@ pub enum S3Action {
|
||||
GetBucketQuotaAction,
|
||||
}
|
||||
|
||||
// #[derive(Serialize, Deserialize, Hash, PartialEq, Eq, Clone, EnumString, IntoStaticStr, Debug, Copy)]
|
||||
// #[serde(try_from = "&str", into = "&str")]
|
||||
// pub enum AdminAction {
|
||||
// #[strum(serialize = "admin:*")]
|
||||
// AllActions,
|
||||
// #[strum(serialize = "admin:Profiling")]
|
||||
// ProfilingAdminAction,
|
||||
// #[strum(serialize = "admin:ServerTrace")]
|
||||
// TraceAdminAction,
|
||||
// #[strum(serialize = "admin:ConsoleLog")]
|
||||
// ConsoleLogAdminAction,
|
||||
// #[strum(serialize = "admin:ServerInfo")]
|
||||
// ServerInfoAdminAction,
|
||||
// #[strum(serialize = "admin:OBDInfo")]
|
||||
// HealthInfoAdminAction,
|
||||
// #[strum(serialize = "admin:TopLocksInfo")]
|
||||
// TopLocksAdminAction,
|
||||
// #[strum(serialize = "admin:LicenseInfo")]
|
||||
// LicenseInfoAdminAction,
|
||||
// #[strum(serialize = "admin:BandwidthMonitor")]
|
||||
// BandwidthMonitorAction,
|
||||
// #[strum(serialize = "admin:InspectData")]
|
||||
// InspectDataAction,
|
||||
// #[strum(serialize = "admin:Prometheus")]
|
||||
// PrometheusAdminAction,
|
||||
// #[strum(serialize = "admin:ListServiceAccounts")]
|
||||
// ListServiceAccountsAdminAction,
|
||||
// #[strum(serialize = "admin:CreateServiceAccount")]
|
||||
// CreateServiceAccountAdminAction,
|
||||
// }
|
||||
|
||||
// AdminAction - admin policy action.
|
||||
#[derive(Serialize, Deserialize, Hash, PartialEq, Eq, Clone, IntoStaticStr, Debug, Copy, EnumString)]
|
||||
#[serde(try_from = "&str", into = "&str")]
|
||||
|
||||
@@ -53,4 +53,16 @@ mod tests {
|
||||
|
||||
assert!(!token.is_empty());
|
||||
}
|
||||
|
||||
// #[test]
|
||||
// fn test_extract_claims() {
|
||||
// let claims = Claims {
|
||||
// sub: "user1".to_string(),
|
||||
// company: "example".to_string(),
|
||||
// };
|
||||
// let secret = "my_secret";
|
||||
// let token = generate_jwt(&claims, secret).unwrap();
|
||||
// let decoded_claims = extract_claims::<Claims>(&token, secret).unwrap();
|
||||
// assert_eq!(decoded_claims.claims, claims);
|
||||
// }
|
||||
}
|
||||
|
||||
@@ -2513,7 +2513,7 @@ mod tests {
|
||||
json_field: "rename_data_resp",
|
||||
bin_field: "rename_data_resp_bin",
|
||||
},
|
||||
json_encoder: "let rename_data_resp_json = compat_response_json(rename_data_resp, request_decoded_from_msgpack)",
|
||||
json_encoder: "let rename_data_resp_json = compat_response_json(&rename_data_resp, false);",
|
||||
},
|
||||
];
|
||||
|
||||
|
||||
@@ -12,26 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! DELIBERATE DUPLICATION — do not merge these declarations into the
|
||||
//! rustfs-utils http module without a maintainer decision on the crate
|
||||
//! boundary.
|
||||
//!
|
||||
//! The canonical owners of these interop-contract values live in the
|
||||
//! rustfs-utils crate: `crates/utils/src/http/metadata_compat.rs` (dual
|
||||
//! x-rustfs-internal-/x-minio-internal- metadata keys),
|
||||
//! `crates/utils/src/http/header_compat.rs` (x-rustfs-/x-minio- header pairs),
|
||||
//! and `crates/utils/src/http/headers.rs` (standard S3 header names). This
|
||||
//! crate keeps a local copy because
|
||||
//! `rustfs-replication` is a wire-contract crate that must stay free of
|
||||
//! internal dependencies: `scripts/check_architecture_migration_rules.sh`
|
||||
//! rejects any `rustfs-utils` import or dependency here ("replication crate
|
||||
//! HTTP/helper contracts must not import or depend on rustfs-utils"), and the
|
||||
//! same rule bans `rustfs-filemeta` and `rustfs-storage-api`.
|
||||
//!
|
||||
//! Drift protection lives in the test module below: every constant's literal
|
||||
//! wire value is pinned, so a change on either side that breaks interop fails
|
||||
//! this crate's tests rather than silently forking the contract.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
const RUSTFS_INTERNAL_PREFIX: &str = "x-rustfs-internal-";
|
||||
@@ -165,31 +145,4 @@ mod tests {
|
||||
assert!(has_prefix_fold("X-Amz-Meta-Foo", "x-amz-meta-"));
|
||||
assert!(!has_prefix_fold("X-Amz-Meta-Foo", "amz-meta"));
|
||||
}
|
||||
|
||||
/// Pins every duplicated interop constant to its literal wire value. The
|
||||
/// canonical owner lives in the rustfs-utils crate (see the module doc);
|
||||
/// an arch guard forbids depending on it from this crate, so byte-for-byte
|
||||
/// pinning here is what keeps the two copies from drifting apart.
|
||||
#[test]
|
||||
fn duplicated_interop_constants_pin_canonical_wire_values() {
|
||||
use super::*;
|
||||
|
||||
assert_eq!(AMZ_BUCKET_REPLICATION_STATUS, "X-Amz-Replication-Status");
|
||||
assert_eq!(AMZ_OBJECT_LOCK_LEGAL_HOLD, "X-Amz-Object-Lock-Legal-Hold");
|
||||
assert_eq!(AMZ_OBJECT_LOCK_MODE, "X-Amz-Object-Lock-Mode");
|
||||
assert_eq!(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, "X-Amz-Object-Lock-Retain-Until-Date");
|
||||
assert_eq!(AMZ_OBJECT_TAGGING, "X-Amz-Tagging");
|
||||
assert_eq!(AMZ_WEBSITE_REDIRECT_LOCATION, "x-amz-website-redirect-location");
|
||||
assert_eq!(CACHE_CONTROL, "Cache-Control");
|
||||
assert_eq!(CONTENT_DISPOSITION, "Content-Disposition");
|
||||
assert_eq!(CONTENT_ENCODING, "Content-Encoding");
|
||||
assert_eq!(CONTENT_LANGUAGE, "Content-Language");
|
||||
assert_eq!(EXPIRES, "Expires");
|
||||
assert_eq!(SSEC_ALGORITHM_HEADER, "x-amz-server-side-encryption-customer-algorithm");
|
||||
assert_eq!(SSEC_KEY_HEADER, "x-amz-server-side-encryption-customer-key");
|
||||
assert_eq!(SSEC_KEY_MD5_HEADER, "x-amz-server-side-encryption-customer-key-md5");
|
||||
assert_eq!(SUFFIX_ACTUAL_SIZE, "actual-size");
|
||||
assert_eq!(SUFFIX_REPLICATION_STATUS, "replication-status");
|
||||
assert_eq!(SUFFIX_REPLICATION_RESET_STATUS, "replication-reset-status");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2403,150 +2403,6 @@ mod tests {
|
||||
assert_eq!(decoded.cache.get("bucket").map(|entry| entry.objects), Some(3));
|
||||
}
|
||||
|
||||
/// Deterministic, fully populated cache used to pin the persisted
|
||||
/// `.usage-cache.bin` wire bytes. Every map/set holds at most one element
|
||||
/// so the map-encoded `marshal_msg` output is byte-stable.
|
||||
fn wire_fixture_cache() -> DataUsageCache {
|
||||
let mut entry = DataUsageEntry {
|
||||
size: 4096,
|
||||
objects: 3,
|
||||
versions: 5,
|
||||
delete_markers: 1,
|
||||
compacted: true,
|
||||
failed_objects: 2,
|
||||
..Default::default()
|
||||
};
|
||||
entry.add_tier_sizes(&HashMap::from([(
|
||||
"WARM".to_string(),
|
||||
TierStats {
|
||||
total_size: 2048,
|
||||
num_versions: 2,
|
||||
num_objects: 1,
|
||||
},
|
||||
)]));
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "wire-bucket".to_string(),
|
||||
next_cycle: 7,
|
||||
leader_epoch: 9,
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000)),
|
||||
skip_healing: true,
|
||||
failed_objects: HashMap::from([("wire-bucket/lost".to_string(), 11)]),
|
||||
scan_resume_after: Some("wire-bucket/resume".to_string()),
|
||||
pending_heals: vec![PendingScannerHeal {
|
||||
kind: PendingScannerHealKind::Object,
|
||||
bucket: "wire-bucket".to_string(),
|
||||
object: Some("broken".to_string()),
|
||||
version_id: None,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
first_seen: 100,
|
||||
last_attempt: 200,
|
||||
attempts: 3,
|
||||
last_admission_result: "deferred".to_string(),
|
||||
last_admission_reason: "budget".to_string(),
|
||||
}],
|
||||
source: Some(DataUsageCacheSource::new(1, 2)),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.replace("wire-bucket", "", entry);
|
||||
cache
|
||||
}
|
||||
|
||||
/// Persisted `.usage-cache.bin` bytes produced by [`wire_fixture_cache`]
|
||||
/// via `DataUsageCache::marshal_msg`: a 2-element array of the 16-field
|
||||
/// map-encoded info block and the map of map-encoded entries.
|
||||
///
|
||||
/// The thin read-only projection in `crates/data-usage` decodes a copy of
|
||||
/// this fixture (`thin_usage_cache_decodes_scanner_wire_fixture`); when
|
||||
/// the encoding legitimately changes, regenerate both copies from
|
||||
/// `wire_fixture_cache().marshal_msg()` and re-verify old readers.
|
||||
const USAGE_CACHE_WIRE_FIXTURE: &[u8] = &[
|
||||
0x92, 0xde, 0x00, 0x10, 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
||||
0x74, 0xaa, 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x07, 0xac, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72,
|
||||
0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x09, 0xab, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x92,
|
||||
0xce, 0x65, 0x53, 0xf1, 0x00, 0x00, 0xac, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0xc3,
|
||||
0xa9, 0x6c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0xc0, 0xab, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74,
|
||||
0x69, 0x6f, 0x6e, 0xc0, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x81,
|
||||
0xb0, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2f, 0x6c, 0x6f, 0x73, 0x74, 0x0b, 0xb1, 0x73,
|
||||
0x63, 0x61, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72, 0xb2, 0x77, 0x69, 0x72,
|
||||
0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x2f, 0x72, 0x65, 0x73, 0x75, 0x6d, 0x65, 0xaf, 0x73, 0x63, 0x61, 0x6e,
|
||||
0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0xc0, 0xad, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67,
|
||||
0x5f, 0x68, 0x65, 0x61, 0x6c, 0x73, 0x91, 0x9a, 0xa6, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0xab, 0x77, 0x69, 0x72, 0x65,
|
||||
0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0xa6, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x6e, 0xc0, 0x01, 0x64, 0xcc, 0xc8, 0x03,
|
||||
0xa8, 0x64, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x64, 0xa6, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0xab, 0x6f, 0x62, 0x6a,
|
||||
0x65, 0x63, 0x74, 0x5f, 0x6c, 0x6f, 0x63, 0x6b, 0xc0, 0xa6, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x92, 0x01, 0x02, 0xb1,
|
||||
0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0xc3, 0xb0, 0x73,
|
||||
0x63, 0x61, 0x6e, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0xdc, 0x00, 0x20, 0x03, 0x03,
|
||||
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03,
|
||||
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0xb0, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6b, 0x65, 0x79,
|
||||
0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x01, 0x81, 0xab, 0x77, 0x69, 0x72, 0x65, 0x2d, 0x62, 0x75, 0x63, 0x6b, 0x65,
|
||||
0x74, 0x8b, 0xa8, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x90, 0xa4, 0x73, 0x69, 0x7a, 0x65, 0xcd, 0x10, 0x00,
|
||||
0xa7, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x03, 0xa8, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x05, 0xae,
|
||||
0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x01, 0xa9, 0x6f, 0x62, 0x6a, 0x5f,
|
||||
0x73, 0x69, 0x7a, 0x65, 0x73, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xac, 0x6f, 0x62,
|
||||
0x6a, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb1, 0x72,
|
||||
0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0xc0, 0xa9, 0x63, 0x6f,
|
||||
0x6d, 0x70, 0x61, 0x63, 0x74, 0x65, 0x64, 0xc3, 0xae, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x6f, 0x62, 0x6a, 0x65,
|
||||
0x63, 0x74, 0x73, 0x02, 0xae, 0x61, 0x6c, 0x6c, 0x5f, 0x74, 0x69, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x91,
|
||||
0x81, 0xa4, 0x57, 0x41, 0x52, 0x4d, 0x93, 0xcd, 0x08, 0x00, 0x02, 0x01,
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn usage_cache_wire_format_is_pinned() {
|
||||
// Writer: the canonical map-encoded serializer must reproduce the
|
||||
// pinned bytes. Round-trip tests cannot see format drift, so any
|
||||
// encoding change (field rename/reorder, map->array switch) fails
|
||||
// here and forces re-verifying old readers and the thin projection
|
||||
// in crates/data-usage before the fixture is regenerated.
|
||||
let encoded = wire_fixture_cache().marshal_msg().expect("marshal fixture cache");
|
||||
assert_eq!(
|
||||
encoded.as_slice(),
|
||||
USAGE_CACHE_WIRE_FIXTURE,
|
||||
"persisted .usage-cache.bin encoding drifted from the pinned fixture"
|
||||
);
|
||||
|
||||
// Reader: the pinned bytes decode with every field intact.
|
||||
let decoded = DataUsageCache::unmarshal(USAGE_CACHE_WIRE_FIXTURE).expect("decode pinned fixture");
|
||||
assert_eq!(decoded.info.name, "wire-bucket");
|
||||
assert_eq!(decoded.info.next_cycle, 7);
|
||||
assert_eq!(decoded.info.leader_epoch, 9);
|
||||
assert_eq!(
|
||||
decoded.info.last_update,
|
||||
Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000))
|
||||
);
|
||||
assert!(decoded.info.skip_healing);
|
||||
assert_eq!(decoded.info.failed_objects.get("wire-bucket/lost"), Some(&11));
|
||||
assert_eq!(decoded.info.scan_resume_after.as_deref(), Some("wire-bucket/resume"));
|
||||
assert_eq!(decoded.info.pending_heals.len(), 1);
|
||||
assert_eq!(decoded.info.pending_heals[0].kind, PendingScannerHealKind::Object);
|
||||
assert_eq!(decoded.info.pending_heals[0].object.as_deref(), Some("broken"));
|
||||
assert_eq!(decoded.info.source, Some(DataUsageCacheSource::new(1, 2)));
|
||||
assert!(decoded.info.snapshot_complete);
|
||||
assert_eq!(decoded.info.scan_plan_digest, Some(TEST_PLAN_DIGEST));
|
||||
assert_eq!(decoded.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
|
||||
|
||||
let entry = decoded.cache.get("wire-bucket").expect("fixture entry decodes");
|
||||
assert_eq!(entry.size, 4096);
|
||||
assert_eq!(entry.objects, 3);
|
||||
assert_eq!(entry.versions, 5);
|
||||
assert_eq!(entry.delete_markers, 1);
|
||||
assert!(entry.compacted);
|
||||
assert_eq!(entry.failed_objects, 2);
|
||||
assert_eq!(
|
||||
entry.all_tier_stats.as_ref().and_then(|tiers| tiers.tiers.get("WARM")),
|
||||
Some(&TierStats {
|
||||
total_size: 2048,
|
||||
num_versions: 2,
|
||||
num_objects: 1,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_usage_cache_prepare_for_scan_rejects_unscoped_distributed_cache() {
|
||||
let mut cache = DataUsageCache {
|
||||
|
||||
+42
-106
@@ -30,8 +30,8 @@ use crate::runtime_config::{
|
||||
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason};
|
||||
use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob};
|
||||
use crate::scanner_io::{
|
||||
ScannerCycleDeferReason, ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified, dirty_usage_buckets_pending,
|
||||
dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation,
|
||||
ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified, dirty_usage_buckets_pending, dirty_usage_generation,
|
||||
scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation,
|
||||
};
|
||||
use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed};
|
||||
use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGuard};
|
||||
@@ -41,8 +41,7 @@ use chrono::{DateTime, Utc};
|
||||
use rustfs_common::heal_channel::HealScanMode;
|
||||
use rustfs_common::metrics::{
|
||||
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource,
|
||||
emit_scan_cycle_complete, emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded,
|
||||
global_metrics,
|
||||
emit_scan_cycle_complete, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, global_metrics,
|
||||
};
|
||||
use rustfs_config::ScannerSpeed;
|
||||
#[cfg(test)]
|
||||
@@ -85,7 +84,7 @@ const METRIC_SCANNER_LEADER_LOCK_TOTAL: &str = "rustfs_scanner_leader_lock_total
|
||||
const CLEAN_IDLE_MAX_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
|
||||
const MAX_SCANNER_SCHEDULE_DELAY: Duration = Duration::from_secs(365 * 24 * 60 * 60);
|
||||
const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
|
||||
/// First-retry delay after a scanner cycle cannot publish authoritative usage.
|
||||
/// First-retry delay after a usage snapshot is superseded by concurrent writes.
|
||||
///
|
||||
/// A superseded cycle is the *expected* outcome of the dirty-usage fast path:
|
||||
/// a write burst marks buckets dirty, the scanner wakes within milliseconds,
|
||||
@@ -95,15 +94,14 @@ const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
|
||||
/// otherwise idle instance whose clean-idle backoff had doubled a 60 s
|
||||
/// interval), which defeats the fast path it is meant to protect.
|
||||
///
|
||||
/// The exponential growth in [`ScannerRetryBackoff::retry_interval`] is
|
||||
/// The exponential growth in [`ScannerSupersededBackoff::retry_interval`] is
|
||||
/// what protects against a persistently hot bucket driving an unbroken
|
||||
/// full-scan loop, so it can start small: 5 s, 10 s, 20 s … capped by
|
||||
/// [`SCANNER_RETRY_MAX_INTERVAL`]. A one-off race recovers in seconds; a
|
||||
/// [`SUPERSEDED_RETRY_MAX_INTERVAL`]. A one-off race recovers in seconds; a
|
||||
/// genuinely hot bucket still reaches minute-scale backoff within a handful of
|
||||
/// cycles. Preflight deferrals use the same bounded schedule so a temporarily
|
||||
/// unavailable peer cannot drive a tight retry loop.
|
||||
const SCANNER_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5);
|
||||
const SCANNER_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
||||
/// cycles.
|
||||
const SUPERSEDED_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5);
|
||||
const SUPERSEDED_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
||||
const SCANNER_LEADER_LOCK_POLL_INTERVAL: Duration = Duration::from_secs(1);
|
||||
#[cfg(not(test))]
|
||||
const SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
@@ -340,7 +338,6 @@ pub(crate) enum ScannerCycleOutcome {
|
||||
CompletedWithPendingMaintenance,
|
||||
Partial,
|
||||
Superseded,
|
||||
Deferred(ScannerCycleDeferReason),
|
||||
Failed,
|
||||
}
|
||||
|
||||
@@ -385,16 +382,22 @@ struct ScannerCleanIdleBackoff {
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
struct ScannerRetryBackoff {
|
||||
struct ScannerSupersededBackoff {
|
||||
consecutive_cycles: u32,
|
||||
}
|
||||
|
||||
impl ScannerRetryBackoff {
|
||||
fn record_retryable_cycle(&mut self, retryable: bool) {
|
||||
if retryable {
|
||||
self.consecutive_cycles = self.consecutive_cycles.saturating_add(1);
|
||||
} else {
|
||||
self.consecutive_cycles = 0;
|
||||
impl ScannerSupersededBackoff {
|
||||
fn record_cycle(&mut self, outcome: ScannerCycleOutcome) {
|
||||
match outcome {
|
||||
ScannerCycleOutcome::Superseded => {
|
||||
self.consecutive_cycles = self.consecutive_cycles.saturating_add(1);
|
||||
}
|
||||
ScannerCycleOutcome::Completed
|
||||
| ScannerCycleOutcome::CompletedWithPendingMaintenance
|
||||
| ScannerCycleOutcome::Partial
|
||||
| ScannerCycleOutcome::Failed => {
|
||||
self.consecutive_cycles = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -403,8 +406,8 @@ impl ScannerRetryBackoff {
|
||||
let multiplier = 1u32.checked_shl(exponent).unwrap_or(u32::MAX);
|
||||
let base_interval = configured_interval
|
||||
.max(Duration::from_secs(1))
|
||||
.min(SCANNER_RETRY_BASE_INTERVAL);
|
||||
let cap = SCANNER_RETRY_MAX_INTERVAL.max(configured_interval.max(Duration::from_secs(1)));
|
||||
.min(SUPERSEDED_RETRY_BASE_INTERVAL);
|
||||
let cap = SUPERSEDED_RETRY_MAX_INTERVAL.max(configured_interval.max(Duration::from_secs(1)));
|
||||
Some(base_interval.saturating_mul(multiplier).min(cap))
|
||||
}
|
||||
}
|
||||
@@ -3005,21 +3008,6 @@ async fn run_data_scanner_cycle(
|
||||
ScannerCycleOutcome::Failed
|
||||
};
|
||||
}
|
||||
ScannerCycleOutcome::Deferred(reason) => {
|
||||
info!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
reason = reason.as_str(),
|
||||
state = "deferred",
|
||||
"Scanner cycle deferred before usage scanning began"
|
||||
);
|
||||
emit_scan_cycle_deferred(cycle_start.elapsed());
|
||||
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
|
||||
return ScannerCycleOutcome::Deferred(reason);
|
||||
}
|
||||
ScannerCycleOutcome::Superseded => {
|
||||
info!(
|
||||
target: "rustfs::scanner",
|
||||
@@ -3213,8 +3201,7 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
let mut dirty_usage_generation_seen = dirty_usage_generation();
|
||||
let mut runtime_config_generation_seen = scanner_runtime_config_generation();
|
||||
let mut clean_idle_backoff = ScannerCleanIdleBackoff::default();
|
||||
let mut superseded_backoff = ScannerRetryBackoff::default();
|
||||
let mut deferred_backoff = ScannerRetryBackoff::default();
|
||||
let mut superseded_backoff = ScannerSupersededBackoff::default();
|
||||
let initial_runtime_config = resolve_scanner_runtime_config();
|
||||
if clean_idle_topology_supported
|
||||
&& scanner_clean_idle_backoff_configured(&initial_runtime_config)
|
||||
@@ -3344,8 +3331,7 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
)
|
||||
.await
|
||||
.unwrap_or(ScannerCycleOutcome::Failed);
|
||||
superseded_backoff.record_retryable_cycle(initial_outcome == ScannerCycleOutcome::Superseded);
|
||||
deferred_backoff.record_retryable_cycle(matches!(initial_outcome, ScannerCycleOutcome::Deferred(_)));
|
||||
superseded_backoff.record_cycle(initial_outcome);
|
||||
dirty_usage_generation_seen = dirty_generation_before_cycle;
|
||||
if guard.is_lock_lost() {
|
||||
record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await;
|
||||
@@ -3430,9 +3416,7 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
let mut wait_plan =
|
||||
scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for);
|
||||
let superseded_retry_interval = superseded_backoff.retry_interval(runtime_config.cycle_interval);
|
||||
let deferred_retry_interval = deferred_backoff.retry_interval(runtime_config.cycle_interval);
|
||||
let convergence_retry_interval = superseded_retry_interval.or(deferred_retry_interval);
|
||||
if let Some(retry_interval) = convergence_retry_interval {
|
||||
if let Some(retry_interval) = superseded_retry_interval {
|
||||
wait_plan.effective_interval = retry_interval;
|
||||
wait_plan.delay = randomized_cycle_delay_for(retry_interval).min(retry_interval);
|
||||
}
|
||||
@@ -3459,8 +3443,6 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
clean_idle_backoff_enabled = backoff_enabled,
|
||||
superseded_retry_backoff_enabled = superseded_retry_interval.is_some(),
|
||||
superseded_cycles = superseded_backoff.consecutive_cycles,
|
||||
deferred_retry_backoff_enabled = deferred_retry_interval.is_some(),
|
||||
deferred_cycles = deferred_backoff.consecutive_cycles,
|
||||
lifecycle_active = maintenance_features.lifecycle,
|
||||
replication_active = maintenance_features.replication,
|
||||
feature_inspection_failed = maintenance_features.inspection_failed,
|
||||
@@ -3475,12 +3457,13 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
activity_poll_interval,
|
||||
&mut scanner_activity_seen,
|
||||
ScannerCycleObservedGenerations {
|
||||
// A non-converged cycle holds further activity notifications
|
||||
// until its bounded retry timer to avoid an unbroken scan loop.
|
||||
dirty_usage: convergence_retry_interval.is_none().then_some(dirty_usage_generation_seen),
|
||||
// A superseded cycle already observed concurrent writes. Hold
|
||||
// further dirty notifications until the bounded retry timer so
|
||||
// a hot bucket cannot drive an unbroken full-scan loop.
|
||||
dirty_usage: superseded_retry_interval.is_none().then_some(dirty_usage_generation_seen),
|
||||
runtime_config: runtime_config_generation_seen,
|
||||
maintenance: maintenance_generation_before_wait,
|
||||
defer_cluster_activity: convergence_retry_interval.is_some(),
|
||||
defer_cluster_activity: superseded_retry_interval.is_some(),
|
||||
},
|
||||
|| guard.is_lock_lost(),
|
||||
|| probe_scanner_activity(storeapi.as_ref(), distributed),
|
||||
@@ -3557,8 +3540,7 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
)
|
||||
.await
|
||||
.unwrap_or(ScannerCycleOutcome::Failed);
|
||||
superseded_backoff.record_retryable_cycle(outcome == ScannerCycleOutcome::Superseded);
|
||||
deferred_backoff.record_retryable_cycle(matches!(outcome, ScannerCycleOutcome::Deferred(_)));
|
||||
superseded_backoff.record_cycle(outcome);
|
||||
dirty_usage_generation_seen = dirty_generation_before_cycle;
|
||||
if guard.is_lock_lost() {
|
||||
record_scanner_leader_lock_lost("Scanner leader lock lost during a scanner cycle").await;
|
||||
@@ -3728,12 +3710,6 @@ fn scanner_cycle_completion_outcome(
|
||||
) -> ScannerCycleOutcome {
|
||||
match (scan_status, usage_persist_outcome) {
|
||||
(_, DataUsagePersistOutcome::Failed) => ScannerCycleOutcome::Failed,
|
||||
(ScannerCycleStatus::Deferred(reason), DataUsagePersistOutcome::NoUpdate)
|
||||
if !has_dirty_usage && !has_failed_dirty_usage =>
|
||||
{
|
||||
ScannerCycleOutcome::Deferred(reason)
|
||||
}
|
||||
(ScannerCycleStatus::Deferred(_), _) => ScannerCycleOutcome::Failed,
|
||||
(ScannerCycleStatus::Superseded, _) if !has_failed_dirty_usage => ScannerCycleOutcome::Superseded,
|
||||
(ScannerCycleStatus::Superseded, _) => ScannerCycleOutcome::Failed,
|
||||
(
|
||||
@@ -6577,46 +6553,6 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_scanner_cycle_completion_prioritizes_persist_failure() {
|
||||
assert_eq!(
|
||||
scanner_cycle_completion_outcome(
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
||||
DataUsagePersistOutcome::NoUpdate,
|
||||
false,
|
||||
false,
|
||||
),
|
||||
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
);
|
||||
assert_eq!(
|
||||
scanner_cycle_completion_outcome(
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
|
||||
DataUsagePersistOutcome::Saved,
|
||||
false,
|
||||
false,
|
||||
),
|
||||
ScannerCycleOutcome::Failed
|
||||
);
|
||||
assert_eq!(
|
||||
scanner_cycle_completion_outcome(
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
|
||||
DataUsagePersistOutcome::NoUpdate,
|
||||
true,
|
||||
false,
|
||||
),
|
||||
ScannerCycleOutcome::Failed
|
||||
);
|
||||
assert_eq!(
|
||||
scanner_cycle_completion_outcome(
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
|
||||
DataUsagePersistOutcome::Failed,
|
||||
false,
|
||||
false,
|
||||
),
|
||||
ScannerCycleOutcome::Failed
|
||||
);
|
||||
assert_eq!(
|
||||
scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::NoUpdate, false, false),
|
||||
ScannerCycleOutcome::Failed
|
||||
);
|
||||
assert_eq!(
|
||||
scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Failed, true, true),
|
||||
ScannerCycleOutcome::Failed
|
||||
@@ -7004,47 +6940,47 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn superseded_retry_backoff_grows_caps_and_resets_after_convergence() {
|
||||
let mut backoff = ScannerRetryBackoff::default();
|
||||
let mut backoff = ScannerSupersededBackoff::default();
|
||||
assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None);
|
||||
|
||||
for expected in [5, 10, 20, 40, 80, 160, 320] {
|
||||
backoff.record_retryable_cycle(true);
|
||||
backoff.record_cycle(ScannerCycleOutcome::Superseded);
|
||||
assert_eq!(
|
||||
backoff.retry_interval(Duration::from_secs(24 * 60 * 60)),
|
||||
Some(Duration::from_secs(expected))
|
||||
);
|
||||
}
|
||||
for _ in 0..20 {
|
||||
backoff.record_retryable_cycle(true);
|
||||
backoff.record_cycle(ScannerCycleOutcome::Superseded);
|
||||
}
|
||||
assert_eq!(
|
||||
backoff.retry_interval(Duration::from_secs(24 * 60 * 60)),
|
||||
Some(Duration::from_secs(24 * 60 * 60))
|
||||
);
|
||||
|
||||
backoff.record_retryable_cycle(false);
|
||||
backoff.record_cycle(ScannerCycleOutcome::Completed);
|
||||
assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn superseded_retry_backoff_respects_a_faster_configured_cycle() {
|
||||
let mut backoff = ScannerRetryBackoff::default();
|
||||
backoff.record_retryable_cycle(true);
|
||||
let mut backoff = ScannerSupersededBackoff::default();
|
||||
backoff.record_cycle(ScannerCycleOutcome::Superseded);
|
||||
|
||||
// A configured cycle shorter than the base still wins: retrying sooner
|
||||
// than the operator's own cadence buys nothing.
|
||||
assert_eq!(backoff.retry_interval(Duration::from_secs(3)), Some(Duration::from_secs(3)));
|
||||
backoff.record_retryable_cycle(true);
|
||||
backoff.record_cycle(ScannerCycleOutcome::Superseded);
|
||||
assert_eq!(backoff.retry_interval(Duration::from_secs(3)), Some(Duration::from_secs(6)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn superseded_retry_backoff_grows_from_the_default_cycle() {
|
||||
let mut backoff = ScannerRetryBackoff::default();
|
||||
let mut backoff = ScannerSupersededBackoff::default();
|
||||
// The first race after a write burst retries in seconds, not a whole
|
||||
// cycle, while repeated supersedes still climb toward the cap.
|
||||
for expected in [5, 10, 20, 40] {
|
||||
backoff.record_retryable_cycle(true);
|
||||
backoff.record_cycle(ScannerCycleOutcome::Superseded);
|
||||
assert_eq!(backoff.retry_interval(Duration::from_secs(60)), Some(Duration::from_secs(expected)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2194,45 +2194,11 @@ pub(crate) async fn scanner_set_disk_inventory(set: &SetDisks) -> Vec<Arc<Disk>>
|
||||
disks
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ScannerCycleDeferReason {
|
||||
ActivityBaselineUnavailable,
|
||||
DataMovement,
|
||||
}
|
||||
|
||||
impl ScannerCycleDeferReason {
|
||||
pub(crate) fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::ActivityBaselineUnavailable => "activity_baseline_unavailable",
|
||||
Self::DataMovement => "data_movement",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ScannerCycleStatus {
|
||||
Complete,
|
||||
Incomplete,
|
||||
Superseded,
|
||||
Deferred(ScannerCycleDeferReason),
|
||||
}
|
||||
|
||||
enum ScannerActivityPreflight {
|
||||
Ready(crate::scanner::ScannerActivitySnapshot),
|
||||
ActivityBaselineUnavailable(String),
|
||||
DataMovement,
|
||||
}
|
||||
|
||||
fn scanner_activity_preflight(
|
||||
activity: std::result::Result<crate::scanner::ScannerActivitySnapshot, String>,
|
||||
) -> ScannerActivityPreflight {
|
||||
match activity {
|
||||
Err(error) => ScannerActivityPreflight::ActivityBaselineUnavailable(error),
|
||||
Ok(snapshot) if !crate::scanner::scanner_activity_allows_usage_publication(&snapshot) => {
|
||||
ScannerActivityPreflight::DataMovement
|
||||
}
|
||||
Ok(snapshot) => ScannerActivityPreflight::Ready(snapshot),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -2341,9 +2307,9 @@ impl ScannerIOCycle for ECStore {
|
||||
let child_token = ctx.child_token();
|
||||
|
||||
let distributed = self.setup_is_dist_erasure().await;
|
||||
let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) {
|
||||
ScannerActivityPreflight::Ready(snapshot) => snapshot,
|
||||
ScannerActivityPreflight::ActivityBaselineUnavailable(err) => {
|
||||
let activity_before = match crate::scanner::probe_scanner_activity(self, distributed).await {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_SET_STATE,
|
||||
@@ -2353,26 +2319,20 @@ impl ScannerIOCycle for ECStore {
|
||||
error = %err,
|
||||
"Scanner cycle skipped because cluster activity could not be baselined"
|
||||
);
|
||||
return Ok(ScannerCycleResult::new(
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
|
||||
None,
|
||||
));
|
||||
}
|
||||
ScannerActivityPreflight::DataMovement => {
|
||||
debug!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_SET_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
state = "cycle_data_movement_active",
|
||||
"Scanner cycle deferred while rebalance or decommission data movement is active"
|
||||
);
|
||||
return Ok(ScannerCycleResult::new(
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
|
||||
None,
|
||||
));
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
|
||||
}
|
||||
};
|
||||
if !crate::scanner::scanner_activity_allows_usage_publication(&activity_before) {
|
||||
debug!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_SET_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
state = "cycle_data_movement_active",
|
||||
"Scanner cycle deferred while rebalance or decommission data movement is active"
|
||||
);
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
|
||||
}
|
||||
let dirty_generation_before_bucket_list = dirty_usage_generation();
|
||||
let bucket_listing = self.list_bucket_for_scanner(&BucketOptions::default()).await?;
|
||||
let mut bucket_plan_complete = bucket_listing.topology_complete;
|
||||
@@ -4022,7 +3982,6 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::scanner_budget::ScannerCycleBudgetConfig;
|
||||
use crate::scanner_folder::ScannerItem;
|
||||
use crate::storage_api::owner::{EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats};
|
||||
use crate::storage_api::scan::{BucketOperations as _, MakeBucketOptions, ObjectIO as _};
|
||||
use crate::{
|
||||
DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions,
|
||||
@@ -4045,20 +4004,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_activity_preflight_defers_a_temporarily_offline_peer() {
|
||||
let preflight = scanner_activity_preflight(Err("peer rustfs-node3:9000 is temporarily offline".to_string()));
|
||||
|
||||
match preflight {
|
||||
ScannerActivityPreflight::ActivityBaselineUnavailable(error) => {
|
||||
assert_eq!(error, "peer rustfs-node3:9000 is temporarily offline");
|
||||
}
|
||||
ScannerActivityPreflight::Ready(_) | ScannerActivityPreflight::DataMovement => {
|
||||
panic!("an unavailable activity baseline must defer the scanner cycle");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
init_ecstore_config_for_scanner_tests();
|
||||
let temp_dir = tempfile::tempdir().expect("multi-pool scanner test directory should be created");
|
||||
@@ -4151,42 +4096,6 @@ mod tests {
|
||||
assert!(!second.is_lock_lost());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
|
||||
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
|
||||
let mut pool_stats = vec![EcstoreRebalanceStats::default(); store.pools.len()];
|
||||
pool_stats[0] = EcstoreRebalanceStats {
|
||||
participating: true,
|
||||
info: EcstoreRebalanceInfo {
|
||||
start_time: Some(OffsetDateTime::now_utc()),
|
||||
status: EcstoreRebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
*store.rebalance_meta.write().await = Some(EcstoreRebalanceMeta {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
pool_stats,
|
||||
..Default::default()
|
||||
});
|
||||
assert!(store.scanner_data_movement_active().await);
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
|
||||
let (updates, mut receiver) = mpsc::channel(1);
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
ScannerIOCycle::nsscanner_with_status(store.as_ref(), ctx, budget, updates, 1, 1, HealScanMode::Normal),
|
||||
)
|
||||
.await
|
||||
.expect("rebalance-deferred scanner cycle should finish")
|
||||
.expect("rebalance-deferred scanner cycle should succeed");
|
||||
|
||||
assert_eq!(result.status, ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement));
|
||||
assert!(receiver.recv().await.is_none(), "rebalance-deferred cycle must not publish usage");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn data_usage_publish_fails_when_receiver_is_closed() {
|
||||
let (updates, receiver) = mpsc::channel(1);
|
||||
|
||||
@@ -83,11 +83,6 @@ pub(crate) use rustfs_ecstore::api::layout::{
|
||||
EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rebalance::{
|
||||
RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta,
|
||||
RebalanceStats as EcstoreRebalanceStats,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext;
|
||||
pub(crate) use rustfs_ecstore::api::runtime::{
|
||||
expiry_state_handle as ecstore_expiry_state_handle, global_tier_config_mgr as ecstore_get_global_tier_config_mgr,
|
||||
@@ -127,9 +122,8 @@ pub(crate) mod owner {
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::{
|
||||
EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints,
|
||||
EcstoreInstanceContext, EcstorePoolEndpoints, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta,
|
||||
EcstoreRebalanceStats, ecstore_config_init, ecstore_init_bucket_metadata_sys, ecstore_init_local_disks_with_instance_ctx,
|
||||
ecstore_new_disk,
|
||||
EcstoreInstanceContext, EcstorePoolEndpoints, ecstore_config_init, ecstore_init_bucket_metadata_sys,
|
||||
ecstore_init_local_disks_with_instance_ctx, ecstore_new_disk,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -103,7 +103,6 @@ pub enum StorageErrorCode {
|
||||
SourceStalled,
|
||||
Timeout,
|
||||
InvalidPath,
|
||||
QuotaExceeded,
|
||||
}
|
||||
|
||||
impl StorageErrorCode {
|
||||
@@ -189,7 +188,6 @@ impl StorageErrorCode {
|
||||
Self::SourceStalled => 0x50,
|
||||
Self::Timeout => 0x51,
|
||||
Self::InvalidPath => 0x52,
|
||||
Self::QuotaExceeded => 0x53,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -275,7 +273,6 @@ impl StorageErrorCode {
|
||||
0x50 => Some(Self::SourceStalled),
|
||||
0x51 => Some(Self::Timeout),
|
||||
0x52 => Some(Self::InvalidPath),
|
||||
0x53 => Some(Self::QuotaExceeded),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -350,7 +347,6 @@ mod tests {
|
||||
(StorageErrorCode::RebalanceAlreadyRunning, 0x40),
|
||||
(StorageErrorCode::OperationCanceled, 0x41),
|
||||
(StorageErrorCode::NamespaceLockQuorumUnavailable, 0x42),
|
||||
(StorageErrorCode::QuotaExceeded, 0x53),
|
||||
];
|
||||
|
||||
const DISK_PRESERVATION_ERROR_CODES: &[(StorageErrorCode, u32)] = &[
|
||||
|
||||
@@ -17,21 +17,14 @@
|
||||
//!
|
||||
//! Use suffix-based API: `get_header(headers, SUFFIX_FORCE_DELETE)` queries both
|
||||
//! x-rustfs-force-delete and x-minio-force-delete.
|
||||
//!
|
||||
//! This module is the canonical owner of these interop values. One deliberate
|
||||
//! copy exists: `crates/replication/src/http.rs` re-declares the subset it
|
||||
//! needs because the wire-contract crate must stay free of internal
|
||||
//! dependencies (arch guard in `scripts/check_architecture_migration_rules.sh`
|
||||
//! bans replication -> rustfs-utils). When changing a value here, check the
|
||||
//! pinned copy there; its tests pin the shared wire values byte-for-byte.
|
||||
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use std::borrow::Cow;
|
||||
|
||||
const RUSTFS_PREFIX: &str = "x-rustfs-";
|
||||
const MINIO_PREFIX: &str = "x-minio-";
|
||||
pub const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
|
||||
pub const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";
|
||||
const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
|
||||
const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";
|
||||
const MINIO_INTERNAL_ENCRYPTION_PREFIX: &str = "x-minio-internal-server-side-encryption-";
|
||||
const MINIO_INTERNAL_ENCRYPTED_MULTIPART: &str = "x-minio-internal-encrypted-multipart";
|
||||
const RUSTFS_ENCRYPTION_ORIGINAL_SIZE: &str = super::object_encryption_keys::INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER;
|
||||
|
||||
@@ -14,13 +14,6 @@
|
||||
|
||||
//! System metadata compatibility: write both x-rustfs-internal-* and x-minio-internal-*
|
||||
//! for MinIO interoperability. Read prefers RustFS, fallback to MinIO.
|
||||
//!
|
||||
//! This module is the canonical owner of these interop values. One deliberate
|
||||
//! copy exists: `crates/replication/src/http.rs` re-declares the subset it
|
||||
//! needs because the wire-contract crate must stay free of internal
|
||||
//! dependencies (arch guard in `scripts/check_architecture_migration_rules.sh`
|
||||
//! bans replication -> rustfs-utils). When changing a value here, check the
|
||||
//! pinned copy there; its tests pin the shared wire values byte-for-byte.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
|
||||
|
||||
@@ -30,13 +30,6 @@ use super::headers::{SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER};
|
||||
pub const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id";
|
||||
pub const INTERNAL_ENCRYPTION_KEY_HEADER: &str = "x-rustfs-encryption-key";
|
||||
pub const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv";
|
||||
/// Carries the AEAD algorithm the object was sealed with.
|
||||
///
|
||||
/// The S3 `x-amz-server-side-encryption` header records the *SSE mode*
|
||||
/// (`AES256` / `aws:kms`), not the cipher, so it cannot round-trip
|
||||
/// `ChaCha20Poly1305`. Without this header a ChaCha-sealed object comes back
|
||||
/// from the projection claiming `aws:kms` and is then opened with the wrong
|
||||
/// cipher.
|
||||
pub const INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "x-rustfs-encryption-algorithm";
|
||||
pub const INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER: &str = "x-rustfs-encryption-original-size";
|
||||
pub const INTERNAL_ENCRYPTION_CONTEXT_HEADER: &str = "x-rustfs-encryption-context";
|
||||
@@ -52,15 +45,6 @@ pub const MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER: &str = "X-Minio-Internal-
|
||||
pub const MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key";
|
||||
pub const MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Context";
|
||||
|
||||
/// Reserved RustFS-branded twin of the MinIO-internal SSE key family.
|
||||
///
|
||||
/// No RustFS writer emits these keys today — the SSE writer persists the
|
||||
/// MinIO-branded `X-Minio-Internal-Server-Side-Encryption-*` keys verbatim for
|
||||
/// interoperability — but redaction (`rustfs_filemeta`) and replication
|
||||
/// stripping treat the family as sensitive so that a future or third-party
|
||||
/// writer cannot leak sealed material through the reserved names.
|
||||
pub const RUSTFS_INTERNAL_ENCRYPTION_PREFIX: &str = "x-rustfs-internal-server-side-encryption-";
|
||||
|
||||
pub const REPLICATION_SSEC_ALGORITHM_HEADER: &str = "X-Rustfs-Replication-Ssec-Algorithm";
|
||||
pub const REPLICATION_SSEC_KEY_MD5_HEADER: &str = "X-Rustfs-Replication-Ssec-Key-Md5";
|
||||
pub const REPLICATION_SSEC_ORIGINAL_SIZE_HEADER: &str = "X-Rustfs-Replication-Ssec-Original-Size";
|
||||
@@ -141,14 +125,13 @@ pub fn ssec_replication_transport_header(stored_key: &str) -> Option<&'static st
|
||||
/// SSE-C material. SSE-C passthrough re-adds its keys through the transport
|
||||
/// mapping instead.
|
||||
pub fn is_replication_stripped_encryption_key(key: &str) -> bool {
|
||||
// The x-rustfs-internal- SSE prefix is a reserved name family with no
|
||||
// writer today (see RUSTFS_INTERNAL_ENCRYPTION_PREFIX); cover it here so
|
||||
// this predicate is safe to use standalone, without an is_internal_key
|
||||
// backstop.
|
||||
// The dual-key invariant writes an x-rustfs-internal- twin next to every
|
||||
// x-minio-internal- SSE key; cover it here so this predicate is safe to
|
||||
// use standalone, without an is_internal_key backstop.
|
||||
super::is_encryption_metadata_key(key)
|
||||
|| super::is_sse_header(key)
|
||||
|| key.eq_ignore_ascii_case(SSEC_ORIGINAL_SIZE_HEADER)
|
||||
|| super::starts_with_ignore_ascii_case(key, RUSTFS_INTERNAL_ENCRYPTION_PREFIX)
|
||||
|| super::starts_with_ignore_ascii_case(key, "x-rustfs-internal-server-side-encryption-")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -12,10 +12,6 @@ for later deletion.
|
||||
|
||||
## Open Items
|
||||
|
||||
- `table-publication-fence-v1` S3 Tables publication fencing: nodes that predate table and table-bucket publication fences can mutate live files while a new node is publishing a catalog pointer. New nodes retain exact object guards until the operator confirms that every serving node uses the new fences. Fleet confirmation also requires non-overlapping active warehouse prefixes and lifecycle workers that exclude table buckets. Remove the exact live-file fallback and the fleet-confirmation gate after the minimum supported RustFS release acquires table fences for registered-table mutations and table-bucket fences for unresolved-prefix mutations.
|
||||
- `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: mixed-version deployments continue writing version 1 snapshots until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2.
|
||||
- `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration.
|
||||
- `table-catalog-backing-manifest-v1-wire-labels` durable strong backing manifest labels: version 1 published "STRONG_KV_WAL" and "CUT_OVER_LINEARIZABLE_READS" before the implementation was narrowed to the ETag-CAS durable snapshot backing. Internal names and operator documentation describe the implemented semantics, while version 1 responses retain those labels for existing clients. Replace the labels only in a new manifest version with an explicit client migration contract.
|
||||
- `cross-pool-fence-v1` authenticated unsupported advertisement: predeployment servers recognize the versioned cross-pool fence capability probe but report support version 0, allowing a later all-peer probe to distinguish predeployment nodes without activating a second lock domain. Replace the unsupported advertisement only when composite lock acquisition, a cluster-wide activation fence, complete fleet proof, commit-time proof revalidation, and fail-closed revocation ship together.
|
||||
- `table-catalog-dotted-namespace` Iceberg REST namespace path compatibility: existing RustFS clients use dotted namespace paths, while the standard multi-level contract uses the URL-encoded unit separator `%1F`. New servers accept both forms so a rolling upgrade does not invalidate existing catalog configuration. Remove the dotted fallback after the minimum supported RustFS release advertises `%1F` and all supported clients have refreshed their catalog configuration.
|
||||
- `rustfs-5509` FileInfo positional MessagePack decoding: beta.11 serialized 28 fields, while beta.12 inserted transition-version fields in the middle and serialized an incompatible 30-field array. New releases write named maps and retain readers for both shipped array layouts so direct and rolling upgrades can read either release. Remove the positional-array readers after every supported direct-upgrade release writes named maps and no retained RPC payload can contain a pre-map FileInfo array.
|
||||
|
||||
@@ -10,28 +10,10 @@ and rollback steps.
|
||||
| Area | Current owner | Size | Split status |
|
||||
|---|---|---:|---|
|
||||
| Bucket lifecycle | `crates/lifecycle/` + `crates/ecstore/src/bucket/lifecycle/` | core contracts + ECStore runtime | Core contract extracted |
|
||||
| Bucket replication | `crates/ecstore/src/bucket/replication/` | 15,619 lines | Contracts extracted; runtime move pending |
|
||||
| Bucket replication | `crates/ecstore/src/bucket/replication/` | 8,730 lines | Contracts extracted; runtime move pending |
|
||||
| Set disks | `crates/ecstore/src/set_disk/` | state carrier plus operation modules | Keep in ECStore |
|
||||
| Public ECStore facade | `crates/ecstore/src/api/mod.rs` | broad compatibility surface | Shrink only through guarded PRs |
|
||||
|
||||
Measured 2026-08-12: the whole crate is 265 files / ~288K lines (roughly half
|
||||
is inline `#[cfg(test)]` code). The largest single files are `disk/local.rs`
|
||||
(21,063 lines), `bucket/lifecycle/bucket_lifecycle_ops.rs` (11,961 lines), and
|
||||
`set_disk/mod.rs` (11,151 lines). Reproduce with:
|
||||
|
||||
```bash
|
||||
find crates/ecstore/src -name '*.rs' | xargs wc -l | sort -rn | head
|
||||
find crates/ecstore/src/bucket/replication -name '*.rs' | xargs wc -l | tail -1
|
||||
```
|
||||
|
||||
No split step has landed since the contract-extraction PRs of 2026-07-04,
|
||||
while the `bucket/replication` runtime grew from 8,730 to 15,619 lines (+79%)
|
||||
through feature work (e.g. SSE-C ciphertext passthrough replication #5898,
|
||||
delete-marker purge retry/replay #5864). To keep the gap from widening: in
|
||||
domains that already have a contract crate, new replication runtime logic that
|
||||
does not need ECStore runtime state must land in `rustfs-replication`, not in
|
||||
`crates/ecstore/src/bucket/replication/`.
|
||||
|
||||
The file split inside `set_disk/` is already operation-oriented: read, write,
|
||||
list, multipart, lock, heal, and replication code live in separate modules.
|
||||
The remaining large surface is the shared `SetDisks` state and cross-cutting
|
||||
|
||||
@@ -92,7 +92,6 @@ catalog extension.
|
||||
|---|---|---|
|
||||
| Metadata retention dry-run | Supported | Reports retained metadata and deletion candidates without moving the table pointer. |
|
||||
| Metadata cleanup delete | Supported | Deletes only candidates that pass the safety window and current-pointer checks. |
|
||||
| Ordinary bucket lifecycle expiry | Disabled for table buckets | Table bucket objects are excluded from ordinary lifecycle expiration, including already queued expiry work. Snapshot expiration and orphan cleanup remain catalog maintenance operations so referenced Iceberg files cannot be deleted outside publication fencing. |
|
||||
| Snapshot expiration planning | Supported | Produces expiration plans with retained and candidate snapshots. |
|
||||
| Snapshot expiration commit | Preview / controlled | Can manually commit safe snapshot expiration through the catalog. Stale plans fail closed. |
|
||||
| Manifest/data/delete reachability cleanup | Supported | Reads manifest-list and manifest Avro references, reports reachable objects, and deletes only unreferenced table objects that pass the safety window. |
|
||||
@@ -113,16 +112,14 @@ catalog extension.
|
||||
|---|---|---|
|
||||
| Single-table CAS | Supported | The table pointer advances only through expected-token and expected-metadata-location validation. |
|
||||
| Idempotent retry | Supported | Repeated commit IDs can return the already finalized result or surface recoverable finalization gaps. |
|
||||
| Commit publication fencing | Supported with rolling-upgrade gate | Existing deployments retain exact object guards so older writers cannot mutate referenced files during publication. Set `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` only after every serving node supports table and table-bucket publication fences. In scalable mode, active table warehouse prefixes must not overlap, ordinary lifecycle expiry remains disabled for table buckets, and first enablement, first publication, drop, and warehouse relocation are serialized by the table-bucket fence. |
|
||||
| Post-CAS finalization recovery | Supported | Diagnostics and recovery can repair stale or missing idempotency indexes without changing the current table pointer. |
|
||||
| Catalog export | Supported | Exposes table state, commit recovery state, and backing migration information for operator inspection. |
|
||||
| Strong backing state transfer | Supported | Object-backed table bucket, namespace, table, view, commit-log, and idempotency state can be materialized into the durable strong snapshot. The transfer is deterministic, ETag-CAS protected, idempotent after an interrupted finalization, validates candidate state through the restart decoder before publication, preserves resource-backed implicit namespaces, and fails closed when an inactive explicit namespace conflicts with active descendants or resources. Snapshot hydration requires a stable non-empty ETag, caps the encoded snapshot at 64 MiB, shares state and reload serialization across requests in one server context, and rejects disappearance or format-version rollback after observation. Configured durable-strong mode rejects a missing snapshot on its first catalog access after startup; only object-backed migration may initialize an empty target. |
|
||||
| Durable backing migration preflight | Supported | `GET /iceberg/v1/{warehouse}/catalog/migration` and the `/_iceberg/v1` alias inspect object-backed catalog inventory, recovery blockers, warehouse prefix index readiness, active table/view identifier collisions, persistent write-fence state, target snapshot agreement, and whether every table bucket is ready for cutover. |
|
||||
| Durable backing migration execution | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/migration` fences table-bucket registry changes, acquires a persistent per-bucket write fence, records whether a global strong snapshot existed before publication, drains in-flight catalog mutations, materializes the target snapshot, and reports `ready_to_enable_durable_strong`. Retries and `DELETE` may restore a known-absent initial target after an ambiguous first write, but fail closed if a previously existing or materialized global snapshot disappears. `DELETE` releases the bucket fence only while its target state has not advanced, and releases the registry fence after the last bucket is cancelled. Both mutations require `admin:MigrateTableCatalog`. |
|
||||
| Strong snapshot rolling compatibility | Supported | Durable strong control-plane reads snapshot versions 1 and 2, writes version 1 by default, and writes version 2 only after both the requested and fleet-confirmed gates are enabled. A running process rejects any lower-format snapshot after observing a higher format. Once version 2 is fleet-confirmed, table data-plane resolution fails closed until the persisted snapshot is version 2; a missing table-bucket entry also fails closed instead of bypassing table-aware authorization. |
|
||||
| Strong backing state transfer | Supported | Object-backed table bucket, namespace, table, view, commit-log, and idempotency state can be materialized into the durable strong snapshot. The transfer is deterministic, ETag-CAS protected, idempotent after an interrupted finalization, and fails closed when a table or view has no owning namespace entry. |
|
||||
| Durable backing migration preflight | Supported | `GET /iceberg/v1/{warehouse}/catalog/migration` and the `/_iceberg/v1` alias inspect object-backed catalog inventory, recovery blockers, warehouse prefix index readiness, persistent write-fence state, target snapshot agreement, and whether every table bucket is ready for cutover. |
|
||||
| Durable backing migration execution | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/migration` fences table-bucket registry changes, acquires a persistent per-bucket write fence, drains in-flight catalog mutations, materializes the target snapshot, and reports `ready_to_enable_durable_strong`. `DELETE` safely releases the bucket fence only while its target state has not advanced, and releases the registry fence after the last bucket is cancelled. Both mutations require `admin:MigrateTableCatalog`. |
|
||||
| Disaster recovery rehearsal | Manual/live harness | `failure_coverage.py --print-disaster-recovery-rehearsal` generates an operator runbook covering catalog export, diagnostics, safe recovery repair, rollback/import, durable backing migration dry-run, post-recovery loadTable, and table data-plane policy probes. |
|
||||
| Scale and fault rehearsal | Manual/live harness | `failure_coverage.py --print-scale-fault-rehearsal` generates an opt-in runbook for concurrent writer stress, maintenance scheduler lease recovery, durable backing cutover preflight, recovery/rollback/import under load, and post-run evidence capture. |
|
||||
| Durable strong snapshot backing cutover | Preview / controlled | Operators can select the ETag-CAS snapshot backing with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong` only after every table bucket reports `SNAPSHOT_MATERIALIZED` and `ready_to_enable_durable_strong: true`. This mode does not claim a separate external KV/WAL service, and object-only advanced operations fail closed. Version 1 backing manifests retain the legacy `STRONG_KV_WAL` and `CUT_OVER_LINEARIZABLE_READS` wire labels for client compatibility; those labels do not expand the implementation claim. |
|
||||
| Strong KV/WAL backing cutover | Preview / controlled | Operators can select durable strong backing with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong` only after every table bucket reports `SNAPSHOT_MATERIALIZED` and `ready_to_enable_durable_strong: true`. Object-only advanced operations fail closed in durable strong mode. |
|
||||
| Single active writer region | Supported policy | Diagnostics publish single-active-writer semantics and read-only replica limits. |
|
||||
| Active-active multi-region writes | Not claimed | A table must not accept independent concurrent writers in multiple active regions. |
|
||||
|
||||
@@ -137,60 +134,23 @@ warehouse:
|
||||
`GetTableCatalogAction` on each table bucket. Treat every `blockers` entry as
|
||||
fail-closed; repair commit recovery state and backfill the warehouse prefix
|
||||
index before continuing.
|
||||
3. Before the migration `POST`, drain every catalog writer that predates the
|
||||
durable-backing migration fence and restart it on a fence-aware release. An
|
||||
older writer does not recognize the persisted fence and can otherwise
|
||||
mutate the object-backed source after the snapshot inventory is captured.
|
||||
Keep all catalog writers on the fence-aware release until cutover completes.
|
||||
4. Inventory object-only advanced operations, including maintenance workers,
|
||||
catalog recovery, export, diagnostics, and external catalog bridge writes.
|
||||
Quiesce mutating operations before cutover and confirm that each required
|
||||
operation is supported by durable-strong mode; unsupported operations fail
|
||||
closed after cutover rather than continuing against object-backed state.
|
||||
5. Run `POST /iceberg/v1/{warehouse}/catalog/migration` with
|
||||
`admin:MigrateTableCatalog`. This acquires the exclusive migration fence to
|
||||
drain in-flight fence-aware mutations, persists the source fence while
|
||||
exclusivity is held, and then copies the catalog state.
|
||||
6. Repeat the preflight and materialization for every table bucket. Do not set
|
||||
3. Run `POST /iceberg/v1/{warehouse}/catalog/migration` with
|
||||
`admin:MigrateTableCatalog`. This persists the source write fence before it
|
||||
drains in-flight mutations and copies the catalog state.
|
||||
4. Repeat the preflight and materialization for every table bucket. Do not set
|
||||
`RUSTFS_TABLE_CATALOG_BACKING=durable-strong` until the preflight reports
|
||||
`SNAPSHOT_MATERIALIZED`, no blockers, and
|
||||
`ready_to_enable_durable_strong: true`.
|
||||
7. Restart with durable strong backing enabled, then verify catalog config,
|
||||
5. Restart with durable strong backing enabled, then verify catalog config,
|
||||
table and view loads, commit idempotency, and table data-plane policy
|
||||
resolution before admitting writers.
|
||||
8. Before restarting into durable-strong mode, `DELETE` on the migration
|
||||
6. Before restarting into durable-strong mode, `DELETE` on the migration
|
||||
endpoint can remove a migration-created target bucket snapshot and release
|
||||
the source fence. After the durable-strong state advances, cancellation
|
||||
fails closed; recovery requires an operator-selected restore or reverse
|
||||
migration instead of restarting against the stale object-backed pointer.
|
||||
9. Preserve the object-backed catalog backup until durable strong backing has
|
||||
7. Preserve the object-backed catalog backup until durable strong backing has
|
||||
passed the operator's retention window.
|
||||
10. Keep strong snapshot writes on version 1 during a rolling binary upgrade.
|
||||
After every catalog writer can read version 2, set both
|
||||
`RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2=true` and
|
||||
`RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED=true`, then restart
|
||||
the catalog writers. Perform a controlled catalog write or migration
|
||||
materialization and confirm that the persisted snapshot is version 2 before
|
||||
serving table data-plane traffic. Setting only one gate does not change the
|
||||
write format.
|
||||
11. After any version 2 snapshot is persisted, do not roll catalog writers back
|
||||
to a binary that only reads version 1. Current binaries preserve version 2
|
||||
even when the gates are later disabled. A running process rejects restored
|
||||
version 1 content after observing version 2, but cannot distinguish an older
|
||||
snapshot with the same format version from a deliberate restore. The format
|
||||
high-water mark is process-local: restoring any older snapshot and restarting
|
||||
every writer is a privileged disaster-recovery rollback that cannot be
|
||||
inferred from the restored object alone. Recovery must restore a compatible
|
||||
binary and a snapshot selected through the operator recovery procedure.
|
||||
12. Migration preflight rejects an active table/view identifier collision before
|
||||
it writes a migration fence. A pre-existing version 1 strong snapshot with
|
||||
such a collision is loaded in cleanup-only quarantine. Ambiguous reads fail
|
||||
closed; each cleanup mutation must reduce the collision set, and unrelated
|
||||
writes remain blocked until all collisions are removed. Drain catalog
|
||||
writers that predate cleanup quarantine before starting this repair, and
|
||||
complete cleanup before the first version 2 write. Restoring any version 1
|
||||
snapshot after a writer has observed version 2 fails closed instead of
|
||||
replacing the in-process catalog state.
|
||||
|
||||
## Production Failure Coverage
|
||||
|
||||
|
||||
@@ -3,10 +3,7 @@
|
||||
RustFS lets operators choose how much fsync work runs on the object write
|
||||
path. The default (`strict`) preserves the fully synced behavior RustFS has
|
||||
always shipped; the relaxed tiers are **opt-in** trades of power-loss
|
||||
durability for latency/IOPS. Note that **newly created buckets default to
|
||||
`relaxed`** (see [New-bucket default](#new-bucket-default)) — a gradual
|
||||
migration that leaves the process-wide default and all pre-existing buckets
|
||||
on `strict`.
|
||||
durability for latency/IOPS.
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -16,9 +13,6 @@ RUSTFS_DURABILITY_MODE=strict|relaxed|none # default: strict
|
||||
|
||||
# Legacy binary switch (kept for compatibility, superseded by the above)
|
||||
RUSTFS_DRIVE_SYNC_ENABLE=true|false # default: true
|
||||
|
||||
# Tier seeded into a NEWLY CREATED bucket's own override (see "New-bucket default")
|
||||
RUSTFS_NEW_BUCKET_DURABILITY_MODE=relaxed|strict|none|inherit # default: relaxed
|
||||
```
|
||||
|
||||
Resolution rules:
|
||||
@@ -144,39 +138,6 @@ curl -X DELETE "http://<host>/rustfs/admin/v3/bucket-durability/<bucket>"
|
||||
`mc` integration is a follow-up; for now the admin API above is the
|
||||
configuration plane.
|
||||
|
||||
### New-bucket default
|
||||
|
||||
A bucket created after this feature ships gets a `relaxed` override **seeded
|
||||
into its own metadata** at creation time (rustfs/backlog#1811), so it opts
|
||||
into MinIO's default posture (object data still fdatasynced; xl.meta and
|
||||
directory-entry fsyncs left to the page cache) without touching the
|
||||
process-wide default. This is a gradual migration:
|
||||
|
||||
- **Pre-existing buckets are unaffected.** A bucket with no `durability.json`
|
||||
entry keeps following the process-wide mode (`strict` by default), exactly
|
||||
as before.
|
||||
- **The process-wide default stays `strict`.** `RUSTFS_DURABILITY_MODE` and
|
||||
`RUSTFS_DRIVE_SYNC_ENABLE` are unchanged; only newly created buckets carry
|
||||
their own `relaxed` override.
|
||||
- **System-critical namespaces stay pinned to `strict`** regardless of any
|
||||
override (see [System-critical pinning](#system-critical-pinning)).
|
||||
|
||||
The seeded tier is controlled by an env var read once per bucket creation
|
||||
(bucket creation is not a hot path):
|
||||
|
||||
```bash
|
||||
RUSTFS_NEW_BUCKET_DURABILITY_MODE=relaxed # default: seed `relaxed`
|
||||
RUSTFS_NEW_BUCKET_DURABILITY_MODE=strict # seed `strict` instead
|
||||
RUSTFS_NEW_BUCKET_DURABILITY_MODE=none # seed `none` instead
|
||||
RUSTFS_NEW_BUCKET_DURABILITY_MODE=inherit # seed nothing: follow the global mode
|
||||
```
|
||||
|
||||
`inherit` (and any unrecognized value, which fails closed to `inherit`) means
|
||||
the new bucket gets no override and follows the process-wide mode. Set this
|
||||
cluster-wide to opt out of the new default entirely. The seed only applies at
|
||||
bucket creation; it never retroactively rewrites existing buckets. To change
|
||||
an existing bucket's tier, use the per-bucket admin API above.
|
||||
|
||||
### Resolution order
|
||||
|
||||
For a write committing into volume `V`:
|
||||
|
||||
@@ -184,22 +184,6 @@ perf record -F 99 -g -- sleep 180
|
||||
perf report --stdio > target/hotpath-abba/cluster-pr-XXXX/telemetry/perf-report.txt
|
||||
```
|
||||
|
||||
When using samply against an already-running RustFS service, attach through the
|
||||
bounded helper instead of calling `samply record -p` directly:
|
||||
|
||||
```bash
|
||||
scripts/run_samply_attach_window.sh \
|
||||
--pid "$RUSTFS_PID" \
|
||||
--duration-secs 180 \
|
||||
--output target/hotpath-abba/cluster-pr-XXXX/telemetry/samply-A1-get-4mib.json.gz
|
||||
```
|
||||
|
||||
Run one attach window per ABBA leg or focused verification cell. After every
|
||||
window, confirm that the `.json.gz` profile and `.syms.json` sidecar are
|
||||
non-empty, that no `samply` process is still attached to the RustFS PID, and
|
||||
that any temporary `perf_event_paranoid` change has been restored before the
|
||||
next cell starts.
|
||||
|
||||
For allocation profiling, build the candidate with:
|
||||
|
||||
```bash
|
||||
@@ -283,14 +267,11 @@ The AI agent should execute this sequence:
|
||||
CPU model, memory size, disk layout, and whether the run is local or cluster.
|
||||
2. Run `scripts/run_hotpath_warp_abba.sh --dry-run` with the final arguments.
|
||||
3. Run the real ABBA command with `--rounds >= 3`.
|
||||
4. For samply CPU attribution, use `scripts/run_samply_attach_window.sh` for
|
||||
each bounded attach window and reject the cell if the profile is empty or a
|
||||
stale `samply` process remains.
|
||||
5. Preserve the full output directory without editing generated CSV files.
|
||||
6. Read `summary.md`, `candidate_gate.md`, and `baseline_drift_gate.md`.
|
||||
7. Summarize only measured facts: candidate deltas, baseline drift, CPU or
|
||||
4. Preserve the full output directory without editing generated CSV files.
|
||||
5. Read `summary.md`, `candidate_gate.md`, and `baseline_drift_gate.md`.
|
||||
6. Summarize only measured facts: candidate deltas, baseline drift, CPU or
|
||||
memory saturation, and any failed workloads.
|
||||
8. Post the summary and artifact location to the tracking issue or PR.
|
||||
7. Post the summary and artifact location to the tracking issue or PR.
|
||||
|
||||
Do not report a performance win or loss when the baseline drift gate failed on
|
||||
the same workload and no rerun was collected.
|
||||
|
||||
@@ -78,34 +78,6 @@ Notes:
|
||||
|
||||
Rotation support differs per backend. Local and Static advertise no `rotate` capability — `capabilities.rotate` is false in the `kms/status` response — and reject rotation with `UnsupportedCapability`; their single key material is never overwritten. Vault Transit delegates rotation to the Transit engine's own key versioning (ciphertext is version-prefixed, e.g. `vault:v1:...`). Vault KV2 rotates by retaining every historical version, as described below. Rotation is reachable through the admin API as `POST /rustfs/admin/v3/kms/keys/rotate`, which the route policy classifies as high risk and gates behind `kms:RotateKey`; it is not exposed through the S3 surface. The upgrade ordering constraint below therefore applies to an operator action, not only to a call from inside the process.
|
||||
|
||||
### Rotation drivers and scheduling, per backend
|
||||
|
||||
The rotate endpoint is one API over three very different mechanisms, and which component actually performs the rotation decides how periodic rotation must be scheduled — on two backends it cannot be scheduled at all.
|
||||
|
||||
| Backend | Can rotate | Who performs the rotation | How to schedule periodic rotation |
|
||||
| --- | --- | --- | --- |
|
||||
| Local | No | Nobody — the backend advertises no `rotate` capability and the rotate endpoint is refused with `UnsupportedCapability` | Cannot be scheduled. Migrating to a rotating backend is the only path to rotation |
|
||||
| Static | No | Nobody — same refusal as Local; the material is supplied out-of-band and read-only | Cannot be scheduled. Migrate to a rotating backend |
|
||||
| Vault KV2 | Yes | **RustFS** owns the whole rotation protocol: freeze the outgoing material as an immutable version record, persist the new version's material, then move the current pointer with a check-and-set write | An **external scheduler** (cron, Kubernetes CronJob, your automation platform) calling `POST /rustfs/admin/v3/kms/keys/rotate`. RustFS deliberately ships no built-in rotation timer — see below |
|
||||
| Vault Transit | Yes | **Vault's Transit engine** — RustFS only forwards the call to Transit's rotate endpoint and records the version bump in its own metadata | Vault's native `auto_rotate_period` on the Transit key. Do **not** additionally point an external scheduler at the RustFS rotate endpoint — see below |
|
||||
| AWS KMS | Yes | **AWS** — the RustFS rotate endpoint maps to `RotateKeyOnDemand` | AWS's native automatic rotation, configured on the AWS side. Do **not** drive periodic rotation through the RustFS endpoint — see below |
|
||||
|
||||
**Local and Static: the wrap ceiling is unmitigable.** These backends wrap every DEK with AES-256-GCM under their single master key using a random 96-bit nonce, and NIST SP 800-38D caps AES-GCM at 2^32 invocations per key when nonces are chosen at random. Each encrypted object write wraps a DEK, so the invocation count tracks the number of encrypted-object writes over the deployment's lifetime. On a rotating backend that count restarts whenever new master key material takes over; on Local and Static it can never restart, because there is no rotation to restart it. The only mitigation is migrating to a backend that rotates. The same 2^32 bound applies to the KV2 backend's wrapping — RustFS wraps DEKs locally there too — but there each rotation mints fresh master key material and resets the count, which is one more reason to actually schedule KV2 rotation rather than merely support it.
|
||||
|
||||
**Vault KV2: bring your own scheduler, deliberately.** RustFS performs the rotation but does not decide when: there is no built-in rotation worker, by design rather than omission. A timer inside the server cannot verify the [cluster-upgrade precondition](#upgrade-before-first-rotation-hard-constraint) before firing, and rotation is not idempotent — without leader election, N nodes running the same schedule would perform N rotations per period, advancing the key version N times. Run exactly one external scheduler, point it at the admin rotate endpoint with credentials scoped to `kms:RotateKey`, and use the [rotation readiness fields](#rotation-readiness-reported-never-acted-on) plus the `KmsKeyRotationOverdue` alert in the [KMS observability runbook](kms-observability-runbook.md#kmskeyrotationoverdue) to verify the schedule is actually keeping up.
|
||||
|
||||
**Vault Transit: exactly one owner of the version cadence.** Configure `auto_rotate_period` on the Transit key and let Vault own the schedule. Layering an external scheduler that calls the RustFS rotate endpoint on top of `auto_rotate_period` creates two competing owners of the key's version cadence, and the effective rotation period stops being the one either owner was configured with. The data path is indifferent to who rotates — Transit ciphertext self-describes the version that wrapped it, so envelopes never pin a version RustFS tracked — but the key version RustFS reports only advances when rotation goes through RustFS, so on an auto-rotating key treat the reported version as a floor, not the truth.
|
||||
|
||||
**AWS KMS: native automatic rotation for cadence, `RotateKeyOnDemand` for incidents.** The RustFS rotate endpoint maps to AWS `RotateKeyOnDemand`, and AWS enforces a lifetime limit on the number of on-demand rotations a key may receive (see the AWS KMS documentation) — a periodic scheduler driving the RustFS endpoint will exhaust that quota and then fail forever. Configure AWS's automatic rotation for periodic cadence and keep the RustFS endpoint for what on-demand rotation is for: incident response and one-off rotations. Note that RustFS neither enables nor observes AWS automatic rotation, and it records no rotation timestamp for AWS keys, so the readiness fields and the rotation-age gauge measure key age on this backend — verify the actual cadence in AWS, not through RustFS.
|
||||
|
||||
**Pre-rotation checklist** (before the first rotation of any key, and before enabling any schedule):
|
||||
|
||||
1. Every node in the cluster runs a build that understands the `master_key_version` envelope field — the [hard upgrade-ordering constraint](#upgrade-before-first-rotation-hard-constraint) below. A timer cannot check this; you must.
|
||||
2. No rolling upgrade is in progress — see [Do not do these during a mixed-version window](#do-not-do-these-during-a-mixed-version-window).
|
||||
3. The [retention and destruction preconditions](#retention-and-destruction-preconditions) are understood: every version record a stored DEK envelope references must remain readable forever, and no retention tooling prunes the version subtree.
|
||||
4. For KV2, exactly one scheduler exists, so no two callers race the same rotation period.
|
||||
5. `RUSTFS_KMS_ROTATION_MAX_AGE_SECS` is set to the rotation period your policy requires, so the per-key `rotation_due` verdict and the rotation-age alert verify the schedule instead of assuming it.
|
||||
|
||||
### Rotation readiness: reported, never acted on
|
||||
|
||||
RustFS does not rotate keys on a schedule. There is no built-in rotation worker, deliberately: rotation is a policy decision with a per-backend cost and a hard upgrade-ordering constraint (see below), and a server that rotated on its own would make that decision on an operator's behalf at a moment it did not choose. What the server does instead is tell you which keys have outlived a period you configure.
|
||||
@@ -121,7 +93,7 @@ The verdict is advisory in the strongest sense: nothing consults it before encry
|
||||
|
||||
`GET /rustfs/admin/v3/kms/keys/{key_id}` does **not** carry these fields. Its response type records a creation date but no rotation timestamp, so a verdict computed there could not tell a key rotated last week from one never rotated at all, and reporting `never_rotated` for a key that was in fact rotated would be worse than reporting nothing. Read the verdict from the listing.
|
||||
|
||||
Driving the rotation itself remains external: call `POST /rustfs/admin/v3/kms/keys/rotate` from your own scheduler, having first satisfied the upgrade-ordering constraint below — and only on the backend where that is the right scheduling model; see [Rotation drivers and scheduling, per backend](#rotation-drivers-and-scheduling-per-backend).
|
||||
Driving the rotation itself remains external: call `POST /rustfs/admin/v3/kms/keys/rotate` from your own scheduler, having first satisfied the upgrade-ordering constraint below.
|
||||
|
||||
### Vault KV2 versioned retention model
|
||||
|
||||
|
||||
@@ -64,7 +64,7 @@ Total damage looks different, and it is worth knowing which you are seeing. When
|
||||
|
||||
The three gauges are republished only by a sweep that saw the whole key set; a sweep that could not finish listing leaves the previous, complete values standing rather than understating them. Keys already on their way out are excluded from the rotation-age gauge, so it does not stay pinned high by a key that will never be rotated again.
|
||||
|
||||
The rotation age comes from whatever the backend reports as the last rotation, and backends only report a rotation they recorded themselves. Today only the Vault KV2 backend persists that timestamp — it is stamped in the same check-and-set write that commits the rotation (`crates/kms/src/backends/vault.rs`), so it exists if and only if the rotation did. Vault Transit and AWS KMS record no rotation timestamp at all: their key listings always report the rotation time as absent, so on those backends every key ages from creation permanently, the gauge measures key age rather than rotation age, and rotating does not reset it. A KV2 key rotated before the timestamp existed likewise ages from creation until its next rotation stamps the record. In every case the gauge overstates rather than invents — it can report an already-rotated key as overdue, never a stale key as fresh — so an alert on it fires early rather than late. Backends that cannot rotate at all (Local, Static) age every key from creation by construction.
|
||||
The rotation age comes from whatever the backend reports as the last rotation, and backends only report a rotation they recorded themselves. A key rotated before its backend persisted rotation timestamps therefore ages from creation until its next rotation stamps the record: the gauge overstates that key's age rather than inventing a rotation it cannot vouch for, so an alert on it fires early rather than late. Backends that cannot rotate at all (Local, Static) age every key from creation by construction.
|
||||
|
||||
### Vault credential metrics
|
||||
|
||||
@@ -195,29 +195,15 @@ Investigation:
|
||||
|
||||
Related signals: `circuit_open`, `backpressure_timeout`, and `backpressure_rejected` on the "Backend Operation Rate by Outcome" panel; `rustfs_kms_backend_in_flight`; Vault availability and seal status.
|
||||
|
||||
### KmsKeyRotationOverdue
|
||||
|
||||
Meaning: `rustfs_kms_oldest_key_rotation_age_seconds` — seconds since the least recently rotated usable key was rotated, counting from creation for keys with no recorded rotation — has been above 400 days for an hour. This is a compliance and hygiene signal, not an outage: encryption and decryption continue unchanged, and nothing in RustFS acts on the verdict. But the longer master key material stays in service the larger the blast radius of its compromise, and on backends where RustFS wraps DEKs locally (Local, Static, Vault KV2) the AES-GCM random-nonce invocation ceiling (NIST SP 800-38D: at most 2^32 wraps under one key) is consumed by every encrypted object write and only ever resets through rotation.
|
||||
|
||||
Investigation:
|
||||
|
||||
1. Find which keys are due. The gauge deliberately names no key — a per-key label would carry key identifiers into the metric stream — so read the per-key verdict from the listing: `GET /rustfs/admin/v3/kms/keys` carries `rotation_due` and `rotation_due_reason` (`age`, `never_rotated`, or `unsupported`) per key, computed against `RUSTFS_KMS_ROTATION_MAX_AGE_SECS`. The verdict appears only on the listing, not on single-key describe. If `RUSTFS_KMS_ROTATION_MAX_AGE_SECS` is unset, set it to your policy's rotation period so the per-key verdict and this alert agree on what "overdue" means.
|
||||
2. If the reason is `unsupported`, the backend cannot rotate at all (Local, Static). There is no key-level response; the decision is a backend migration, and the wrap ceiling above is the reason it cannot be deferred forever. See the [rotation drivers and scheduling matrix](kms-backend-security.md#rotation-drivers-and-scheduling-per-backend).
|
||||
3. On a backend that can rotate, act per the driver matrix: on **Vault KV2**, check why your external rotation scheduler did not run (or set one up — RustFS deliberately ships none) and satisfy the [pre-rotation checklist](kms-backend-security.md#rotation-drivers-and-scheduling-per-backend) before rotating, above all the [upgrade-ordering hard constraint](kms-backend-security.md#upgrade-before-first-rotation-hard-constraint) — never respond to this alert by rotating in the middle of a rolling upgrade. On **Vault Transit**, check `auto_rotate_period` on the key in Vault. On **AWS KMS**, check the key's automatic rotation status in AWS — and do not schedule rotation through the RustFS endpoint, which maps to quota-limited `RotateKeyOnDemand`.
|
||||
4. Know the gauge's blind spot on Transit and AWS before chasing a rotation that already happened: only KV2 persists a rotation timestamp, so Transit and AWS keys age from creation permanently and this alert will not clear after a rotation there. Confirm the real cadence at the owning system — the Transit key's version history in Vault, or the key's rotation status in AWS — and treat a confirmed-healthy cadence as a known overstatement of this gauge rather than an overdue key.
|
||||
5. If a KV2 key was genuinely rotated and the gauge stays high, remember the gauge is republished only by a sweep that saw the whole key set: check `rustfs_kms_deletion_sweep_keys_total` for `unreadable` or `failed` outcomes freezing the lifecycle gauges (see [Key lifecycle metrics](#key-lifecycle-metrics)), and that the deletion worker is running at all — it only runs on backends with the `schedule_deletion` capability, which is also why the Static backend never emits this series.
|
||||
|
||||
Related signals: `rotation_due` / `rotation_due_reason` on the key listing; `rustfs_kms_deletion_sweep_keys_total{outcome=~"unreadable|failed"}` (a frozen gauge is stale, not healthy); the [rotation drivers and scheduling matrix](kms-backend-security.md#rotation-drivers-and-scheduling-per-backend) and pre-rotation checklist in the backend security properties document.
|
||||
|
||||
## Threshold calibration
|
||||
|
||||
Every numeric traffic or latency threshold in `rustfs-kms-alerts.yml` (5% error ratio, 2s p99, 0.5/s attempt failures, 0.05/s budget exhaustion) is a conservative default chosen without a production baseline, biased toward not paging on healthy-but-busy systems. Before relying on these alerts for paging: run the workload in staging for at least a week, record the steady-state values of the expressions above, then tighten thresholds to sit clearly above observed peaks. `KmsBackendCircuitOpen` is different: its gauge is direct state, and the one-minute hold only suppresses a circuit that recovers immediately. `KmsKeyRotationOverdue` is different in the other direction: its 400-day threshold is a policy default (sitting above a common one-year rotation period), not a traffic default — calibrate it against the rotation period your compliance policy requires and against `RUSTFS_KMS_ROTATION_MAX_AGE_SECS`, not against a staging baseline. Once a stable baseline exists, consider converting `KmsBackendAttemptFailureSpike` to a baseline-relative form (`offset 1d` ratio, see `.docker/observability/prometheus-rules/rustfs-get-optimization-alerts.yaml` for the pattern). Formal SLO targets for KMS operations are deliberately out of scope until that baseline exists (rustfs/backlog#1584).
|
||||
Every numeric traffic or latency threshold in `rustfs-kms-alerts.yml` (5% error ratio, 2s p99, 0.5/s attempt failures, 0.05/s budget exhaustion) is a conservative default chosen without a production baseline, biased toward not paging on healthy-but-busy systems. Before relying on these alerts for paging: run the workload in staging for at least a week, record the steady-state values of the expressions above, then tighten thresholds to sit clearly above observed peaks. `KmsBackendCircuitOpen` is different: its gauge is direct state, and the one-minute hold only suppresses a circuit that recovers immediately. Once a stable baseline exists, consider converting `KmsBackendAttemptFailureSpike` to a baseline-relative form (`offset 1d` ratio, see `.docker/observability/prometheus-rules/rustfs-get-optimization-alerts.yaml` for the pattern). Formal SLO targets for KMS operations are deliberately out of scope until that baseline exists (rustfs/backlog#1584).
|
||||
|
||||
## Coverage gaps
|
||||
|
||||
The four metric families designed under rustfs/backlog#1584 — key-cache effectiveness, key lifecycle, Vault credentials, synthetic probe — have all landed and are documented in [Metric reference](#metric-reference). What is still missing:
|
||||
|
||||
- **No dashboard panels for those four families, and an alert rule for only one of them.** The key lifecycle family has one rule — [`KmsKeyRotationOverdue`](#kmskeyrotationoverdue) on the rotation-age gauge — while the cache, Vault credential, and probe families are emitted but neither visualized nor alerted on, so they surface only in ad-hoc queries. Building against them is safe now: the names and label values above are what the code emits.
|
||||
- **No dashboard panels and no alert rules for those four families.** They are emitted but neither visualized nor alerted on, so they surface only in ad-hoc queries. Building against them is safe now: the names and label values above are what the code emits.
|
||||
- **The Local and Static backends emit no operation metrics**, because they do not flow through the operation-policy choke point; bringing them under the same instrumentation is tracked separately (rustfs/backlog#1569). Their cache metrics are emitted normally.
|
||||
- **No formal SLO targets**, deliberately, until a production baseline exists — see [Threshold calibration](#threshold-calibration).
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@
|
||||
| list_objects_v2_metadata_extension_test | 1 | |
|
||||
| list_objects_v2_pagination_test | 12 | ✅ |
|
||||
| mc_mirror_small_bucket_test | 1 | |
|
||||
| multipart_auth_test | 103 | |
|
||||
| multipart_auth_test | 109 | |
|
||||
| multipart_storage_class_test | 3 | ✅ |
|
||||
| namespace_lock_quorum_test | 2 | |
|
||||
| negative_sigv4_test | 6 | ✅ |
|
||||
|
||||
Generated
+12
-23
@@ -908,15 +908,6 @@ dependencies = [
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "castaway"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a"
|
||||
dependencies = [
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
@@ -1004,19 +995,6 @@ version = "0.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
|
||||
|
||||
[[package]]
|
||||
name = "compact_str"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416"
|
||||
dependencies = [
|
||||
"castaway",
|
||||
"cfg-if",
|
||||
"itoa",
|
||||
"static_assertions",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "combine"
|
||||
version = "4.6.7"
|
||||
@@ -4363,7 +4341,6 @@ name = "rustfs-lock"
|
||||
version = "1.0.0-beta.11"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"compact_str",
|
||||
"crossbeam-queue",
|
||||
"futures",
|
||||
"parking_lot",
|
||||
@@ -4372,6 +4349,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
"smartstring",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tonic",
|
||||
@@ -5156,6 +5134,17 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smartstring"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"static_assertions",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "snap"
|
||||
version = "1.1.2"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user