gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-11-01 16:50:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/gateway/integration' into gw/T0.3-test-count-ratchet

Browse Source
This commit is contained in:
overtrue
2026-10-08 21:36:23 +08:00
parent 1302ca0893 484f1254ca
commit 4e28cafe02
159 changed files with 9817 additions and 3624 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+99 -6
View File
@@ -16,7 +16,9 @@ name: Continuous Integration
on:
push:
branches: [ main ]
# gateway/integration receives the daily merge of main as a direct push
# (rustfs/backlog#2734), so its pushes must prove the result like main's.
branches: [ main, gateway/integration ]
paths-ignore:
- "**.md"
- "docs/**"
@@ -36,7 +38,9 @@ on:
- "flake.lock"
pull_request:
types: [ opened, synchronize, reopened, closed ]
branches: [ main ]
# gateway/integration carries main's ruleset (rustfs/backlog#2734), so its
# pull requests must produce the same required checks.
branches: [ main, gateway/integration ]
merge_group:
types: [ checks_requested ]
schedule:
@@ -778,6 +782,8 @@ jobs:
needs: [ build-rustfs-debug-binary ]
runs-on: sm-standard-2
timeout-minutes: 30
env:
RUSTFS_S3_STACK: legacy
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -896,18 +902,105 @@ jobs:
path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
retention-days: 3
# The same e2e-smoke suite against the same debug binary with the gateway
# S3 stack selected (rustfs/backlog#2734). Report-only: it is outside the
# Test and Lint aggregate and becomes required at T2.10. The archive and run
# steps must stay identical to e2e-tests; scripts/ci_gate.py enforces that.
e2e-tests-gateway:
name: e2e-smoke (gateway, report-only until T2.10)
needs: [ build-rustfs-debug-binary ]
runs-on: sm-standard-2
timeout-minutes: 30
continue-on-error: true
env:
RUSTFS_S3_STACK: gateway
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-dev
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: Install awscurl
run: |
python3 -m pip install --user --upgrade pip "awscurl==0.44"
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
- name: Verify awscurl
run: test -x "$AWSCURL_PATH"
- name: Download debug binary
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: rustfs-debug-binary
path: target/debug
- name: Make binary executable
run: chmod +x ./target/debug/rustfs
- name: Archive e2e smoke test binaries
env:
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
run: |
cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
- name: Run e2e smoke suite
env:
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
SAMPLER_INTERVAL_SECS: "15"
run: |
./scripts/ci/resource_sampler.sh start e2e-smoke
trap './scripts/ci/resource_sampler.sh stop' EXIT
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
--status-level all --final-status-level all --failure-output final
- name: Upload e2e smoke diagnostics
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-smoke-gateway-diagnostics-${{ github.run_number }}
path: |
${{ runner.temp }}/rustfs-e2e-smoke-logs/
${{ runner.temp }}/rustfs-e2e-smoke-list.json
if-no-files-found: warn
- name: Upload e2e smoke JUnit report
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-smoke-gateway-junit-${{ github.run_number }}
path: target/nextest/e2e-smoke/junit.xml
if-no-files-found: warn
e2e-full:
name: End-to-End Tests (full merge gate)
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
# suites — KMS, object_lock, multipart_auth, quota, checksum, encryption,
# security-boundary, ... — via the e2e-full nextest profile. Too heavy for
# every PR, so it is gated to main pushes, the merge queue, and manual
# dispatch. protocols / the 7 cluster suites / replication / #[ignore] are
# owned by other lanes (see .config/nextest.toml profile.e2e-full).
# every PR, so it is gated to main and gateway/integration pushes, the merge
# queue, and manual dispatch. protocols / the 7 cluster suites / replication
# / #[ignore] are owned by other lanes (see .config/nextest.toml profile.e2e-full).
if: >-
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
(github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/gateway/integration'))
needs: [ build-rustfs-debug-binary ]
runs-on: sm-standard-2
timeout-minutes: 55
.github/workflows/gateway-integration-sync.yml
+197
View File
@@ -0,0 +1,197 @@
# Copyright 2026 RustFS Team
# SPDX-License-Identifier: Apache-2.0
name: Gateway Integration Sync
# Merges `main` into the long-lived `gateway/integration` branch every weekday
# so the s3s -> gateway migration (rustfs/backlog#2734, task T0.1 in #2736)
# never accumulates merge debt. A clean merge lands as a `--no-ff` merge commit
# pushed straight to the branch. A conflicting merge is aborted and surfaced as
# a pull request from `sync/<source>-<date>`, whose head is the un-merged
# source commit, so the conflict is resolved the same day, in review, with no
# other change riding along. The run itself stays green in that case: the pull
# request is the signal. Any other failure (push rejected, pull request not
# created) fails the run and, on the schedule, files the shared failure issue.
#
# Resolving a conflict pull request: merge `origin/gateway/integration` into
# the sync branch, resolve, push, get the review. Land it with a fast-forward
# push of the resolved merge commit to `gateway/integration`, not with the
# squash button: a squash drops the merge ancestry and the same hunks conflict
# again on the next run.
#
# Write scope: only `gateway/integration` and `sync/*` are ever pushed, and
# both push targets below are literal refs. The pushes use a one-hour
# installation token of the organisation's own GitHub App (contents: write)
# because that app is the only non-admin bypass actor of the
# `gateway/integration` ruleset; GITHUB_TOKEN cannot be a ruleset bypass actor,
# so it cannot push through the required reviews and checks. The conflict pull
# request is opened with FLAKE_UPDATE_TOKEN, the PAT that already opens the
# weekly flake.lock pull requests, because this repository keeps "Allow GitHub
# Actions to create and approve pull requests" switched off. GITHUB_TOKEN is
# therefore read-only here.
#
# `schedule` fires only from the default branch, so the cron is live once this
# file exists on `main`. `workflow_dispatch` runs from any ref that carries the
# file and always acts on `gateway/integration`.
on:
schedule:
- cron: "0 2 * * 1-5" # Weekdays 02:00 UTC
workflow_dispatch:
inputs:
source_ref:
description: >-
Branch to merge into gateway/integration. The schedule always merges
main; only the conflict acceptance test (rustfs/backlog#2736) sets
another branch.
required: false
default: main
type: string
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}
cancel-in-progress: false
env:
TARGET_BRANCH: gateway/integration
jobs:
sync:
name: Merge main into gateway/integration
if: github.repository == 'rustfs/rustfs'
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Create the branch push token
id: push-token
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3
with:
app-id: ${{ vars.CLA_BOT_APP_ID }}
private-key: ${{ secrets.CLA_BOT_APP_PRIVATE_KEY }}
owner: ${{ github.repository_owner }}
repositories: rustfs
permission-contents: write
- name: Checkout gateway/integration
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
# persist-credentials-exempt: this checkout's token IS the push
# credential — the merge commit and the sync branch are pushed with it
# below. It is a one-hour installation token scoped to contents.
ref: gateway/integration
fetch-depth: 0
token: ${{ steps.push-token.outputs.token }}
- name: Configure the bot identity
run: |
git config user.name "rustfs-cla-assistant[bot]"
git config user.email "273121464+rustfs-cla-assistant[bot]@users.noreply.github.com"
- name: Merge the source branch
id: merge
env:
SOURCE_REF: ${{ inputs.source_ref || 'main' }}
run: |
set -euo pipefail
if ! [[ "$SOURCE_REF" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] \
|| [[ "$SOURCE_REF" == *..* ]] \
|| [[ "$SOURCE_REF" == "$TARGET_BRANCH" ]]; then
echo "::error::refusing source_ref '${SOURCE_REF}'"
exit 1
fi
git fetch origin "refs/heads/${SOURCE_REF}:refs/remotes/origin/${SOURCE_REF}"
source_sha=$(git rev-parse "origin/${SOURCE_REF}")
target_sha=$(git rev-parse HEAD)
{
echo "source_ref=${SOURCE_REF}"
echo "source_sha=${source_sha}"
echo "target_sha=${target_sha}"
} >> "$GITHUB_OUTPUT"
if git merge --no-ff --no-edit "origin/${SOURCE_REF}"; then
if [[ "$(git rev-parse HEAD)" == "$target_sha" ]]; then
echo "result=up-to-date" >> "$GITHUB_OUTPUT"
echo "\`${TARGET_BRANCH}\` already contains \`${SOURCE_REF}\` (${source_sha}); nothing to push." >> "$GITHUB_STEP_SUMMARY"
else
echo "result=merged" >> "$GITHUB_OUTPUT"
fi
exit 0
fi
git diff --name-only --diff-filter=U > "$RUNNER_TEMP/conflicts.txt"
git merge --abort
if [[ ! -s "$RUNNER_TEMP/conflicts.txt" ]]; then
echo "::error::git merge failed without reporting conflicting paths; this is not a conflict"
exit 1
fi
echo "result=conflict" >> "$GITHUB_OUTPUT"
- name: Push the merge commit
if: steps.merge.outputs.result == 'merged'
env:
SOURCE_REF: ${{ steps.merge.outputs.source_ref }}
SOURCE_SHA: ${{ steps.merge.outputs.source_sha }}
run: |
set -euo pipefail
git push origin HEAD:refs/heads/gateway/integration
echo "Merged \`${SOURCE_REF}\` (${SOURCE_SHA}) into \`${TARGET_BRANCH}\` as $(git rev-parse HEAD)." >> "$GITHUB_STEP_SUMMARY"
- name: Open the conflict pull request
if: steps.merge.outputs.result == 'conflict'
env:
GH_TOKEN: ${{ secrets.FLAKE_UPDATE_TOKEN }}
SOURCE_REF: ${{ steps.merge.outputs.source_ref }}
SOURCE_SHA: ${{ steps.merge.outputs.source_sha }}
TARGET_SHA: ${{ steps.merge.outputs.target_sha }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
set -euo pipefail
date_utc=$(date -u +%Y-%m-%d)
sync_branch="sync/${SOURCE_REF//\//-}-${date_utc}"
git push origin "refs/remotes/origin/${SOURCE_REF}:refs/heads/${sync_branch}"
{
echo "The scheduled merge of \`${SOURCE_REF}\` into \`${TARGET_BRANCH}\` stopped on conflicts and was aborted; \`${TARGET_BRANCH}\` is unchanged."
echo
echo "This branch is the un-merged \`${SOURCE_REF}\` head. Merge \`origin/${TARGET_BRANCH}\` into it, resolve the files below, push, and get the review. Land the resolved merge commit with a fast-forward push to \`${TARGET_BRANCH}\`; a squash merge drops the merge ancestry and the same hunks conflict again on the next run. Do not include any change that is not conflict resolution."
echo
echo "- Run: ${RUN_URL}"
echo "- Source: \`${SOURCE_REF}\` at ${SOURCE_SHA}"
echo "- Target: \`${TARGET_BRANCH}\` at ${TARGET_SHA}"
echo
echo "## Conflicting files"
echo
echo "Output of \`git diff --name-only --diff-filter=U\` captured before \`git merge --abort\`:"
echo
sed 's/^/- /' "$RUNNER_TEMP/conflicts.txt"
} > "$RUNNER_TEMP/pr-body.md"
title="sync: merge ${SOURCE_REF} ${date_utc} (conflicts)"
existing=$(gh pr list --repo "$GITHUB_REPOSITORY" --base "$TARGET_BRANCH" --head "$sync_branch" --state open --json number --jq '.[0].number // empty')
if [[ -n "$existing" ]]; then
gh pr edit "$existing" --repo "$GITHUB_REPOSITORY" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md"
pr_url=$(gh pr view "$existing" --repo "$GITHUB_REPOSITORY" --json url --jq .url)
else
pr_url=$(gh pr create --repo "$GITHUB_REPOSITORY" --base "$TARGET_BRANCH" --head "$sync_branch" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md")
fi
echo "Conflicts in \`${SOURCE_REF}\` -> \`${TARGET_BRANCH}\`; pull request: ${pr_url}" >> "$GITHUB_STEP_SUMMARY"
alert-on-failure:
name: Alert on scheduled failure
needs: [sync]
# Same ci-8 mechanism audit.yml and coverage.yml use: a scheduled run that
# could not push or could not open the conflict pull request must not stay
# visible only in the Actions tab.
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
Cargo.lock
Generated
+6 -4
View File
@@ -10078,9 +10078,9 @@ dependencies = [
[[package]]
name = "rustfs-kafka"
version = "1.3.1"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "759f8ddf709b497006e0f422a89890f11ac86c84c2c39df282891477c22f1a13"
checksum = "bd2f8b46ec4dd462c2235948d1efa2103957230c171e118c6ff8e5d1a845ab48"
dependencies = [
"base64 0.23.1",
"bytes",
@@ -10103,9 +10103,9 @@ dependencies = [
[[package]]
name = "rustfs-kafka-async"
version = "1.3.1"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aa574a13d91ef529e3a08f059a5c3ee8d4c3b16cf62f62c72595a6dc4f79f8f"
checksum = "4b3b98705f6f9f5071b8793d29935de764c31668c6e9877553b229eb67127af6"
dependencies = [
"base64 0.23.1",
"bytes",
@@ -10703,6 +10703,8 @@ name = "rustfs-s3-types"
version = "1.0.1"
dependencies = [
"hotpath",
"http 1.5.0",
"s3s",
"serde",
"serde_json",
]
Cargo.toml
+6 -2
View File
@@ -48,7 +48,7 @@ members = [
"crates/replication", # Replication contracts and wire formats
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
"crates/s3-client", # S3 client for engine-side consumption of remote S3 endpoints (tiering, transition targets)
"crates/s3-types", # S3 event type definitions
"crates/s3-types", # S3 event and error type definitions
"crates/s3-ops", # S3 operation definitions and mapping
"crates/s3select-api", # S3 Select API interface
"crates/s3select-query", # S3 Select query engine
@@ -166,7 +166,7 @@ http-body-util = "0.1.5"
memchr = "2.8.3"
minlz = "1.2.3"
reqwest = "0.13.5"
rustfs-kafka-async = { version = "1.3.1" }
rustfs-kafka-async = { version = "1.4.0" }
socket2 = { version = "0.6.5" }
tokio = { version = "1.53.2" }
tokio-rustls = { default-features = false, version = "0.26.6" }
@@ -323,6 +323,10 @@ rust-embed = { version = "8.12.0" }
rustc-hash = { version = "2.1.3" }
s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" }
# The same s3s pin under the name the legacy edge of the gateway migration uses
# (crates/s3-types, feature `compat-s3s`). Keep `rev` identical to `s3s` above.
# DELETE BY T4.2 (rustfs/backlog#2784).
legacy-s3s = { package = "s3s", git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
serial_test = "4.0.1"
shadow-rs = { default-features = false, version = "2.0.0" }
siphasher = "1.0.4"
crates/e2e_test/src/fake_s3_target/mod.rs
+329 -14
View File
@@ -50,8 +50,8 @@ use std::collections::{BTreeMap, HashMap, VecDeque};
use std::error::Error;
use std::io;
use std::net::SocketAddr;
use std::sync::{Arc, Mutex, MutexGuard};
use std::time::{Duration, SystemTime};
use std::sync::{Arc, Mutex, MutexGuard, OnceLock};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use tokio::net::TcpListener;
use tokio::sync::{OwnedSemaphorePermit, Semaphore, watch};
use tokio::task::JoinHandle;
@@ -363,6 +363,14 @@ pub struct RequestRecord {
pub upload_id: Option<String>,
pub part_number: Option<i32>,
pub content_length: Option<u64>,
pub source_version_id: Option<String>,
pub source_mtime: Option<String>,
pub source_etag: Option<String>,
pub source_replication_request: bool,
pub journaled_at_unix_millis: Option<u64>,
/// Computed by the target, before response stall/disconnect faults. This
/// does not prove that any response bytes reached the client.
pub prepared_response: Option<PreparedResponse>,
pub consumed_bytes: Option<usize>,
pub replication_timestamps: ReplicationTimestampHeaders,
pub proxy_headers: ProxyHeaderSnapshot,
@@ -381,6 +389,20 @@ pub struct RequestRecord {
pub fault: Option<FaultAction>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PreparedResponse {
pub status: u16,
pub version_id: Option<String>,
pub etag: Option<String>,
pub last_modified: Option<String>,
pub content_length: Option<u64>,
pub prepared_at_unix_millis: Option<u64>,
pub elapsed: Duration,
}
#[derive(Clone, Default)]
struct RequestSequence(Arc<OnceLock<u64>>);
/// Integrity and framing headers of an upload. A remote target's acceptance
/// rules key off exactly these (rustfs#6853: `aws-chunked` framing stored
/// verbatim; rustfs#7082: Object Lock PUTs need `Content-MD5` or
@@ -430,6 +452,10 @@ impl TransportSnapshot {
#[derive(Debug, Clone, Default, PartialEq, Eq)]
struct JournaledHeaders {
content_length: Option<u64>,
source_version_id: Option<String>,
source_mtime: Option<String>,
source_etag: Option<String>,
source_replication_request: bool,
replication_timestamps: ReplicationTimestampHeaders,
proxy_headers: ProxyHeaderSnapshot,
transport: TransportSnapshot,
@@ -444,6 +470,10 @@ impl JournaledHeaders {
.get(CONTENT_LENGTH)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.parse().ok()),
source_version_id: header_value(headers, &SOURCE_VERSION_ID_HEADERS).map(bounded_journal_value),
source_mtime: header_value(headers, &SOURCE_MTIME_HEADERS).map(bounded_journal_value),
source_etag: header_value(headers, &SOURCE_ETAG_HEADERS).map(bounded_journal_value),
source_replication_request: header_value(headers, &SOURCE_REPLICATION_REQUEST_HEADERS).as_deref() == Some("true"),
replication_timestamps: ReplicationTimestampHeaders::from_headers(headers),
proxy_headers: ProxyHeaderSnapshot::from_headers(headers),
transport: TransportSnapshot::from_headers(headers),
@@ -742,6 +772,7 @@ impl FakeS3Target {
let connection_limit = Arc::new(Semaphore::new(MAX_CONNECTIONS));
let (active_connections_tx, active_connections) = watch::channel(0usize);
let task_connection_limit = Arc::clone(&connection_limit);
let task_control = Arc::clone(&control);
let task = tokio::spawn(async move {
let mut connections = tokio::task::JoinSet::new();
@@ -755,6 +786,7 @@ impl FakeS3Target {
};
active_connections_tx.send_modify(|count| *count += 1);
let service = service.clone();
let control = Arc::clone(&task_control);
let active_connections = active_connections_tx.clone();
let mut connection_gate = connection_gate.clone();
connections.spawn(async move {
@@ -768,7 +800,7 @@ impl FakeS3Target {
{
return;
}
let handler = service_fn(move |request| handle_request(request, service.clone()));
let handler = service_fn(move |request| handle_request(request, service.clone(), Arc::clone(&control)));
let mut connection = http1::Builder::new();
connection
.keep_alive(false)
@@ -1202,7 +1234,10 @@ impl S3Access for FaultAccess {
let operation = operation_from_s3_name(context.s3_op().name());
let journaled = JournaledHeaders::from_headers(context.headers());
let aws_chunked = journaled.transport.aws_chunked;
let fault = record_request(&self.control, operation, context.method().clone(), parsed, journaled);
let (sequence, fault) = record_request(&self.control, operation, context.method().clone(), parsed, journaled);
if let Some(slot) = context.extensions_mut().get::<RequestSequence>() {
let _ = slot.0.set(sequence);
}
if let Some(status) = fault.as_ref().and_then(|fault| scripted_status(&fault.action)) {
return Err(scripted_status_error(status));
}
@@ -1278,7 +1313,7 @@ fn record_request(
method: Method,
parsed: ParsedRequest,
headers: JournaledHeaders,
) -> Option<RequestFault> {
) -> (u64, Option<RequestFault>) {
let mut state = lock(control);
let action = parsed
.key
@@ -1301,6 +1336,12 @@ fn record_request(
upload_id: parsed.upload_id.map(bounded_journal_value),
part_number: parsed.part_number,
content_length: headers.content_length,
source_version_id: headers.source_version_id,
source_mtime: headers.source_mtime,
source_etag: headers.source_etag,
source_replication_request: headers.source_replication_request,
journaled_at_unix_millis: unix_millis(),
prepared_response: None,
consumed_bytes: None,
replication_timestamps: headers.replication_timestamps,
proxy_headers: headers.proxy_headers,
@@ -1311,7 +1352,14 @@ fn record_request(
continuation_token: parsed.continuation_token.map(bounded_journal_value),
fault: action.clone(),
});
action.map(|action| RequestFault { sequence, action })
(sequence, action.map(|action| RequestFault { sequence, action }))
}
fn unix_millis() -> Option<u64> {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.ok()
.and_then(|elapsed| u64::try_from(elapsed.as_millis()).ok())
}
fn bounded_journal_value(mut value: String) -> String {
@@ -1319,14 +1367,31 @@ fn bounded_journal_value(mut value: String) -> String {
while !value.is_char_boundary(end) {
end -= 1;
}
value.truncate(end);
value
if value.capacity() > MAX_RETAINED_IDENTIFIER_BYTES {
value[..end].to_owned()
} else {
value.truncate(end);
value
}
}
async fn handle_request(request: Request<Incoming>, service: S3Service) -> Result<Response<Body>, BoxError> {
let mut response = timeout(MAX_REQUEST_DURATION, call_s3(service, request.map(Body::from)))
.await
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "fake target request exceeded 65 seconds"))??;
async fn handle_request(
mut request: Request<Incoming>,
service: S3Service,
control: Arc<Mutex<ControlState>>,
) -> Result<Response<Body>, BoxError> {
let started = Instant::now();
let sequence = RequestSequence::default();
request.extensions_mut().insert(sequence.clone());
let mut response = timeout(MAX_REQUEST_DURATION, async {
let result = call_s3(service, request.map(Body::from)).await;
if let (Ok(response), Some(sequence)) = (&result, sequence.0.get()) {
update_prepared_response(&control, *sequence, response, started.elapsed());
}
result
})
.await
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "fake target request exceeded 65 seconds"))??;
if response.headers_mut().remove(DISCONNECT_HEADER).is_some() {
return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "fake target scripted disconnect").into());
}
@@ -1581,6 +1646,25 @@ fn update_consumed(control: &Mutex<ControlState>, sequence: u64, consumed: usize
}
}
fn update_prepared_response(control: &Mutex<ControlState>, sequence: u64, response: &Response<Body>, elapsed: Duration) {
let prepared = PreparedResponse {
status: response.status().as_u16(),
version_id: header_value(response.headers(), &["x-amz-version-id"]).map(bounded_journal_value),
etag: header_value(response.headers(), &[ETAG.as_str()]).map(bounded_journal_value),
last_modified: header_value(response.headers(), &[LAST_MODIFIED.as_str()]).map(bounded_journal_value),
content_length: response
.headers()
.get(CONTENT_LENGTH)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.parse().ok()),
prepared_at_unix_millis: unix_millis(),
elapsed,
};
if let Some(record) = lock(control).requests.iter_mut().find(|record| record.sequence == sequence) {
record.prepared_response = Some(prepared);
}
}
async fn apply_non_body_fault(fault: Option<&RequestFault>, control: &Mutex<ControlState>) -> S3Result<()> {
match fault.map(|fault| &fault.action) {
Some(action @ (FaultAction::Status(_) | FaultAction::ResponseStatus(_))) => {
@@ -3280,10 +3364,38 @@ mod tests {
let retry = client.clone();
requests.spawn(async move { get_bytes(&retry, bucket, "held", None).await });
timeout(Duration::from_secs(2), gate.entered.wait_for(|count| *count == 2)).await??;
let held_retry = target
.requests()
.into_iter()
.rev()
.find(|record| record.operation == Operation::GetObject && record.key.as_deref() == Some("held"))
.expect("the held retry must be journaled");
assert_eq!(held_retry.prepared_response, None);
assert_eq!(
timeout(Duration::from_secs(2), get_bytes(&client, bucket, "unrelated", None)).await??,
Bytes::from_static(b"payload")
);
let unrelated = target
.requests()
.into_iter()
.find(|record| record.operation == Operation::GetObject && record.key.as_deref() == Some("unrelated"))
.expect("the unrelated GET must be journaled");
let unrelated_response = unrelated.prepared_response.expect("the unrelated GET prepared a response");
assert_eq!(unrelated_response.status, 200);
assert_eq!(
unrelated_response.version_id.as_deref(),
Some(target.stored_versions(bucket, "unrelated")[0].0.as_str())
);
assert!(held_retry.sequence < unrelated.sequence);
assert_eq!(
target
.requests()
.into_iter()
.find(|record| record.sequence == held_retry.sequence)
.expect("the held retry remains journaled")
.prepared_response,
None
);
assert!(requests.try_join_next().is_none(), "the retry must remain behind the gate");
drop(gate);
assert_eq!(
@@ -3292,6 +3404,18 @@ mod tests {
.expect("retried GET task")??,
Bytes::from_static(b"payload")
);
let held_response = target
.requests()
.into_iter()
.find(|record| record.sequence == held_retry.sequence)
.and_then(|record| record.prepared_response)
.expect("the released retry prepared its own response");
assert_eq!(held_response.status, 200);
assert_eq!(
held_response.version_id.as_deref(),
Some(target.stored_versions(bucket, "held")[0].0.as_str())
);
assert_ne!(held_response.version_id, unrelated_response.version_id);
assert_eq!(get_bytes(&client, bucket, "held", None).await?, Bytes::from_static(b"payload"));
assert_eq!(target.count_requests(Operation::GetObject, "held"), 3);
Ok(())
@@ -3636,6 +3760,23 @@ mod tests {
.await?;
assert_eq!(first_head.content_length(), Some(5));
assert_eq!(first_head.version_id(), first.version_id.as_deref());
let journaled_head = target
.requests()
.into_iter()
.rev()
.find(|record| record.operation == Operation::HeadObject)
.expect("the version HEAD must be journaled");
assert_eq!(journaled_head.version_id, first.version_id);
assert!(journaled_head.journaled_at_unix_millis.is_some());
let prepared_head = journaled_head
.prepared_response
.expect("the version HEAD prepared a response");
assert_eq!(prepared_head.status, 200);
assert_eq!(prepared_head.version_id.as_deref(), first_head.version_id());
assert_eq!(prepared_head.etag.as_deref(), first_head.e_tag());
assert_eq!(prepared_head.content_length, Some(5));
assert!(prepared_head.last_modified.is_some());
assert!(prepared_head.prepared_at_unix_millis.is_some());
assert_eq!(
get_bytes(&client, "target-bucket", "nested/key", None).await?,
Bytes::from_static(b"second")
@@ -3661,6 +3802,20 @@ mod tests {
.await
.expect_err("deleted version must not remain readable");
assert_sdk_error!(deleted, 404, "NoSuchVersion");
let deleted_record = target
.requests()
.into_iter()
.rev()
.find(|record| record.operation == Operation::GetObject)
.expect("the deleted version GET must be journaled");
assert_eq!(deleted_record.version_id, first.version_id);
assert_eq!(
deleted_record
.prepared_response
.expect("the error response was prepared")
.status,
404
);
let marker = client
.delete_object()
.bucket("target-bucket")
@@ -4092,7 +4247,21 @@ mod tests {
.await
.expect_err("wrong part ETag must fail without consuming the upload");
assert_sdk_error!(invalid_complete, 400, "InvalidPart");
client
let rejected_complete = target
.requests()
.into_iter()
.rev()
.find(|record| record.operation == Operation::CompleteMultipartUpload)
.expect("the rejected completion must be journaled");
assert_eq!(rejected_complete.upload_id.as_deref(), Some(upload_id.as_str()));
assert_eq!(
rejected_complete
.prepared_response
.expect("the rejection was prepared")
.status,
400
);
let successful_complete = client
.complete_multipart_upload()
.bucket("target-bucket")
.key("multipart")
@@ -4100,6 +4269,18 @@ mod tests {
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(parts)).build())
.send()
.await?;
let completed_record = target
.requests()
.into_iter()
.rev()
.find(|record| record.operation == Operation::CompleteMultipartUpload)
.expect("the successful completion must be journaled");
assert_eq!(completed_record.upload_id.as_deref(), Some(upload_id.as_str()));
let prepared_complete = completed_record
.prepared_response
.expect("the completion response was prepared");
assert_eq!(prepared_complete.status, 200);
assert_eq!(prepared_complete.version_id.as_deref(), successful_complete.version_id());
let completed = get_bytes(&client, "target-bucket", "multipart", None).await?;
assert_eq!(completed.len(), MIN_MULTIPART_PART_BYTES + 5);
assert_eq!(&completed[..MIN_MULTIPART_PART_BYTES], first_body.as_ref());
@@ -4480,6 +4661,32 @@ mod tests {
.send()
.await?;
assert_eq!(completed.version_id(), Some(completed_version.as_str()));
let receipt_records = target.requests();
let receipt_create = receipt_records
.iter()
.find(|record| {
record.operation == Operation::CreateMultipartUpload && record.key.as_deref() == Some("multipart-receipt-time")
})
.expect("the source version must be retained on MPU creation");
assert_eq!(receipt_create.source_version_id.as_deref(), Some(completed_version.as_str()));
let receipt_complete = receipt_records
.iter()
.find(|record| {
record.operation == Operation::CompleteMultipartUpload
&& record.upload_id.as_deref() == Some(receipt_upload.as_str())
})
.expect("the source mtime must be retained on completion");
assert_eq!(receipt_complete.source_mtime.as_deref(), Some("1970-01-01T00:00:00Z"));
assert!(receipt_complete.source_replication_request);
assert_eq!(
receipt_complete
.prepared_response
.as_ref()
.expect("the receipt-time response was prepared")
.version_id
.as_deref(),
completed.version_id()
);
assert_eq!(
get_bytes(&client, "target-bucket", "multipart-receipt-time", None).await?,
Bytes::from_static(b"completed"),
@@ -4698,6 +4905,23 @@ mod tests {
.expect_err("disconnected request must not store an object");
assert_sdk_error!(disconnected, 404, "NotFound");
target.inject_for_key(Operation::PutObject, "lost-response", FaultAction::DisconnectAfterResponse, 1);
client
.put_object()
.bucket("target-bucket")
.key("lost-response")
.body(ByteStream::from_static(b"stored"))
.send()
.await
.expect_err("a prepared response does not prove client delivery");
let lost_response_head = client
.head_object()
.bucket("target-bucket")
.key("lost-response")
.send()
.await?;
assert_eq!(lost_response_head.content_length(), Some(6));
assert_eq!(
parse_request(&Method::PUT, &"/target-bucket/key?uploadId=upload&partNumber=bad".parse::<Uri>()?).operation,
Operation::Unknown
@@ -4711,6 +4935,28 @@ mod tests {
.count(),
4
);
assert!(
requests
.iter()
.filter(|request| request.fault == Some(FaultAction::Status(StatusCode::SERVICE_UNAVAILABLE)))
.all(|request| {
request
.prepared_response
.as_ref()
.is_some_and(|response| response.status == 503)
})
);
let lost_response = requests
.iter()
.find(|request| request.operation == Operation::PutObject && request.key.as_deref() == Some("lost-response"))
.expect("the disconnected response must be journaled");
assert_eq!(lost_response.fault, Some(FaultAction::DisconnectAfterResponse));
let prepared_lost_response = lost_response
.prepared_response
.as_ref()
.expect("the target prepared the successful response");
assert_eq!(prepared_lost_response.status, 200);
assert_eq!(prepared_lost_response.version_id.as_deref(), lost_response_head.version_id());
assert!(
requests
.iter()
@@ -4901,8 +5147,44 @@ mod tests {
assert_eq!(records.len(), MAX_REQUEST_RECORDS);
assert_eq!(records.front().map(|record| record.sequence), Some(2));
assert_eq!(records.back().map(|record| record.sequence), Some((MAX_REQUEST_RECORDS + 1) as u64));
let mut oversized_allocation = String::with_capacity(MAX_RETAINED_IDENTIFIER_BYTES * 16);
oversized_allocation.push_str("short");
let short_value = bounded_journal_value(oversized_allocation);
assert_eq!(short_value, "short");
assert!(short_value.capacity() <= MAX_RETAINED_IDENTIFIER_BYTES);
let mut response = Response::new(Body::empty());
for name in ["x-amz-version-id", "etag", "last-modified"] {
response.headers_mut().insert(
name,
HeaderValue::from_str(&"r".repeat(MAX_RETAINED_IDENTIFIER_BYTES + 1)).expect("header"),
);
}
response.headers_mut().insert(CONTENT_LENGTH, HeaderValue::from_static("17"));
response
.headers_mut()
.insert("authorization", HeaderValue::from_static("unrecorded-response-authorization"));
response.headers_mut().insert(
"x-amz-server-side-encryption-customer-key",
HeaderValue::from_static("unrecorded-response-customer-key"),
);
update_prepared_response(&control, 1, &response, Duration::ZERO);
update_prepared_response(&control, u64::MAX, &response, Duration::ZERO);
assert_eq!(lock(&control).requests, records, "misses and evicted records must not be resurrected");
let bounded_control = Mutex::new(ControlState::default());
let utf8_boundary = format!("{}é", "a".repeat(MAX_RETAINED_IDENTIFIER_BYTES - 1));
let mut source_headers = HeaderMap::new();
for name in ["x-rustfs-source-version-id", "x-minio-source-mtime", "x-rustfs-source-etag"] {
source_headers.insert(
name,
HeaderValue::from_str(&"s".repeat(MAX_RETAINED_IDENTIFIER_BYTES + 1)).expect("header"),
);
}
source_headers.insert("x-minio-source-replication-request", HeaderValue::from_static("true"));
source_headers.insert("authorization", HeaderValue::from_static("unrecorded-request-authorization"));
source_headers.insert(
"x-amz-server-side-encryption-customer-key",
HeaderValue::from_static("unrecorded-request-customer-key"),
);
record_request(
&bounded_control,
Operation::GetObject,
@@ -4915,8 +5197,9 @@ mod tests {
upload_id: Some("u".repeat(MAX_RETAINED_IDENTIFIER_BYTES + 1)),
..Default::default()
},
JournaledHeaders::default(),
JournaledHeaders::from_headers(&source_headers),
);
update_prepared_response(&bounded_control, 1, &response, Duration::from_millis(7));
{
let bounded_records = lock(&bounded_control);
let bounded = &bounded_records.requests[0];
@@ -4924,7 +5207,39 @@ mod tests {
assert_eq!(bounded.key.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES - 1));
assert_eq!(bounded.version_id.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
assert_eq!(bounded.upload_id.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
assert_eq!(bounded.source_version_id.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
assert_eq!(bounded.source_mtime.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
assert_eq!(bounded.source_etag.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
for value in [&bounded.source_version_id, &bounded.source_mtime, &bounded.source_etag]
.into_iter()
.flatten()
{
assert!(value.capacity() <= MAX_RETAINED_IDENTIFIER_BYTES);
}
assert!(bounded.source_replication_request);
let prepared = bounded
.prepared_response
.as_ref()
.expect("the retained record receives its response");
assert_eq!(prepared.version_id.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
assert_eq!(prepared.etag.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
assert_eq!(prepared.last_modified.as_ref().map(String::len), Some(MAX_RETAINED_IDENTIFIER_BYTES));
for value in [&prepared.version_id, &prepared.etag, &prepared.last_modified]
.into_iter()
.flatten()
{
assert!(value.capacity() <= MAX_RETAINED_IDENTIFIER_BYTES);
}
assert_eq!(prepared.content_length, Some(17));
assert_eq!(prepared.elapsed, Duration::from_millis(7));
assert!(
!format!("{bounded:?}").contains("unrecorded-"),
"authorization and customer keys must not enter the journal"
);
}
lock(&bounded_control).requests.clear();
update_prepared_response(&bounded_control, 1, &response, Duration::ZERO);
assert!(lock(&bounded_control).requests.is_empty(), "a drained record must not be resurrected");
let target = FakeS3Target::start().await.expect("target");
target.create_bucket("b".repeat(63));
let empty_bucket = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
crates/e2e_test/src/upgrade_compatibility_test.rs
+268 -11
View File
@@ -376,7 +376,162 @@ async fn mixed_multipart_before(
) -> TestResult {
tokio::time::timeout_at(deadline, upload)
.await
.map_err(|_| format!("node {node}: multipart upload {MIXED_BUCKET}/{key} acknowledgement deadline exceeded"))?
.map_err(|_| MultipartAcknowledgementDeadline {
node,
key: key.to_owned(),
})?
}
#[derive(Debug)]
struct MultipartAcknowledgementDeadline {
node: usize,
key: String,
}
impl std::fmt::Display for MultipartAcknowledgementDeadline {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
formatter,
"node {}: multipart upload {MIXED_BUCKET}/{} acknowledgement deadline exceeded",
self.node, self.key
)
}
}
impl std::error::Error for MultipartAcknowledgementDeadline {}
// Decode only the public lock snapshot fields. Unknown fields, including
// credential/configuration payloads, never enter the failure artifact.
#[derive(serde::Deserialize, serde::Serialize)]
struct MixedLockSnapshot {
total: Option<u64>,
truncated: Option<bool>,
capability_note: Option<String>,
locks: Vec<MixedLockEntry>,
}
#[derive(serde::Deserialize, serde::Serialize)]
struct MixedLockEntry {
resource: Option<String>,
bucket: Option<String>,
object: Option<String>,
version: Option<String>,
owner: Option<String>,
#[serde(rename = "type")]
lock_type: Option<String>,
priority: Option<String>,
since: Option<String>,
elapsed_secs: Option<u64>,
ttl_secs: Option<u64>,
}
async fn read_mixed_lock_snapshot(response: &mut reqwest::Response) -> Result<MixedLockSnapshot, &'static str> {
const MAX_BODY_BYTES: usize = 1024 * 1024;
let mut body = Vec::new();
while let Some(chunk) = response.chunk().await.map_err(|_| "body_read_error")? {
if chunk.len() > MAX_BODY_BYTES - body.len() {
return Err("body_limit_exceeded");
}
body.extend_from_slice(&chunk);
}
serde_json::from_slice(&body).map_err(|_| "invalid_lock_snapshot")
}
async fn probe_mixed_multipart_failure(
cluster: &RustFSTestClusterEnvironment,
node: usize,
path: &str,
head: bool,
deadline: Instant,
) -> Value {
let started = Instant::now();
let method = if head { Method::HEAD } else { Method::GET };
let mut observation = serde_json::json!({
"node": node, "method": method.as_str(), "path": path,
"status": null, "observation": "unknown", "timeout": false,
});
// The shared absolute deadline bounds both headers and a slow/infinite
// response body; a send-only timeout would leave the cluster alive forever.
let result = tokio::time::timeout_at(deadline, async {
let mut response = signed_request(
method,
&format!("{}{path}", cluster.nodes[node].url.trim_end_matches('/')),
&cluster.access_key,
&cluster.secret_key,
None,
None,
)
.await
.map_err(|_| "request_error")?;
let status = response.status();
observation["status"] = serde_json::json!(status.as_u16());
if head {
observation["observation"] = serde_json::json!(match status {
StatusCode::OK => "visible",
StatusCode::NOT_FOUND => "not_visible",
_ => "unknown",
});
for (field, header) in [
("content_length", "content-length"),
("etag", "etag"),
("version", "x-amz-version-id"),
] {
observation[field] = serde_json::json!(response.headers().get(header).and_then(|value| value.to_str().ok()));
}
} else if status == StatusCode::OK {
observation["snapshot"] =
serde_json::to_value(read_mixed_lock_snapshot(&mut response).await?).map_err(|_| "snapshot_encode_error")?;
observation["observation"] = serde_json::json!("captured");
}
Ok::<_, &'static str>(())
})
.await;
match result {
Ok(Ok(())) => {}
Ok(Err(reason)) => observation["error"] = serde_json::json!(reason),
Err(_) => observation["timeout"] = serde_json::json!(true),
}
observation["elapsed_secs"] = serde_json::json!(started.elapsed().as_secs_f64());
observation
}
async fn observe_mixed_multipart_failure(
cluster: &RustFSTestClusterEnvironment,
phase: &str,
error: BoxError,
log_dir: Option<&Path>,
) -> TestResult {
if let (Some(log_dir), Some(failure)) = (log_dir, error.downcast_ref::<MultipartAcknowledgementDeadline>()) {
let started = Instant::now();
let deadline = started + Duration::from_secs(5);
// Reserve the last second for serialization and bounded artifact I/O.
// All probes run directly in this future; no diagnostic task is spawned.
let network_deadline = started + Duration::from_secs(4);
let path = format!("/{MIXED_BUCKET}/{}", failure.key);
let _ = tokio::time::timeout_at(deadline, async {
let (heads, locks) = tokio::join!(
futures::future::join_all((0..cluster.nodes.len()).map(|node| probe_mixed_multipart_failure(
cluster,
node,
&path,
true,
network_deadline
))),
probe_mixed_multipart_failure(cluster, 0, "/rustfs/admin/v3/top/locks?count=1000", false, network_deadline),
);
let snapshot = serde_json::json!({
"phase": phase, "bucket": MIXED_BUCKET, "key": failure.key,
"writer_node": failure.node, "heads": heads, "top_locks": locks,
});
let body = serde_json::to_vec_pretty(&snapshot).map_err(|_| ())?;
tokio::fs::write(log_dir.join("mixed-multipart-failure.json"), body)
.await
.map_err(|_| ())
})
.await;
}
// Visibility and diagnostic failures cannot acknowledge the original MPU.
Err(error)
}
/// Assert that `client` eventually lists exactly `expected` objects under
@@ -449,7 +604,13 @@ async fn exercise_mixed_cluster(
.await?;
assert_eq!(read_object(current_client, MIXED_BUCKET, &previous_key, None).await?.1, previous_body);
let multipart_keys = write_multipart_load(&clients, phase).await?;
let multipart_keys = match write_multipart_load(&clients, phase).await {
Ok(keys) => keys,
Err(error) => {
let log_dir = std::env::var_os("RUSTFS_E2E_LOG_DIR");
return observe_mixed_multipart_failure(cluster, phase, error, log_dir.as_deref().map(Path::new)).await;
}
};
let expected_count = multipart_keys.len() + 2;
for (label, client) in [("current", current_client), ("previous", previous_client)] {
assert_eq!(
@@ -739,6 +900,86 @@ mod upgrade_write_readiness_tests {
assert!(error.to_string().contains(stalled_key), "{error}");
assert!(error.to_string().contains("acknowledgement deadline exceeded"), "{error}");
assert_eq!(target.count_requests(FakeTargetOperation::CompleteMultipartUpload, stalled_key), 1);
let failure = error
.downcast_ref::<MultipartAcknowledgementDeadline>()
.expect("the timeout must retain typed writer/object context");
assert_eq!(failure.node, 2);
assert_eq!(failure.key, stalled_key);
let original_message = error.to_string();
// These four probe addresses share an in-process peer. This verifies
// bounded diagnostic fan-out, not a four-node RustFS cluster.
let mut probes = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
probes.access_key = FAKE_ACCESS_KEY.to_owned();
probes.secret_key = FAKE_SECRET_KEY.to_owned();
for node in &mut probes.nodes {
node.url = target.endpoint().to_owned();
}
let log_dir = tempfile::tempdir()?;
target.inject_for_key(
FakeTargetOperation::HeadObject,
stalled_key,
FaultAction::Stall(Duration::from_secs(30)),
1,
);
let error = tokio::time::timeout(
Duration::from_secs(7),
observe_mixed_multipart_failure(&probes, phase, error, Some(log_dir.path())),
)
.await
.expect("the shared diagnostic budget must include artifact writing")
.expect_err("observing committed data cannot acknowledge the stalled upload");
assert_eq!(error.to_string(), original_message);
assert!(error.downcast_ref::<MultipartAcknowledgementDeadline>().is_some());
let snapshot: Value =
serde_json::from_slice(&tokio::fs::read(log_dir.path().join("mixed-multipart-failure.json")).await?)?;
assert_eq!(snapshot["key"], stalled_key);
assert_eq!(snapshot["writer_node"], 2);
let heads = snapshot["heads"]
.as_array()
.expect("failure artifact must retain every probe");
assert_eq!(heads.len(), MIXED_NODE_COUNT);
assert_eq!(
heads
.iter()
.filter(|head| head["timeout"] == true && head["observation"] == "unknown")
.count(),
1
);
assert_eq!(
heads
.iter()
.filter(|head| head["status"] == 200 && head["observation"] == "visible")
.count(),
3
);
assert_eq!(target.count_requests(FakeTargetOperation::HeadObject, stalled_key), MIXED_NODE_COUNT);
assert_eq!(target.count_requests(FakeTargetOperation::CompleteMultipartUpload, stalled_key), 1);
assert_eq!(snapshot["top_locks"]["observation"], "unknown", "the fake peer has no admin lock API");
let locks: MixedLockSnapshot = serde_json::from_value(serde_json::json!({
"total": 1, "truncated": true, "capability_note": "partial snapshot",
"credentials": { "secret_key": "must not persist" },
"locks": [{
"resource": format!("{MIXED_BUCKET}/{stalled_key}"), "owner": "writer", "type": "WRITE",
"elapsed_secs": 30, "ttl_secs": 1, "merged_config": { "secret_key": "must not persist" },
}],
}))?;
let safe_locks = serde_json::to_value(locks)?;
assert_eq!(safe_locks["total"], 1);
assert_eq!(safe_locks["truncated"], true);
assert_eq!(safe_locks["locks"][0]["owner"], "writer");
assert_eq!(safe_locks["locks"][0]["elapsed_secs"], 30);
assert!(safe_locks.get("credentials").is_none());
assert!(safe_locks["locks"][0].get("merged_config").is_none());
let mut body = br#"{"locks":[]}"#.to_vec();
body.resize(1024 * 1024, b' ');
let mut at_limit = reqwest::Response::from(http::Response::new(body.clone()));
assert!(read_mixed_lock_snapshot(&mut at_limit).await?.locks.is_empty());
body.push(b' ');
let mut oversized = reqwest::Response::from(http::Response::new(body));
assert_eq!(read_mixed_lock_snapshot(&mut oversized).await.err(), Some("body_limit_exceeded"));
target.shutdown().await;
Ok(())
}
@@ -2248,9 +2489,9 @@ struct LayoutTransport {
uploaded_parts: Vec<i32>,
single_puts: usize,
completes: usize,
/// Raw per-key journal in target order: (sequence, operation, part number,
/// upload id), so duplicate drives can be told apart from retries.
journal: Vec<(u64, String, Option<i32>, Option<String>)>,
/// Allowlisted request identity and prepared response metadata. A prepared
/// response does not prove that the client received it.
journal: Vec<Value>,
}
/// Configure every layout bucket to replicate its existing objects to a fresh
@@ -2312,12 +2553,28 @@ async fn replicate_layouts(
journal: key_requests
.iter()
.map(|record| {
(
record.sequence,
format!("{:?}", record.operation),
record.part_number,
record.upload_id.as_ref().map(|id| id.chars().take(12).collect()),
)
serde_json::json!({
"sequence": record.sequence,
"operation": format!("{:?}", record.operation),
"part_number": record.part_number,
"upload_id": record.upload_id,
"requested_version_id": record.version_id,
"source_version_id": record.source_version_id,
"source_mtime": record.source_mtime,
"source_etag": record.source_etag,
"source_replication_request": record.source_replication_request,
"request_content_length": record.content_length,
"journaled_at_unix_millis": record.journaled_at_unix_millis,
"prepared_response": record.prepared_response.as_ref().map(|response| serde_json::json!({
"status": response.status,
"version_id": response.version_id,
"etag": response.etag,
"last_modified": response.last_modified,
"content_length": response.content_length,
"prepared_at_unix_millis": response.prepared_at_unix_millis,
"elapsed_secs": response.elapsed.as_secs_f64(),
})),
})
})
.collect(),
};
crates/ecstore/src/cache_value/metacache_set.rs
+80 -3
View File
@@ -12,9 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(test)]
use crate::disk::DiskIoClass;
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
use crate::disk::error::DiskError;
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions, current_disk_io_context, with_optional_disk_io_context};
use futures::future::join_all;
use metrics::counter;
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
@@ -245,6 +247,8 @@ pub struct ListPathRawOptions {
pub(crate) test_fallback_reader_behaviors: Vec<TestReaderBehavior>,
#[cfg(test)]
pub(crate) peek_timeout: Option<Duration>,
#[cfg(test)]
pub(crate) test_disk_io_context_observed: Option<Arc<AtomicBool>>,
// pub agreed: Option<Arc<dyn Fn(MetaCacheEntry) + Send + Sync>>,
// pub partial: Option<Arc<dyn Fn(MetaCacheEntries, &[Option<Error>]) + Send + Sync>>,
// pub finished: Option<Arc<dyn Fn(&[Option<Error>]) + Send + Sync>>,
@@ -276,6 +280,8 @@ impl Clone for ListPathRawOptions {
test_fallback_reader_behaviors: self.test_fallback_reader_behaviors.clone(),
#[cfg(test)]
peek_timeout: self.peek_timeout,
#[cfg(test)]
test_disk_io_context_observed: self.test_disk_io_context_observed.clone(),
..Default::default()
}
}
@@ -344,6 +350,7 @@ async fn list_path_raw_inner(
let log_path = opts.path.clone();
let mut jobs = JoinSet::new();
let disk_io_context = current_disk_io_context();
let mut readers = Vec::with_capacity(opts.disks.len());
let fds = Arc::new(TokioMutex::new(opts.fallback_disks.iter().flatten().cloned().collect::<VecDeque<_>>()));
#[cfg(test)]
@@ -364,11 +371,19 @@ async fn list_path_raw_inner(
let test_fallbacks_clone = test_fallbacks.clone();
let cancel_rx_clone = cancel_rx.clone();
let producer_errs_clone = producer_errs.clone();
let disk_io_context_clone = disk_io_context.clone();
let (rd, wr) = tokio::io::duplex(64);
readers.push(MetacacheReader::new(rd));
jobs.spawn(async move {
jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
let mut wr = PublishedBytesWriter::new(wr);
#[cfg(test)]
if let Some(observer) = &opts_clone.test_disk_io_context_observed {
observer.store(
crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
std::sync::atomic::Ordering::Release,
);
}
#[cfg(test)]
let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
match behavior {
TestReaderBehavior::Eof => return Ok(()),
@@ -652,7 +667,7 @@ async fn list_path_raw_inner(
// warn!("list_path_raw: while need_fallback done");
Ok(())
});
}));
}
let revjob_rx = rx.clone();
@@ -1168,6 +1183,68 @@ mod tests {
assert_eq!(err, DiskError::ErasureReadQuorum);
}
#[tokio::test]
async fn list_path_raw_joinset_producers_inherit_scanner_disk_io_class() {
let observed = Arc::new(AtomicBool::new(false));
crate::disk::with_background_disk_io(
CancellationToken::new(),
list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![None],
min_disks: 1,
test_reader_behaviors: vec![TestReaderBehavior::Eof],
test_disk_io_context_observed: Some(observed.clone()),
..Default::default()
},
),
)
.await
.expect("empty test producer should complete the listing");
assert!(
observed.load(std::sync::atomic::Ordering::Acquire),
"the per-disk JoinSet producer must retain the scanner background class"
);
}
#[tokio::test]
async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
let root = tempfile::tempdir().expect("local disk root should be created");
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("disk path should be UTF-8"))
.expect("endpoint should parse");
let local = Arc::new(
crate::disk::local::LocalDisk::new(&endpoint, false)
.await
.expect("local disk should open"),
);
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
wrapper.make_volume("bucket").await.expect("bucket volume should be created");
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper.clone())));
let cancellation = CancellationToken::new();
crate::disk::with_background_disk_io(
cancellation.clone(),
list_path_raw(
cancellation,
ListPathRawOptions {
disks: vec![Some(disk)],
bucket: "bucket".to_string(),
min_disks: 1,
recursive: true,
..Default::default()
},
),
)
.await
.expect("scanner metacache walk should complete");
assert!(
wrapper.background_io_admissions_for_tests() > 0,
"the LocalDiskWrapper walk should acquire background admission inside its JoinSet producer"
);
}
#[tokio::test]
async fn list_path_raw_returns_typed_quorum_error_for_multiple_drive_failures() {
let err = list_path_raw(
crates/ecstore/src/cluster/rpc/peer_rest_client.rs
+37 -7
View File
@@ -640,6 +640,7 @@ pub struct PeerLiveEventsBatch {
pub struct ScannerPublicationLease {
pub token: Uuid,
pub movement_generation: u64,
pub observational_only: bool,
/// Stable storage owner identity. This is distinct from the activity
/// session and is bound into both acquire and release proofs.
pub owner_id: String,
@@ -658,6 +659,7 @@ fn validate_scanner_publication_lease_response_fields(
response: &ScannerPublicationLeaseResponse,
expected_session_id: &str,
expected_generation: u64,
expected_observational_only: bool,
) -> Result<(Uuid, String)> {
if !response.success {
return Err(Error::other(
@@ -674,6 +676,9 @@ fn validate_scanner_publication_lease_response_fields(
if response.session_id != expected_session_id {
return Err(Error::other("peer returned a different scanner publication lease session"));
}
if response.observational_only != expected_observational_only {
return Err(Error::other("peer returned a different scanner publication lease purpose"));
}
let owner_id = Uuid::parse_str(&response.owner_id)
.ok()
.filter(|owner_id| !owner_id.is_nil())
@@ -2418,6 +2423,7 @@ impl PeerRestClient {
&self,
expected_session_id: &str,
expected_generation: u64,
observational_only: bool,
) -> Result<ScannerPublicationLease> {
let request_started = std::time::Instant::now();
self.finalize_result(
@@ -2434,6 +2440,7 @@ impl PeerRestClient {
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
expected_session_id: expected_session_id.to_string(),
token: Bytes::new(),
observational_only,
});
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
.map_err(|_| Error::other("scanner publication lease request is too large to authenticate"))?;
@@ -2444,11 +2451,16 @@ impl PeerRestClient {
.map_err(|_| Error::other("scanner publication lease response is too large to authenticate"))?;
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
.map_err(|_| Error::other("peer returned an invalid scanner publication lease proof"))?;
let (token, owner_id) =
validate_scanner_publication_lease_response_fields(&response, expected_session_id, expected_generation)?;
let (token, owner_id) = validate_scanner_publication_lease_response_fields(
&response,
expected_session_id,
expected_generation,
observational_only,
)?;
Ok(ScannerPublicationLease {
token,
movement_generation: response.movement_generation,
observational_only,
owner_id,
session_id: response.session_id,
expires_at: scanner_publication_lease_deadline(
@@ -2483,6 +2495,7 @@ impl PeerRestClient {
ttl_ms: crate::store::SCANNER_PUBLICATION_LEASE_TTL_MS,
expected_session_id: lease.session_id.clone(),
token: lease.token.as_bytes().to_vec().into(),
observational_only: lease.observational_only,
});
let canonical = rustfs_protos::canonical_scanner_publication_lease_request_body(request.get_ref())
.map_err(|_| Error::other("scanner publication lease validation request is too large to authenticate"))?;
@@ -2494,8 +2507,12 @@ impl PeerRestClient {
)?;
verify_tonic_rpc_response_proof(&response_body, &response.response_proof)
.map_err(|_| Error::other("peer returned an invalid scanner publication lease validation proof"))?;
let (token, owner_id) =
validate_scanner_publication_lease_response_fields(&response, &lease.session_id, lease.movement_generation)?;
let (token, owner_id) = validate_scanner_publication_lease_response_fields(
&response,
&lease.session_id,
lease.movement_generation,
lease.observational_only,
)?;
if token != lease.token {
return Err(Error::other("peer returned a different scanner publication lease token"));
}
@@ -3001,15 +3018,16 @@ mod tests {
response_proof: Bytes::new(),
owner_id: Uuid::new_v4().to_string(),
session_id: "session-a".to_string(),
observational_only: false,
};
assert!(validate_scanner_publication_lease_response_fields(&response, "session-a", 7).is_ok());
assert!(validate_scanner_publication_lease_response_fields(&response, "session-a", 7, false).is_ok());
let stale_generation = ScannerPublicationLeaseResponse {
movement_generation: 6,
..response.clone()
};
let error = validate_scanner_publication_lease_response_fields(&stale_generation, "session-a", 7)
let error = validate_scanner_publication_lease_response_fields(&stale_generation, "session-a", 7, false)
.expect_err("a response from an older movement generation must be rejected");
assert!(error.to_string().contains("different scanner publication lease generation"));
@@ -3017,9 +3035,21 @@ mod tests {
session_id: "session-b".to_string(),
..response
};
let error = validate_scanner_publication_lease_response_fields(&stale_session, "session-a", 7)
let error = validate_scanner_publication_lease_response_fields(&stale_session, "session-a", 7, false)
.expect_err("a response from an older scanner session must be rejected");
assert!(error.to_string().contains("different scanner publication lease session"));
let observational = ScannerPublicationLeaseResponse {
observational_only: true,
..stale_session
};
assert!(
validate_scanner_publication_lease_response_fields(&observational, "session-b", 7, true).is_ok(),
"the signed purpose echo should validate an observation lease"
);
let error = validate_scanner_publication_lease_response_fields(&observational, "session-b", 7, false)
.expect_err("a peer must not upgrade an observational lease to authoritative");
assert!(error.to_string().contains("different scanner publication lease purpose"));
}
#[test]
crates/ecstore/src/disk/disk_store.rs
+241 -35
View File
@@ -12,6 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::disk::io_admission::{
AdmissionReader, BoxedChunkReader, DiskIoAdmission, DiskIoClass, DiskIoContext, current_disk_io_context,
with_disk_io_context, with_disk_io_permit,
};
use crate::disk::{
CheckPartsResp, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint,
Error, FileInfoVersions, MmapCopyStageMetrics, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result,
@@ -813,6 +817,7 @@ impl Drop for DiskMetricWaitingGuard<'_> {
pub(crate) struct ReconnectDiskHealthState {
pub(crate) health: Arc<DiskHealthTracker>,
pub(crate) metrics: Arc<DiskHealthMetricEpoch>,
pub(crate) io_admission: Arc<DiskIoAdmission>,
}
#[derive(Debug)]
@@ -1329,6 +1334,8 @@ pub struct LocalDiskWrapper {
health: Arc<DiskHealthTracker>,
/// Internal metrics epoch preserved across local disk reconnects.
metrics: Arc<DiskHealthMetricEpoch>,
/// Per-drive foreground/background admission shared across wrapper reconnects.
io_admission: Arc<DiskIoAdmission>,
/// Whether health checking is enabled
health_check: bool,
/// Cancellation token for monitoring tasks
@@ -1342,11 +1349,13 @@ pub struct LocalDiskWrapper {
impl LocalDiskWrapper {
/// Create a new LocalDiskWrapper
pub fn new(disk: Arc<LocalDisk>, health_check: bool) -> Self {
let admission = DiskIoAdmission::new(disk.endpoint().to_string());
Self::new_with_health_and_metrics(
disk,
health_check,
Arc::new(DiskHealthTracker::new()),
Arc::new(DiskHealthMetricEpoch::default()),
admission,
)
}
@@ -1386,11 +1395,13 @@ impl LocalDiskWrapper {
health_check: bool,
reconnect: Option<ReconnectDiskHealthState>,
) -> Self {
let endpoint = disk.endpoint().to_string();
let reconnect = reconnect.unwrap_or_else(|| ReconnectDiskHealthState {
health: Arc::new(DiskHealthTracker::new()),
metrics: Arc::new(DiskHealthMetricEpoch::default()),
io_admission: DiskIoAdmission::new(endpoint),
});
Self::new_with_health_and_metrics(disk, health_check, reconnect.health, reconnect.metrics)
Self::new_with_health_and_metrics(disk, health_check, reconnect.health, reconnect.metrics, reconnect.io_admission)
}
fn new_with_health_and_metrics(
@@ -1398,6 +1409,7 @@ impl LocalDiskWrapper {
health_check: bool,
health: Arc<DiskHealthTracker>,
metrics: Arc<DiskHealthMetricEpoch>,
io_admission: Arc<DiskIoAdmission>,
) -> Self {
// Check environment variable for health check override.
// Only enable if both param and env are true.
@@ -1408,6 +1420,7 @@ impl LocalDiskWrapper {
disk,
health,
metrics,
io_admission,
health_check: health_check && env_health_check,
cancel_token: CancellationToken::new(),
disk_id: Arc::new(RwLock::new(None)),
@@ -1421,6 +1434,7 @@ impl LocalDiskWrapper {
ReconnectDiskHealthState {
health: Arc::new(self.health.metric_epoch_for_reconnect()),
metrics: self.metrics.clone(),
io_admission: self.io_admission.clone(),
}
}
@@ -1428,6 +1442,58 @@ impl LocalDiskWrapper {
self.disk.clone()
}
#[cfg(test)]
pub(crate) fn background_io_admissions_for_tests(&self) -> usize {
self.io_admission.background_admissions_for_tests()
}
#[cfg(test)]
pub(crate) fn background_io_active_for_tests(&self) -> bool {
self.io_admission.background_active_for_tests()
}
fn disk_io_context(&self) -> DiskIoContext {
current_disk_io_context().unwrap_or_else(|| DiskIoContext {
class: DiskIoClass::Foreground,
cancellation: CancellationToken::new(),
admission: None,
permit: None,
})
}
fn disk_walk_io_context(&self) -> DiskIoContext {
let mut context = self.disk_io_context();
context.admission = Some(self.io_admission.clone());
context.permit = None;
context
}
async fn acquire_disk_io_permit(&self) -> Result<Option<crate::disk::io_admission::DiskIoPermit>> {
if let Some(context) = current_disk_io_context() {
if let Some(permit) = context.permit
&& permit.is_for(&self.io_admission)
{
return Ok(Some(permit));
}
if context
.admission
.as_ref()
.is_some_and(|admission| Arc::ptr_eq(admission, &self.io_admission))
{
return Ok(None);
}
if context.class == DiskIoClass::Background {
return self
.io_admission
.background(&context.cancellation)
.await
.map(Some)
.map_err(DiskError::Io);
}
}
Ok(Some(self.io_admission.foreground()))
}
pub fn get_object_path_if_local(&self, volume: &str, path: &str) -> crate::disk::error::Result<std::path::PathBuf> {
self.disk.get_object_path(volume, path)
}
@@ -1899,6 +1965,31 @@ impl LocalDiskWrapper {
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = Result<T>>,
{
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
op,
move |io_permit| async move {
let _io_permit = io_permit;
operation().await
},
timeout_duration,
timeout_health_action,
mutation,
)
.await
}
async fn track_disk_health_with_op_timeout_action_and_mutation_with_io_permit<T, F, Fut>(
&self,
op: &'static str,
operation: F,
timeout_duration: Duration,
timeout_health_action: TimeoutHealthAction,
mutation: DiskMetricMutation,
) -> Result<T>
where
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
Fut: std::future::Future<Output = Result<T>>,
{
self.metrics.record_operation_call(op);
// Check if disk is faulty
@@ -1921,6 +2012,8 @@ impl LocalDiskWrapper {
return Err(err);
}
let io_permit = self.acquire_disk_io_permit().await?;
// Record operation start
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
let _waiting_guard = self.health.waiting_guard();
@@ -1928,7 +2021,7 @@ impl LocalDiskWrapper {
let started = Instant::now();
if timeout_duration == Duration::ZERO {
let result = operation().await;
let result = operation(io_permit).await;
self.metrics.record_operation_latency(op, started.elapsed());
self.record_result_error_metrics(&result);
if result.is_ok() {
@@ -1938,7 +2031,7 @@ impl LocalDiskWrapper {
return result;
}
// Execute the operation with timeout
let result = tokio::time::timeout(timeout_duration, operation()).await;
let result = tokio::time::timeout(timeout_duration, operation(io_permit)).await;
match result {
Ok(operation_result) => {
@@ -1989,11 +2082,12 @@ impl DiskAPI for LocalDiskWrapper {
}
async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
self.track_disk_health_with_op_and_timeout_action(
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
"read_metadata",
|| async { self.disk.read_metadata(volume, path).await },
|io_permit| async { self.disk.read_metadata_with_permit(volume, path, io_permit).await },
get_drive_metadata_timeout(),
self.scanner_timeout_health_action(),
DiskMetricMutation::None,
)
.await
}
@@ -2165,12 +2259,15 @@ impl DiskAPI for LocalDiskWrapper {
opts.timeout_duration().unwrap_or_else(get_drive_walkdir_timeout)
};
self.track_disk_health_with_op_and_timeout_action(
"walk_dir",
|| async { self.disk.walk_dir(opts, wr).await },
timeout_duration,
// Listing/scanner backpressure should fail only the current walk, not poison drive health.
TimeoutHealthAction::IgnoreFailure,
with_disk_io_context(
self.disk_walk_io_context(),
self.track_disk_health_with_op_and_timeout_action(
"walk_dir",
|| async { self.disk.walk_dir(opts, wr).await },
timeout_duration,
// Listing/scanner backpressure should fail only the current walk, not poison drive health.
TimeoutHealthAction::IgnoreFailure,
),
)
.await
}
@@ -2193,6 +2290,7 @@ impl DiskAPI for LocalDiskWrapper {
}
async fn delete_versions(&self, volume: &str, versions: Vec<FileInfoVersions>, opts: DeleteOptions) -> Vec<Option<Error>> {
let version_count = versions.len();
self.metrics.record_operation_call("delete_versions");
// Check if disk is faulty before proceeding
if self.health.is_faulty() {
@@ -2206,6 +2304,11 @@ impl DiskAPI for LocalDiskWrapper {
return vec![Some(e); versions.len()];
}
let _io_permit = match self.acquire_disk_io_permit().await {
Ok(permit) => permit,
Err(error) => return vec![Some(error); version_count],
};
// Record operation start
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
self.health.increment_waiting();
@@ -2322,19 +2425,25 @@ impl DiskAPI for LocalDiskWrapper {
version_id: &str,
opts: &ReadOptions,
) -> Result<FileInfo> {
self.track_disk_health_with_op(
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
"read_version",
|| async { self.disk.read_version(org_volume, volume, path, version_id, opts).await },
|io_permit| async move {
with_disk_io_permit(io_permit, self.disk.read_version(org_volume, volume, path, version_id, opts)).await
},
get_max_timeout_duration(),
TimeoutHealthAction::MarkFailure,
DiskMetricMutation::None,
)
.await
}
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
self.track_disk_health_with_op(
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
"read_xl",
|| async { self.disk.read_xl(volume, path, read_data).await },
|io_permit| async move { with_disk_io_permit(io_permit, self.disk.read_xl(volume, path, read_data)).await },
get_max_timeout_duration(),
TimeoutHealthAction::MarkFailure,
DiskMetricMutation::None,
)
.await
}
@@ -2362,21 +2471,25 @@ impl DiskAPI for LocalDiskWrapper {
}
async fn read_file(&self, volume: &str, path: &str) -> Result<crate::disk::FileReader> {
self.track_disk_health_with_op(
"read_file",
|| async { self.disk.read_file(volume, path).await },
get_max_timeout_duration(),
)
.await
let reader = self
.track_disk_health_with_op(
"read_file",
|| async { self.disk.read_file(volume, path).await },
get_max_timeout_duration(),
)
.await?;
Ok(Box::new(AdmissionReader::new(reader, self.io_admission.clone(), self.disk_io_context())))
}
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<crate::disk::FileReader> {
self.track_disk_health_with_op(
"read_file_stream",
|| async { self.disk.read_file_stream(volume, path, offset, length).await },
get_max_timeout_duration(),
)
.await
let reader = self
.track_disk_health_with_op(
"read_file_stream",
|| async { self.disk.read_file_stream(volume, path, offset, length).await },
get_max_timeout_duration(),
)
.await?;
Ok(Box::new(AdmissionReader::new(reader, self.io_admission.clone(), self.disk_io_context())))
}
async fn read_file_stream_chunks(
@@ -2386,12 +2499,20 @@ impl DiskAPI for LocalDiskWrapper {
offset: usize,
length: usize,
) -> Result<Option<rustfs_rio::ChunkReaderBox>> {
self.track_disk_health_with_op(
"read_file_stream_chunks",
|| async { self.disk.read_file_stream_chunks(volume, path, offset, length).await },
get_max_timeout_duration(),
)
.await
let reader = self
.track_disk_health_with_op(
"read_file_stream_chunks",
|| async { self.disk.read_file_stream_chunks(volume, path, offset, length).await },
get_max_timeout_duration(),
)
.await?;
Ok(reader.map(|reader| {
Box::new(AdmissionReader::new(
BoxedChunkReader(reader),
self.io_admission.clone(),
self.disk_io_context(),
)) as rustfs_rio::ChunkReaderBox
}))
}
async fn read_file_mmap_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<bytes::Bytes> {
@@ -2524,8 +2645,14 @@ impl DiskAPI for LocalDiskWrapper {
}
async fn read_parts(&self, bucket: &str, paths: &[String]) -> Result<Vec<ObjectPartInfo>> {
self.track_disk_health_with_op("read_parts", || async { self.disk.read_parts(bucket, paths).await }, Duration::ZERO)
.await
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
"read_parts",
|io_permit| async move { with_disk_io_permit(io_permit, self.disk.read_parts(bucket, paths)).await },
Duration::ZERO,
TimeoutHealthAction::MarkFailure,
DiskMetricMutation::None,
)
.await
}
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>> {
@@ -2592,6 +2719,84 @@ mod tests {
}
}
struct StalledMetacacheWriter {
entered: Arc<tokio::sync::Notify>,
}
impl AsyncWrite for StalledMetacacheWriter {
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<io::Result<usize>> {
self.entered.notify_one();
Poll::Pending
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
}
#[tokio::test]
async fn scanner_walk_releases_disk_admission_while_output_is_backpressured() {
let root = tempfile::tempdir().expect("disk root should be created");
let endpoint =
Endpoint::try_from(root.path().to_str().expect("disk root should be UTF-8")).expect("endpoint should parse");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should open"));
let wrapper = Arc::new(LocalDiskWrapper::new(disk, false));
wrapper.make_volume("bucket").await.expect("bucket should be created");
let object_path = root.path().join("bucket/object");
tokio::fs::create_dir_all(&object_path)
.await
.expect("object directory should be created");
tokio::fs::write(object_path.join("xl.meta"), b"scanner metadata")
.await
.expect("object metadata should be created");
let entered = Arc::new(tokio::sync::Notify::new());
let writer = StalledMetacacheWriter {
entered: entered.clone(),
};
let walk_wrapper = wrapper.clone();
let walk = tokio::spawn(async move {
crate::disk::with_background_disk_io(CancellationToken::new(), async move {
let mut writer = writer;
walk_wrapper
.walk_dir(
WalkDirOptions {
bucket: "bucket".to_string(),
recursive: true,
skip_total_timeout: true,
..Default::default()
},
&mut writer,
)
.await
})
.await
});
tokio::time::timeout(Duration::from_secs(2), entered.notified())
.await
.expect("walk should reach the blocked output writer");
assert!(
!wrapper.background_io_active_for_tests(),
"filesystem read permits must be released before metacache output backpressure"
);
let checkpoint_permit = tokio::time::timeout(
Duration::from_secs(1),
crate::disk::with_background_disk_io(CancellationToken::new(), wrapper.acquire_disk_io_permit()),
)
.await
.expect("checkpoint admission should progress while the scanner consumer is paused")
.expect("checkpoint should acquire the disk gate");
drop(checkpoint_permit);
walk.abort();
let _ = walk.await;
}
#[tokio::test]
async fn owned_mutation_keeps_publication_guard_after_waiter_cancellation() {
let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
@@ -2742,6 +2947,7 @@ mod tests {
let reconnect = ReconnectDiskHealthState {
health: Arc::new(health.metric_epoch_for_reconnect()),
metrics: metrics.clone(),
io_admission: DiskIoAdmission::new("test-disk"),
};
metrics.record_operation_call("read_all");
reconnect.metrics.record_timeout_error();
crates/ecstore/src/disk/io_admission.rs
+706
View File
@@ -0,0 +1,706 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Per-local-disk foreground-priority admission for scanner I/O.
use std::collections::VecDeque;
use std::future::Future;
use std::io;
use std::pin::Pin;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use std::time::Duration;
use metrics::{counter, gauge, histogram};
use rustfs_rio::ChunkReader;
use tokio::io::{AsyncRead, ReadBuf};
use tokio::sync::Notify;
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
const BACKGROUND_FOREGROUND_BURST: u32 = 8;
const BACKGROUND_MAX_WAIT: Duration = Duration::from_millis(100);
tokio::task_local! {
static DISK_IO_CONTEXT: DiskIoContext;
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum DiskIoClass {
Foreground,
Background,
}
#[derive(Clone, Debug)]
pub(crate) struct DiskIoContext {
pub(crate) class: DiskIoClass,
pub(crate) cancellation: CancellationToken,
pub(crate) admission: Option<Arc<DiskIoAdmission>>,
pub(crate) permit: Option<DiskIoPermit>,
}
/// Mark disk I/O issued by `future` as background work for local-disk admission.
#[doc(hidden)]
pub async fn with_background_disk_io<F: Future>(cancellation: CancellationToken, future: F) -> F::Output {
with_disk_io_context(
DiskIoContext {
class: DiskIoClass::Background,
cancellation,
admission: None,
permit: None,
},
future,
)
.await
}
pub(crate) async fn with_disk_io_context<F: Future>(context: DiskIoContext, future: F) -> F::Output {
DISK_IO_CONTEXT.scope(context, future).await
}
pub(crate) async fn with_disk_io_permit<F: Future>(permit: Option<DiskIoPermit>, future: F) -> F::Output {
let Some(permit) = permit else {
return future.await;
};
let mut context = current_disk_io_context().unwrap_or_else(|| DiskIoContext {
class: DiskIoClass::Foreground,
cancellation: CancellationToken::new(),
admission: None,
permit: None,
});
context.admission = Some(permit.admission());
with_disk_io_context(
DiskIoContext {
permit: Some(permit),
..context
},
future,
)
.await
}
pub(crate) async fn with_optional_disk_io_context<F: Future>(context: Option<DiskIoContext>, future: F) -> F::Output {
match context {
Some(context) => with_disk_io_context(context, future).await,
None => future.await,
}
}
pub(crate) fn current_disk_io_context() -> Option<DiskIoContext> {
DISK_IO_CONTEXT.try_with(Clone::clone).ok()
}
/// Acquire one low-level permit only when the current walker explicitly owns a gate.
/// Ordinary wrapper calls use their outer operation permit instead.
pub(crate) async fn acquire_context_disk_io_permit() -> io::Result<Option<DiskIoPermit>> {
let Some(context) = current_disk_io_context() else {
return Ok(None);
};
let Some(admission) = context.admission else {
return Ok(None);
};
if let Some(permit) = context.permit
&& permit.is_for(&admission)
{
return Ok(Some(permit));
}
match context.class {
DiskIoClass::Foreground => Ok(Some(admission.foreground())),
DiskIoClass::Background => Ok(Some(admission.background(&context.cancellation).await?)),
}
}
#[derive(Debug, Default)]
struct AdmissionState {
foreground_completions: u32,
next_background_ticket: u64,
background_queue: VecDeque<u64>,
background_front_since: Option<Instant>,
}
/// One gate is shared by all handles to a local disk, including reconnects.
/// Foreground I/O never waits on this gate. Background I/O runs exclusively
/// while the disk is idle, or after a bounded foreground burst/wait.
#[derive(Debug)]
pub(crate) struct DiskIoAdmission {
state: Mutex<AdmissionState>,
changed: Notify,
disk: String,
foreground_active: AtomicUsize,
background_active: AtomicBool,
background_waiters: AtomicUsize,
#[cfg(test)]
background_admissions: AtomicUsize,
}
impl DiskIoAdmission {
pub(crate) fn new(disk: impl Into<String>) -> Arc<Self> {
Arc::new(Self {
state: Mutex::new(AdmissionState::default()),
changed: Notify::new(),
disk: disk.into(),
foreground_active: AtomicUsize::new(0),
background_active: AtomicBool::new(false),
background_waiters: AtomicUsize::new(0),
#[cfg(test)]
background_admissions: AtomicUsize::new(0),
})
}
#[cfg(test)]
pub(crate) fn background_admissions_for_tests(&self) -> usize {
self.background_admissions.load(Ordering::Relaxed)
}
#[cfg(test)]
pub(crate) fn background_active_for_tests(&self) -> bool {
self.background_active.load(Ordering::Acquire)
}
#[cfg(test)]
fn foreground_active_for_tests(&self) -> usize {
self.foreground_active.load(Ordering::Acquire)
}
pub(crate) fn foreground(self: &Arc<Self>) -> DiskIoPermit {
self.foreground_active.fetch_add(1, Ordering::AcqRel);
if self.background_waiters.load(Ordering::Acquire) > 0 {
self.record_state();
}
DiskIoPermit {
lease: Arc::new(DiskIoPermitLease {
admission: self.clone(),
class: DiskIoClass::Foreground,
}),
}
}
pub(crate) async fn background(self: &Arc<Self>, cancellation: &CancellationToken) -> std::io::Result<DiskIoPermit> {
let started_at = Instant::now();
let ticket = {
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
let ticket = state.next_background_ticket;
state.next_background_ticket = state.next_background_ticket.wrapping_add(1);
if state.background_queue.is_empty() && !self.background_active.load(Ordering::Acquire) {
state.background_front_since = Some(Instant::now());
}
state.background_queue.push_back(ticket);
self.background_waiters.fetch_add(1, Ordering::Release);
ticket
};
self.record_state();
let mut waiter = BackgroundWaiter {
admission: self.clone(),
ticket,
waiting: true,
};
loop {
let notified = self.changed.notified();
tokio::pin!(notified);
notified.as_mut().enable();
let granted = {
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
let is_front = state.background_queue.front() == Some(&ticket);
let bounded_wait_elapsed = is_front
&& state
.background_front_since
.is_some_and(|front_since| front_since.elapsed() >= BACKGROUND_MAX_WAIT);
if is_front
&& !self.background_active.load(Ordering::Acquire)
&& (self.foreground_active.load(Ordering::Acquire) == 0
|| state.foreground_completions >= BACKGROUND_FOREGROUND_BURST
|| bounded_wait_elapsed)
{
self.background_active.store(true, Ordering::Release);
state.background_queue.pop_front();
state.background_front_since = None;
self.background_waiters.fetch_sub(1, Ordering::AcqRel);
state.foreground_completions = 0;
true
} else {
false
}
};
if granted {
waiter.waiting = false;
#[cfg(test)]
self.background_admissions.fetch_add(1, Ordering::Relaxed);
histogram!("rustfs_disk_io_admission_wait_seconds", "disk" => self.disk.clone(), "class" => "background")
.record(started_at.elapsed().as_secs_f64());
counter!("rustfs_disk_io_admission_total", "disk" => self.disk.clone(), "class" => "background").increment(1);
self.record_state();
return Ok(DiskIoPermit {
lease: Arc::new(DiskIoPermitLease {
admission: self.clone(),
class: DiskIoClass::Background,
}),
});
}
let remaining = {
let state = self.state.lock().unwrap_or_else(|error| error.into_inner());
if state.background_queue.front() == Some(&ticket) {
state
.background_front_since
.map(|front_since| BACKGROUND_MAX_WAIT.saturating_sub(front_since.elapsed()))
} else {
None
}
};
if let Some(remaining) = remaining.filter(|remaining| !remaining.is_zero()) {
tokio::select! {
_ = cancellation.cancelled() => {
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "background disk I/O admission cancelled"));
}
_ = tokio::time::sleep(remaining) => {
self.changed.notify_waiters();
}
_ = &mut notified => {}
}
} else {
tokio::select! {
_ = cancellation.cancelled() => {
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "background disk I/O admission cancelled"));
}
_ = &mut notified => {}
}
}
}
}
fn record_state(&self) {
let (foreground_active, background_waiters) = {
let state = self.state.lock().unwrap_or_else(|error| error.into_inner());
(
if state.background_queue.is_empty() {
0
} else {
self.foreground_active.load(Ordering::Acquire)
},
state.background_queue.len(),
)
};
let foreground_active = u32::try_from(foreground_active).unwrap_or(u32::MAX);
let background_waiters = u32::try_from(background_waiters).unwrap_or(u32::MAX);
gauge!("rustfs_disk_io_admission_active", "disk" => self.disk.clone(), "class" => "foreground")
.set(f64::from(foreground_active));
gauge!("rustfs_disk_io_admission_active", "disk" => self.disk.clone(), "class" => "background").set(
if self.background_active.load(Ordering::Acquire) {
1.0
} else {
0.0
},
);
gauge!("rustfs_disk_io_admission_waiters", "disk" => self.disk.clone(), "class" => "background")
.set(f64::from(background_waiters));
}
}
struct BackgroundWaiter {
admission: Arc<DiskIoAdmission>,
ticket: u64,
waiting: bool,
}
impl Drop for BackgroundWaiter {
fn drop(&mut self) {
if !self.waiting {
return;
}
let mut state = self.admission.state.lock().unwrap_or_else(|error| error.into_inner());
if let Some(index) = state.background_queue.iter().position(|ticket| *ticket == self.ticket) {
state.background_queue.remove(index);
if index == 0 {
state.background_front_since =
if state.background_queue.is_empty() || self.admission.background_active.load(Ordering::Acquire) {
None
} else {
Some(Instant::now())
};
}
self.admission.background_waiters.fetch_sub(1, Ordering::AcqRel);
}
drop(state);
self.admission.record_state();
self.admission.changed.notify_waiters();
}
}
#[derive(Debug, Clone)]
pub(crate) struct DiskIoPermit {
lease: Arc<DiskIoPermitLease>,
}
impl DiskIoPermit {
pub(crate) fn is_for(&self, admission: &Arc<DiskIoAdmission>) -> bool {
Arc::ptr_eq(&self.lease.admission, admission)
}
fn admission(&self) -> Arc<DiskIoAdmission> {
self.lease.admission.clone()
}
}
#[derive(Debug)]
struct DiskIoPermitLease {
admission: Arc<DiskIoAdmission>,
class: DiskIoClass,
}
pub(crate) fn spawn_blocking_with_disk_io_permit<T, F>(
io_permit: Option<DiskIoPermit>,
operation: F,
) -> tokio::task::JoinHandle<T>
where
T: Send + 'static,
F: FnOnce() -> T + Send + 'static,
{
tokio::task::spawn_blocking(move || {
let _io_permit = io_permit;
operation()
})
}
type AdmissionWait = Pin<Box<dyn Future<Output = io::Result<DiskIoPermit>> + Send>>;
impl Drop for DiskIoPermitLease {
fn drop(&mut self) {
match self.class {
DiskIoClass::Foreground => {
self.admission.foreground_active.fetch_sub(1, Ordering::AcqRel);
let record_state = self.admission.background_waiters.load(Ordering::Acquire) > 0;
if record_state {
let mut state = self.admission.state.lock().unwrap_or_else(|error| error.into_inner());
if !state.background_queue.is_empty() {
state.foreground_completions = state.foreground_completions.saturating_add(1);
}
}
if record_state {
self.admission.record_state();
self.admission.changed.notify_waiters();
}
}
DiskIoClass::Background => {
let mut state = self.admission.state.lock().unwrap_or_else(|error| error.into_inner());
self.admission.background_active.store(false, Ordering::Release);
state.background_front_since = if state.background_queue.is_empty() {
None
} else {
Some(Instant::now())
};
drop(state);
self.admission.record_state();
self.admission.changed.notify_waiters();
}
}
}
}
pub(crate) struct AdmissionReader<R> {
inner: R,
admission: Arc<DiskIoAdmission>,
context: DiskIoContext,
wait: Mutex<Option<AdmissionWait>>,
permit: Option<DiskIoPermit>,
}
impl<R> AdmissionReader<R> {
pub(crate) fn new(inner: R, admission: Arc<DiskIoAdmission>, context: DiskIoContext) -> Self {
Self {
inner,
admission,
context,
wait: Mutex::new(None),
permit: None,
}
}
fn poll_permit(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
if self.permit.is_some() {
return Poll::Ready(Ok(()));
}
if self.context.class == DiskIoClass::Foreground {
self.permit = Some(self.admission.foreground());
return Poll::Ready(Ok(()));
}
let mut wait = self.wait.lock().unwrap_or_else(|error| error.into_inner());
if wait.is_none() {
let admission = self.admission.clone();
let cancellation = self.context.cancellation.clone();
*wait = Some(Box::pin(async move { admission.background(&cancellation).await }));
}
let Some(wait_future) = wait.as_mut() else {
return Poll::Ready(Err(io::Error::other("background disk I/O admission waiter is missing")));
};
let result = wait_future.as_mut().poll(cx);
match result {
Poll::Ready(Ok(permit)) => {
wait.take();
drop(wait);
self.permit = Some(permit);
Poll::Ready(Ok(()))
}
Poll::Ready(Err(error)) => {
wait.take();
Poll::Ready(Err(error))
}
Poll::Pending => Poll::Pending,
}
}
fn finish_poll<T>(&mut self, result: Poll<io::Result<T>>) -> Poll<io::Result<T>> {
if result.is_ready() {
self.permit = None;
}
result
}
}
impl<R: AsyncRead + Unpin> AsyncRead for AdmissionReader<R> {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
match self.poll_permit(cx) {
Poll::Ready(Ok(())) => {}
Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
Poll::Pending => return Poll::Pending,
}
let result = Pin::new(&mut self.inner).poll_read(cx, buf);
self.finish_poll(result)
}
}
impl<R: ChunkReader + Unpin> ChunkReader for AdmissionReader<R> {
fn poll_read_chunk(mut self: Pin<&mut Self>, cx: &mut Context<'_>, max: usize) -> Poll<io::Result<Option<bytes::Bytes>>> {
match self.poll_permit(cx) {
Poll::Ready(Ok(())) => {}
Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
Poll::Pending => return Poll::Pending,
}
let result = Pin::new(&mut self.inner).poll_read_chunk(cx, max);
self.finish_poll(result)
}
}
pub(crate) struct BoxedChunkReader(pub(crate) rustfs_rio::ChunkReaderBox);
impl AsyncRead for BoxedChunkReader {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
Pin::new(&mut self.0).poll_read(cx, buf)
}
}
impl ChunkReader for BoxedChunkReader {
fn poll_read_chunk(self: Pin<&mut Self>, cx: &mut Context<'_>, max: usize) -> Poll<io::Result<Option<bytes::Bytes>>> {
Pin::new(self.get_mut().0.as_mut()).poll_read_chunk(cx, max)
}
}
#[cfg(test)]
mod tests {
use super::*;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
struct BlockingWorker(tokio::task::JoinHandle<()>);
async fn wait_for_background_waiters(admission: &DiskIoAdmission, expected: usize) {
tokio::time::timeout(Duration::from_secs(1), async {
loop {
let waiters = admission
.state
.lock()
.unwrap_or_else(|error| error.into_inner())
.background_queue
.len();
if waiters == expected {
return;
}
tokio::task::yield_now().await;
}
})
.await
.expect("background admission waiter should reach expected count");
}
async fn wait_for_foreground_active(admission: &DiskIoAdmission, expected: usize) {
tokio::time::timeout(Duration::from_secs(1), async {
loop {
let active = admission.foreground_active.load(Ordering::Acquire);
if active == expected {
return;
}
tokio::task::yield_now().await;
}
})
.await
.expect("foreground disk read should retain its permit while pending");
}
#[tokio::test]
async fn foreground_bypasses_background_and_background_progress_is_bounded() {
let admission = DiskIoAdmission::new("disk-a");
let foreground = admission.foreground();
let background_admission = admission.clone();
let cancellation = CancellationToken::new();
let background = tokio::spawn(async move { background_admission.background(&cancellation).await });
wait_for_background_waiters(&admission, 1).await;
for _ in 0..BACKGROUND_FOREGROUND_BURST {
drop(admission.foreground());
}
let background_permit = tokio::time::timeout(Duration::from_secs(1), background)
.await
.expect("background admission should progress after the foreground burst")
.expect("background waiter task should complete")
.expect("background admission should not be cancelled");
assert_eq!(
admission.foreground_active.load(Ordering::Acquire),
1,
"foreground operations remain admitted while one background operation runs"
);
drop(background_permit);
drop(foreground);
}
#[tokio::test]
async fn cancellation_removes_a_queued_background_waiter() {
let admission = DiskIoAdmission::new("disk-a");
let background = admission
.background(&CancellationToken::new())
.await
.expect("first background operation should be admitted");
let cancellation = CancellationToken::new();
let waiter_admission = admission.clone();
let waiter_cancellation = cancellation.clone();
let waiter = tokio::spawn(async move { waiter_admission.background(&waiter_cancellation).await });
wait_for_background_waiters(&admission, 1).await;
cancellation.cancel();
let error = tokio::time::timeout(Duration::from_secs(1), waiter)
.await
.expect("cancelled background waiter should stop")
.expect("background waiter task should complete")
.expect_err("cancelled admission must not start disk I/O");
assert_eq!(error.kind(), io::ErrorKind::Interrupted);
assert_eq!(
admission
.state
.lock()
.unwrap_or_else(|error| error.into_inner())
.background_queue
.len(),
0
);
drop(background);
}
#[tokio::test]
async fn background_admission_is_isolated_per_disk() {
let disk_a = DiskIoAdmission::new("disk-a");
let disk_b = DiskIoAdmission::new("disk-b");
let _disk_a_permit = disk_a
.background(&CancellationToken::new())
.await
.expect("disk A background operation should be admitted");
let _disk_b_permit = tokio::time::timeout(Duration::from_secs(1), disk_b.background(&CancellationToken::new()))
.await
.expect("disk A must not block disk B")
.expect("disk B background operation should be admitted");
}
#[tokio::test]
async fn streamed_read_holds_admission_until_disk_read_resolves() {
let admission = DiskIoAdmission::new("disk-a");
let (mut writer, reader) = tokio::io::duplex(8);
let mut reader = AdmissionReader::new(
reader,
admission.clone(),
DiskIoContext {
class: DiskIoClass::Foreground,
cancellation: CancellationToken::new(),
admission: None,
permit: None,
},
);
let read = tokio::spawn(async move {
let mut bytes = [0; 4];
reader
.read_exact(&mut bytes)
.await
.expect("streamed disk read should complete");
bytes
});
wait_for_foreground_active(&admission, 1).await;
writer.write_all(b"rust").await.expect("test bytes should be written");
assert_eq!(read.await.expect("reader task should finish"), *b"rust");
wait_for_foreground_active(&admission, 0).await;
}
#[tokio::test]
async fn blocking_worker_keeps_permit_after_caller_timeout() {
let admission = DiskIoAdmission::new("disk-a");
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let worker = spawn_blocking_with_disk_io_permit(Some(admission.foreground()), move || {
started_tx.send(()).expect("worker start should be observed");
std::thread::sleep(Duration::from_millis(100));
});
started_rx.await.expect("blocking worker should start");
assert!(tokio::time::timeout(Duration::from_millis(10), worker).await.is_err());
assert_eq!(admission.foreground_active_for_tests(), 1);
wait_for_foreground_active(&admission, 0).await;
}
#[tokio::test]
async fn cloned_permit_remains_active_until_all_workers_finish() {
let admission = DiskIoAdmission::new("disk-a");
let permit = admission.foreground();
let worker_permit = permit.clone();
drop(permit);
assert_eq!(admission.foreground_active_for_tests(), 1);
drop(worker_permit);
assert_eq!(admission.foreground_active_for_tests(), 0);
}
#[tokio::test]
async fn scoped_foreground_permit_is_inherited_by_blocking_worker() {
let admission = DiskIoAdmission::new("disk-a");
let permit = admission.foreground();
let worker = with_disk_io_permit(Some(permit), async {
let worker_permit = acquire_context_disk_io_permit()
.await
.expect("context permit should be available")
.expect("scoped permit should be inherited");
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let worker = spawn_blocking_with_disk_io_permit(Some(worker_permit), move || {
started_tx.send(()).expect("worker start should be observed");
std::thread::sleep(Duration::from_millis(100));
});
started_rx.await.expect("blocking worker should start");
BlockingWorker(worker)
})
.await;
assert_eq!(admission.foreground_active_for_tests(), 1);
assert!(tokio::time::timeout(Duration::from_millis(10), worker.0).await.is_err());
wait_for_foreground_active(&admission, 0).await;
}
}
crates/ecstore/src/disk/local.rs
+192 -83
View File
@@ -48,6 +48,7 @@ use crate::diagnostics::get::{
#[cfg(test)]
use crate::disk::HEALING_MARKER_PATH;
use crate::disk::disk_store::{get_drive_walkdir_stall_timeout, get_object_disk_read_timeout};
use crate::disk::io_admission::{DiskIoPermit, spawn_blocking_with_disk_io_permit};
use crate::disk::{
BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, ConditionalFileUpdate, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskInfo,
@@ -55,7 +56,7 @@ use crate::disk::{
PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, RUSTFS_META_BUCKET,
RUSTFS_META_TMP_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
conv_part_err_to_int,
acquire_context_disk_io_permit, conv_part_err_to_int,
endpoint::Endpoint,
error::{DiskError, Error, FileAccessDeniedWithContext, Result},
error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error},
@@ -2809,39 +2810,44 @@ where
with_walk_stall_deadline(stall, fut).await?
}
async fn await_walk_stall_progress<T>(
stall: Option<Duration>,
mut task: tokio::task::JoinHandle<Result<T>>,
mut progress: tokio::sync::mpsc::UnboundedReceiver<()>,
) -> Result<T> {
let Some(stall) = stall.filter(|stall| !stall.is_zero()) else {
return task.await.map_err(DiskError::from)?;
};
let mut deadline = Box::pin(tokio::time::sleep(stall));
loop {
tokio::select! {
result = &mut task => return result.map_err(DiskError::from)?,
progress_update = progress.recv() => match progress_update {
Some(()) => deadline.as_mut().reset(tokio::time::Instant::now() + stall),
None => return task.await.map_err(DiskError::from)?,
},
_ = &mut deadline => return Err(DiskError::Timeout),
}
}
}
async fn read_dir_entries_with_walk_stall(path: &Path, count: i32, stall: Option<Duration>) -> Result<Vec<String>> {
let mut entries = with_walk_stall_deadline(stall, fs::read_dir(path))
.await?
.map_err(to_file_error)?;
let mut entries = read_dir_with_walk_stall(path.to_path_buf(), stall).await?;
let mut names = Vec::new();
let mut remaining = count;
loop {
let Some(entry) = with_walk_stall_deadline(stall, entries.next_entry())
.await?
.map_err(to_file_error)?
else {
break;
};
let name = entry.file_name().to_string_lossy().to_string();
if name.is_empty() || name == "." || name == ".." {
continue;
}
let file_type = with_walk_stall_deadline(stall, entry.file_type())
.await?
.map_err(to_file_error)?;
if file_type.is_file() {
names.push(name);
} else if file_type.is_dir() {
names.push(format!("{name}{SLASH_SEPARATOR}"));
let result_limit = if remaining > 0 {
(remaining as usize).min(WALK_DIR_BATCH_SIZE)
} else {
continue;
}
remaining -= 1;
if remaining == 0 {
WALK_DIR_BATCH_SIZE
};
let (next_entries, next_names, reached_eof) = read_dir_entry_batch_with_walk_stall(entries, result_limit, stall).await?;
entries = next_entries;
remaining -= next_names.len() as i32;
names.extend(next_names);
if (count > 0 && remaining == 0) || reached_eof {
break;
}
}
@@ -2849,6 +2855,52 @@ async fn read_dir_entries_with_walk_stall(path: &Path, count: i32, stall: Option
Ok(names)
}
const WALK_DIR_BATCH_SIZE: usize = 32;
async fn read_dir_with_walk_stall(path: PathBuf, stall: Option<Duration>) -> Result<std::fs::ReadDir> {
let io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
let read_dir = spawn_blocking_with_disk_io_permit(io_permit, move || std::fs::read_dir(path).map_err(to_file_error));
let read_dir = with_walk_stall_deadline(stall, read_dir).await?.map_err(DiskError::from)?;
read_dir.map_err(|error| DiskError::from(to_file_error(error)))
}
async fn read_dir_entry_batch_with_walk_stall(
mut entries: std::fs::ReadDir,
result_limit: usize,
stall: Option<Duration>,
) -> Result<(std::fs::ReadDir, Vec<String>, bool)> {
let io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
let (progress_tx, progress_rx) = tokio::sync::mpsc::unbounded_channel();
let batch = spawn_blocking_with_disk_io_permit(io_permit, move || {
let mut names = Vec::with_capacity(result_limit);
let mut reached_eof = false;
for _ in 0..WALK_DIR_BATCH_SIZE {
if names.len() == result_limit {
break;
}
let Some(entry) = entries.next() else {
reached_eof = true;
break;
};
let _ = progress_tx.send(());
let entry = entry.map_err(to_file_error)?;
let name = entry.file_name().to_string_lossy().to_string();
if name.is_empty() || name == "." || name == ".." {
continue;
}
let file_type = entry.file_type().map_err(to_file_error)?;
let _ = progress_tx.send(());
if file_type.is_file() {
names.push(name);
} else if file_type.is_dir() {
names.push(format!("{name}{SLASH_SEPARATOR}"));
}
}
Ok::<_, DiskError>((entries, names, reached_eof))
});
await_walk_stall_progress(stall, batch, progress_rx).await
}
impl FileCacheReclaimReader {
fn new(inner: File, reclaim_offset: u64, reclaim_len: usize, reclaim_on_drop: bool) -> Self {
#[cfg(target_os = "macos")]
@@ -6899,7 +6951,8 @@ impl LocalDisk {
let res = {
if read_data {
self.read_all_data_with_dmtime(bucket, volume_dir, meta_path).await
let io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
self.read_all_data_with_dmtime(bucket, volume_dir, meta_path, io_permit).await
} else {
match self.read_metadata_with_dmtime(meta_path).await {
Ok(res) => Ok(res),
@@ -6932,6 +6985,7 @@ impl LocalDisk {
#[hotpath::measure(impl_type = "LocalDisk")]
async fn read_metadata_with_dmtime(&self, file_path: impl AsRef<Path>) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
check_path_length(file_path.as_ref().to_string_lossy().as_ref())?;
let io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
// HP-12 item 1 (sub-change A): fold the open + fstat + bounded xl.meta
// read into a single spawn_blocking dispatch instead of three separate
@@ -6944,30 +6998,31 @@ impl LocalDisk {
// - metadata failure -> to_file_error
// - parse failure -> propagated verbatim from read_xl_meta_no_data_sync (`?`)
let path = file_path.as_ref().to_path_buf();
let (data, modtime) = tokio::task::spawn_blocking(move || -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
// Read-only open, equivalent to O_RDONLY (get_readonly_options only sets read(true)).
let mut f = std::fs::File::open(&path).map_err(to_file_error)?;
let (data, modtime) =
spawn_blocking_with_disk_io_permit(io_permit, move || -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
// Read-only open, equivalent to O_RDONLY (get_readonly_options only sets read(true)).
let mut f = std::fs::File::open(&path).map_err(to_file_error)?;
let meta = f.metadata().map_err(to_file_error)?;
let meta = f.metadata().map_err(to_file_error)?;
if meta.is_dir() {
// fix use io::Error
return Err(Error::FileNotFound);
}
if meta.is_dir() {
// fix use io::Error
return Err(Error::FileNotFound);
}
let size = meta.len() as usize;
let size = meta.len() as usize;
let data = read_xl_meta_no_data_sync(&mut f, size)?;
let data = read_xl_meta_no_data_sync(&mut f, size)?;
let modtime = match meta.modified() {
Ok(md) => Some(OffsetDateTime::from(md)),
Err(_) => None,
};
let modtime = match meta.modified() {
Ok(md) => Some(OffsetDateTime::from(md)),
Err(_) => None,
};
Ok((data, modtime))
})
.await
.map_err(DiskError::from)??;
Ok((data, modtime))
})
.await
.map_err(DiskError::from)??;
Ok((data, modtime))
}
@@ -6975,11 +7030,12 @@ impl LocalDisk {
#[hotpath::measure(impl_type = "LocalDisk")]
async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
// TODO(backlog): add configurable timeout for read_all_data operations
let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path, None).await?;
Ok(data)
}
async fn read_part_metadata(&self, bucket: &str, volume_dir: &Path, path_str: &str) -> Result<ObjectPartInfo> {
let io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
let path = Path::new(path_str);
let num = path
.file_name()
@@ -6999,7 +7055,7 @@ impl LocalDisk {
let metadata_path = self.io_get_object_path(bucket, path.to_string_lossy().as_ref());
// A part's existence check, metadata read and decode share one dispatch.
// Keep open errors unmapped for the existing missing-volume fallback.
let result = tokio::task::spawn_blocking(move || -> Result<_> {
let result = spawn_blocking_with_disk_io_permit(io_permit, move || -> Result<_> {
let part_error = |error: String| ObjectPartInfo {
number: num,
error: Some(error),
@@ -7027,10 +7083,11 @@ impl LocalDisk {
}
async fn read_listing_metadata(&self, volume: &str, object_name: &str) -> Result<ListingMetadataRead> {
let io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
let object_dir = self.io_get_object_path(volume, object_name)?;
let metadata_path = object_dir.join(STORAGE_FORMAT_FILE);
let volume_dir = self.io_get_bucket_path(volume)?;
let result = tokio::task::spawn_blocking(move || {
let result = spawn_blocking_with_disk_io_permit(io_permit, move || {
let (bytes, _) = read_all_data_std(&metadata_path)?;
let file_meta = FileMeta::load(&bytes).ok();
let data_dirs: HashSet<String> = file_meta
@@ -7113,6 +7170,7 @@ impl LocalDisk {
volume: &str,
volume_dir: impl AsRef<Path>,
file_path: impl AsRef<Path>,
io_permit: Option<DiskIoPermit>,
) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
// HP-12 item 1 (sub-change A): fold open + fstat + is_dir + try_reserve +
// read_to_end into a single spawn_blocking dispatch. The closure mirrors
@@ -7128,7 +7186,7 @@ impl LocalDisk {
// gating the volume fallback on the open error alone is equivalent to
// the original code, where the fallback lived solely in the open match arm.
let path = file_path.as_ref().to_path_buf();
let res = tokio::task::spawn_blocking(move || read_all_data_std(&path))
let res = spawn_blocking_with_disk_io_permit(io_permit, move || read_all_data_std(&path))
.await
.map_err(DiskError::from)?;
@@ -7215,7 +7273,10 @@ impl LocalDisk {
.await;
}
let (data, _) = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await {
let (data, _) = match self
.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath, None)
.await
{
Ok(data) => data,
Err(DiskError::FileNotFound) => {
// `deleted` alone can be an explicit marker purge; only
@@ -8352,13 +8413,11 @@ impl LocalDisk {
if err == Error::FileNotFound || err == Error::IsNotRegular {
// NOT an object, append to stack (with slash)
// If dirObject, but no metadata (which is unexpected) we skip it.
if !is_dir_obj
&& !with_walk_stall_deadline(
stall,
is_empty_dir(self.io_get_object_path(&opts.bucket, &meta.name)?),
)
.await?
{
if !is_dir_obj && {
let _io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
!with_walk_stall_deadline(stall, is_empty_dir(self.io_get_object_path(&opts.bucket, &meta.name)?))
.await?
} {
meta.name.push_str(SLASH_SEPARATOR);
// Conservative listings verify physical prefixes. Never-versioned
// buckets use the bounded fast path, which only has to rule out
@@ -10393,6 +10452,7 @@ impl DiskAPI for LocalDisk {
/// listing resilient while preserving existing full-enumeration behavior.
#[tracing::instrument(level = "trace", skip_all)]
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
let _io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
if !origvolume.is_empty() {
let origvolume_dir = self.io_get_bucket_path(origvolume)?;
if !skip_access_checks(origvolume)
@@ -10405,21 +10465,16 @@ impl DiskAPI for LocalDisk {
let volume_dir = self.io_get_bucket_path(volume)?;
let dir_path_abs = self.io_get_object_path(volume, dir_path.trim_start_matches(SLASH_SEPARATOR))?;
// Whole-directory enumeration in one syscall path (see the wide-directory
// stall hazard on this fn): with `count < 0` this reads every entry, and
// the caller's stall budget bounds the entire call as a unit.
let entries = match os::read_dir(&dir_path_abs, count).await {
Ok(res) => res,
Err(e) => {
if e.kind() == ErrorKind::NotFound
let entries = match read_dir_entries_with_walk_stall(&dir_path_abs, count, None).await {
Ok(entries) => entries,
Err(err)
if err == DiskError::FileNotFound
&& !skip_access_checks(volume)
&& let Err(e) = cached_access(&volume_dir).await
{
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
}
return Err(to_file_error(e).into());
&& let Err(err) = cached_access(&volume_dir).await =>
{
return Err(to_access_error(err, DiskError::VolumeAccessDenied).into());
}
Err(err) => return Err(err),
};
Ok(entries)
@@ -10441,10 +10496,11 @@ impl DiskAPI for LocalDisk {
let volume_dir = self.io_get_bucket_path(&opts.bucket)?;
if !skip_access_checks(&opts.bucket)
&& let Err(e) = with_walk_stall_deadline(stall, cached_access(&volume_dir)).await?
{
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
if !skip_access_checks(&opts.bucket) {
let _io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
if let Err(e) = with_walk_stall_deadline(stall, cached_access(&volume_dir)).await? {
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
}
}
let mut wr = wr;
@@ -10483,8 +10539,10 @@ impl DiskAPI for LocalDisk {
let fpath =
self.io_get_object_path(&opts.bucket, path_join_buf(&[opts.base_dir.as_str(), STORAGE_FORMAT_FILE]).as_str())?;
if let Ok(meta) = with_walk_stall_deadline(stall, tokio::fs::metadata(&fpath)).await?
&& meta.is_file()
if let Ok(meta) = {
let _io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
with_walk_stall_deadline(stall, tokio::fs::metadata(&fpath)).await?
} && meta.is_file()
{
skip_current_dir_object = true;
if let Ok(meta_bytes) = with_walk_stall_deadline(
@@ -11209,11 +11267,7 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(level = "trace", skip_all)]
async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
crate::hp_guard!("LocalDisk::read_metadata");
let file_path = self.io_get_object_path(volume, path)?;
let volume_dir = self.io_get_bucket_path(volume)?;
let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
Ok(data.into())
self.read_metadata_with_permit(volume, path, None).await
}
}
@@ -11233,6 +11287,25 @@ fn should_read_legacy_inline_part(fi: &FileInfo, storage_class_config: &crate::c
}
impl LocalDisk {
pub(crate) async fn read_metadata_with_permit(
&self,
volume: &str,
path: &str,
io_permit: Option<DiskIoPermit>,
) -> Result<Bytes> {
crate::hp_guard!("LocalDisk::read_metadata");
let io_permit = match io_permit {
Some(io_permit) => Some(io_permit),
None => acquire_context_disk_io_permit().await.map_err(DiskError::Io)?,
};
let file_path = self.io_get_object_path(volume, path)?;
let volume_dir = self.io_get_bucket_path(volume)?;
let (data, _) = self
.read_all_data_with_dmtime(volume, volume_dir, file_path, io_permit)
.await?;
Ok(data.into())
}
pub(crate) async fn rename_data_borrowed(
&self,
src_volume: &str,
@@ -11303,6 +11376,42 @@ mod test {
use tokio::io::{AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf};
use tracing_subscriber::fmt::MakeWriter;
#[cfg(unix)]
#[tokio::test]
async fn read_dir_entries_skip_symlinks_without_truncating_listing() {
use std::os::unix::fs::symlink;
let dir = tempfile::tempdir().expect("directory should be created");
std::fs::write(dir.path().join("before"), b"file").expect("file should be created");
symlink(dir.path().join("before"), dir.path().join("link")).expect("symlink should be created");
std::fs::write(dir.path().join("after"), b"file").expect("file should be created");
let names = read_dir_entries_with_walk_stall(dir.path(), -1, None)
.await
.expect("directory listing should complete");
assert_eq!(names.len(), 2);
assert!(names.iter().any(|name| name == "before"));
assert!(names.iter().any(|name| name == "after"));
assert!(!names.iter().any(|name| name == "link"));
}
#[tokio::test]
async fn stalled_walk_batch_reports_timeout_at_one_stall_interval() {
let (release_tx, release_rx) = std::sync::mpsc::channel();
let task = tokio::task::spawn_blocking(move || {
let _ = release_rx.recv();
Ok::<_, DiskError>(())
});
let (_progress_tx, progress_rx) = tokio::sync::mpsc::unbounded_channel();
let started = Instant::now();
let result = await_walk_stall_progress(Some(Duration::from_millis(50)), task, progress_rx).await;
assert_eq!(result, Err(DiskError::Timeout));
assert!(started.elapsed() < Duration::from_millis(500));
release_tx.send(()).expect("blocking worker should be released");
}
#[derive(Clone, Default)]
struct CapturedLogs {
buffer: Arc<Mutex<Vec<u8>>>,
crates/ecstore/src/disk/mod.rs
+6
View File
@@ -22,9 +22,15 @@ pub mod error_reduce;
pub mod format;
pub mod fs;
pub mod health_state;
mod io_admission;
pub mod local;
pub mod os;
#[cfg(test)]
pub(crate) use io_admission::DiskIoClass;
pub use io_admission::with_background_disk_io;
pub(crate) use io_admission::{acquire_context_disk_io_permit, current_disk_io_context, with_optional_disk_io_context};
pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
pub const MIGRATING_META_BUCKET: &str = ".minio.sys";
pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart";
crates/ecstore/src/lib.rs
+1
View File
@@ -55,6 +55,7 @@ mod set_disk;
mod storage_api_contracts;
mod store;
pub use disk::with_background_disk_io;
pub use store::PoolMetaWriteGateStatus;
// pub mod checksum;
crates/ecstore/src/runtime/instance.rs
+36 -12
View File
@@ -72,10 +72,19 @@ pub(crate) const SCANNER_PUBLICATION_LEASE_MAX_ENTRIES: usize = 256;
/// have started movement after the lease was abandoned.
pub(crate) const SCANNER_PUBLICATION_LEASE_TTL: std::time::Duration = std::time::Duration::from_secs(60);
/// Observational leases hold data-movement fencing but may coexist with
/// ordinary namespace commits; authoritative leases require a stable namespace.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum ScannerPublicationLeasePurpose {
Authoritative,
Observational,
}
pub(crate) struct ScannerPublicationLeaseEntry {
pub(crate) expires_at: Instant,
pub(crate) movement_generation: u64,
pub(crate) namespace_generation: u64,
pub(crate) namespace_generation: Option<u64>,
pub(crate) purpose: ScannerPublicationLeasePurpose,
pub(crate) _operation_guard: OwnedRwLockReadGuard<()>,
}
@@ -306,7 +315,8 @@ impl InstanceContext {
token: Uuid,
expires_at: Instant,
movement_generation: u64,
namespace_generation: u64,
namespace_generation: Option<u64>,
purpose: ScannerPublicationLeasePurpose,
operation_guard: OwnedRwLockReadGuard<()>,
) -> bool {
let mut leases = self.scanner_publication_leases.lock().await;
@@ -319,6 +329,7 @@ impl InstanceContext {
expires_at,
movement_generation,
namespace_generation,
purpose,
_operation_guard: operation_guard,
},
);
@@ -329,21 +340,24 @@ impl InstanceContext {
self.scanner_publication_leases.lock().await.remove(&token).is_some()
}
/// Return both generations from the same live lease while the caller holds
/// the movement read guard. Namespace commits do not take that guard, so
/// the caller must compare the saved namespace generation after this await.
/// Return the generations and purpose from the same live lease while the
/// caller holds the movement read guard. Namespace commits do not take that
/// guard, so strict leases compare their saved generation after this await.
/// The process-owned table rejects tokens from a prior instance or expiry.
pub(crate) async fn scanner_publication_lease_generations(&self, token: Uuid) -> Option<(u64, u64)> {
pub(crate) async fn scanner_publication_lease_generations(
&self,
token: Uuid,
) -> Option<(u64, Option<u64>, ScannerPublicationLeasePurpose)> {
let mut leases = self.scanner_publication_leases.lock().await;
let now = Instant::now();
let (expires_at, movement_generation, namespace_generation) = leases
let (expires_at, movement_generation, namespace_generation, purpose) = leases
.get(&token)
.map(|entry| (entry.expires_at, entry.movement_generation, entry.namespace_generation))?;
.map(|entry| (entry.expires_at, entry.movement_generation, entry.namespace_generation, entry.purpose))?;
if expires_at <= now {
leases.remove(&token);
return None;
}
Some((movement_generation, namespace_generation))
Some((movement_generation, namespace_generation, purpose))
}
pub(crate) async fn expire_scanner_publication_lease(&self, token: Uuid, expires_at: Instant) {
@@ -859,12 +873,22 @@ mod tests {
let gate = ctx.data_movement_operation_gate();
let permit = gate.clone().read_owned().await;
assert!(
ctx.install_scanner_publication_lease(token, Instant::now() + SCANNER_PUBLICATION_LEASE_TTL, 7, 11, permit)
.await
ctx.install_scanner_publication_lease(
token,
Instant::now() + SCANNER_PUBLICATION_LEASE_TTL,
7,
Some(11),
ScannerPublicationLeasePurpose::Authoritative,
permit,
)
.await
);
drop(ctx.begin_namespace_commit());
assert_eq!(ctx.namespace_commit_generation(), 2);
assert_eq!(ctx.scanner_publication_lease_generations(token).await, Some((7, 11)));
assert_eq!(
ctx.scanner_publication_lease_generations(token).await,
Some((7, Some(11), ScannerPublicationLeasePurpose::Authoritative))
);
assert!(gate.clone().try_write_owned().is_err(), "lookup must retain the stored permit");
tokio::time::advance(SCANNER_PUBLICATION_LEASE_TTL).await;
assert_eq!(ctx.scanner_publication_lease_generations(token).await, None);
crates/ecstore/src/services/notification_sys.rs
+20 -1
View File
@@ -2888,8 +2888,23 @@ impl NotificationSys {
/// leases are released before returning so a partial acquisition cannot
/// pin movement on one peer.
pub async fn acquire_scanner_publication_leases(
&self,
targets: Vec<(String, String, u64)>,
) -> Result<Vec<ScannerPublicationLeaseGrant>> {
self.acquire_scanner_publication_leases_with_purpose(targets, false).await
}
pub async fn acquire_observational_scanner_publication_leases(
&self,
targets: Vec<(String, String, u64)>,
) -> Result<Vec<ScannerPublicationLeaseGrant>> {
self.acquire_scanner_publication_leases_with_purpose(targets, true).await
}
async fn acquire_scanner_publication_leases_with_purpose(
&self,
mut targets: Vec<(String, String, u64)>,
observational_only: bool,
) -> Result<Vec<ScannerPublicationLeaseGrant>> {
targets.sort_by(|left, right| left.0.cmp(&right.0));
for pair in targets.windows(2) {
@@ -2910,7 +2925,10 @@ impl NotificationSys {
let _ = self.release_scanner_publication_leases(grants).await;
return Err(Error::other(format!("scanner publication lease peer {host} is unavailable")));
};
match client.acquire_scanner_publication_lease(&session_id, generation).await {
match client
.acquire_scanner_publication_lease(&session_id, generation, observational_only)
.await
{
Ok(lease) => grants.push(ScannerPublicationLeaseGrant { host, lease }),
Err(err) => {
let _ = self.release_scanner_publication_leases(grants).await;
@@ -4949,6 +4967,7 @@ mod tests {
lease: ScannerPublicationLease {
token: Uuid::new_v4(),
movement_generation: 3,
observational_only: false,
owner_id: Uuid::new_v4().to_string(),
session_id: "session-a".to_string(),
expires_at: Instant::now() + Duration::from_secs(30),
crates/ecstore/src/store/mod.rs
+171 -19
View File
@@ -42,7 +42,7 @@ use crate::error::{
is_err_strict_volume_not_found, is_err_version_not_found, to_object_err,
};
use crate::runtime::global::DISK_RESERVE_FRACTION;
use crate::runtime::instance::InstanceContext;
use crate::runtime::instance::{InstanceContext, ScannerPublicationLeasePurpose};
use crate::runtime::sources as runtime_sources;
use crate::services::rebalance::{RebalStatus, RebalanceMeta, is_rebalance_conflicting_with_decommission};
use crate::storage_api_contracts::{
@@ -1098,36 +1098,67 @@ impl ECStore {
&self,
expected_generation: u64,
ttl: std::time::Duration,
) -> Result<(Uuid, u64)> {
self.acquire_scanner_publication_lease_with_purpose(expected_generation, ttl, false)
.await
}
pub async fn acquire_scanner_publication_lease_with_purpose(
&self,
expected_generation: u64,
ttl: std::time::Duration,
observational_only: bool,
) -> Result<(Uuid, u64)> {
if ttl != crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL {
return Err(Error::other("scanner publication lease TTL is not supported"));
}
// Bind the original activity generation across the asynchronous checks;
// a completed namespace commit must never refresh an existing proof.
let namespace_generation = self.scanner_namespace_mutation_generation();
let purpose = if observational_only {
ScannerPublicationLeasePurpose::Observational
} else {
ScannerPublicationLeasePurpose::Authoritative
};
// Strict leases bind the namespace generation before their async
// admission checks. An observation only needs the movement fence, but
// still fails closed if generation tracking is exhausted.
let observed_namespace_generation = self.scanner_namespace_mutation_generation();
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.data_movement_generation_exhausted()
|| self.ctx.data_movement_operation_epoch_exhausted()
|| self.ctx.data_movement_generation() != expected_generation
|| namespace_generation == u64::MAX
|| observed_namespace_generation == u64::MAX
{
return Err(Error::other("scanner publication lease generation is stale"));
}
if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
if self.scanner_data_movement_snapshot_locked().await.1 {
return Err(Error::other("scanner publication lease is blocked by data movement"));
}
if self.scanner_namespace_mutation_generation() != namespace_generation {
return Err(Error::other("scanner publication lease generation is stale"));
}
let namespace_generation = if observational_only {
None
} else {
if self.ctx.namespace_commits_pending() {
return Err(Error::other("scanner publication lease is blocked by pending namespace commit"));
}
if self.scanner_namespace_mutation_generation() != observed_namespace_generation {
return Err(Error::other("scanner publication lease namespace changed during acquisition"));
}
Some(observed_namespace_generation)
};
let token = Uuid::new_v4();
let expires_at = tokio::time::Instant::now() + ttl;
if !self
.ctx
.install_scanner_publication_lease(token, expires_at, expected_generation, namespace_generation, operation_guard)
.install_scanner_publication_lease(
token,
expires_at,
expected_generation,
namespace_generation,
purpose,
operation_guard,
)
.await
{
return Err(Error::other("scanner publication lease capacity is exhausted"));
@@ -1151,24 +1182,46 @@ impl ECStore {
/// admission; a restarted context has no old token and therefore fails
/// closed even if its generation counter has returned to zero.
pub async fn validate_scanner_publication_lease(&self, token: Uuid, expected_generation: u64) -> Result<()> {
let _operation_guard = self.acquire_scanner_publication_lease_guard(token).await?;
self.validate_scanner_publication_lease_with_purpose(token, expected_generation, false)
.await
}
pub async fn validate_scanner_publication_lease_with_purpose(
&self,
token: Uuid,
expected_generation: u64,
observational_only: bool,
) -> Result<()> {
let expected_purpose = if observational_only {
ScannerPublicationLeasePurpose::Observational
} else {
ScannerPublicationLeasePurpose::Authoritative
};
let _operation_guard = self
.acquire_scanner_publication_lease_guard_with_target(token, None, true, Some(expected_purpose))
.await?;
if self.ctx.data_movement_generation_exhausted()
|| self.ctx.data_movement_operation_epoch_exhausted()
|| self.ctx.data_movement_generation() != expected_generation
{
return Err(Error::other("scanner publication lease generation is stale"));
}
if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
if self.scanner_data_movement_snapshot_locked().await.1 {
return Err(Error::other("scanner publication lease is blocked by data movement"));
}
let Some((lease_generation, lease_namespace_generation)) = self.ctx.scanner_publication_lease_generations(token).await
let Some((lease_generation, lease_namespace_generation, purpose)) =
self.ctx.scanner_publication_lease_generations(token).await
else {
return Err(Error::other("scanner publication lease is unknown or expired"));
};
if purpose == ScannerPublicationLeasePurpose::Authoritative && self.ctx.namespace_commits_pending() {
return Err(Error::other("scanner publication lease is blocked by pending namespace commit"));
}
let namespace_generation = self.scanner_namespace_mutation_generation();
if lease_generation != self.ctx.data_movement_generation()
|| lease_namespace_generation != namespace_generation
|| namespace_generation == u64::MAX
|| (purpose == ScannerPublicationLeasePurpose::Authoritative
&& lease_namespace_generation != Some(namespace_generation))
{
return Err(Error::other("scanner publication lease generation is stale"));
}
@@ -1181,22 +1234,65 @@ impl ECStore {
/// restart, expiry, generation changes, or blocked movement all reject it
/// before the target disk is touched.
pub async fn acquire_scanner_publication_lease_guard(&self, token: Uuid) -> Result<tokio::sync::OwnedRwLockReadGuard<()>> {
self.acquire_scanner_publication_lease_guard_with_target(
token,
None,
false,
Some(ScannerPublicationLeasePurpose::Authoritative),
)
.await
}
pub async fn acquire_scanner_publication_lease_guard_for_rename(
&self,
token: Uuid,
volume: &str,
path: &str,
) -> Result<tokio::sync::OwnedRwLockReadGuard<()>> {
let target = format!(
"{}/{}",
crate::disk::BUCKET_META_PREFIX,
rustfs_data_usage::DATA_USAGE_OBSERVED_OBJECT_NAME
);
let target = (volume == RUSTFS_META_BUCKET && path == target).then_some((volume, path));
self.acquire_scanner_publication_lease_guard_with_target(token, target, false, None)
.await
}
async fn acquire_scanner_publication_lease_guard_with_target(
&self,
token: Uuid,
observed_target: Option<(&str, &str)>,
validation_only: bool,
expected_purpose: Option<ScannerPublicationLeasePurpose>,
) -> Result<tokio::sync::OwnedRwLockReadGuard<()>> {
let operation_gate = self.ctx.data_movement_operation_gate();
let operation_guard = operation_gate.read_owned().await;
if self.ctx.data_movement_generation_exhausted() || self.ctx.data_movement_operation_epoch_exhausted() {
return Err(Error::other("scanner publication lease generation is exhausted"));
}
if self.scanner_data_movement_snapshot_locked().await.1 || self.ctx.namespace_commits_pending() {
if self.scanner_data_movement_snapshot_locked().await.1 {
return Err(Error::other("scanner publication lease is blocked by data movement"));
}
let Some((lease_generation, lease_namespace_generation)) = self.ctx.scanner_publication_lease_generations(token).await
let Some((lease_generation, lease_namespace_generation, purpose)) =
self.ctx.scanner_publication_lease_generations(token).await
else {
return Err(Error::other("scanner publication lease is unknown or expired"));
};
if expected_purpose.is_some_and(|expected| expected != purpose) {
return Err(Error::other("scanner publication lease purpose does not match its request"));
}
if purpose == ScannerPublicationLeasePurpose::Observational && observed_target.is_none() && !validation_only {
return Err(Error::other("observational lease is limited to observed usage publication"));
}
if purpose == ScannerPublicationLeasePurpose::Authoritative && self.ctx.namespace_commits_pending() {
return Err(Error::other("scanner publication lease is blocked by pending namespace commit"));
}
let namespace_generation = self.scanner_namespace_mutation_generation();
if lease_generation != self.ctx.data_movement_generation()
|| lease_namespace_generation != namespace_generation
|| namespace_generation == u64::MAX
|| (purpose == ScannerPublicationLeasePurpose::Authoritative
&& lease_namespace_generation != Some(namespace_generation))
{
return Err(Error::other("scanner publication lease generation is stale"));
}
@@ -2571,6 +2667,59 @@ mod tests {
assert!(!store.release_scanner_publication_lease(expiring_token).await);
}
#[tokio::test]
async fn observational_lease_only_allows_observed_usage_rename_during_namespace_commits() {
let ctx = Arc::new(InstanceContext::new());
let store = build_store_with_ctx(ctx.clone());
let namespace_commit = ctx.begin_namespace_commit();
let strict_error = store
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect_err("authoritative lease must wait for namespace commits");
assert!(strict_error.to_string().contains("pending namespace commit"));
let (token, movement_generation) = store
.acquire_scanner_publication_lease_with_purpose(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL, true)
.await
.expect("an observation may lease-fence movement while a namespace commit is active");
assert_eq!(movement_generation, 0);
store
.acquire_scanner_publication_lease_guard_for_rename(token, RUSTFS_META_BUCKET, "buckets/.usage.observed.json")
.await
.expect("the lease must admit the exact observational object");
assert!(
store
.acquire_scanner_publication_lease_guard_for_rename(token, RUSTFS_META_BUCKET, "buckets/.usage.v2.json")
.await
.is_err(),
"an observational lease must not admit the authoritative object"
);
assert!(
store.acquire_scanner_publication_lease_guard(token).await.is_err(),
"an observational lease must not authorize an unscoped mutation such as delete or cleanup"
);
drop(namespace_commit);
assert!(store.ctx.namespace_commit_generation() > 0);
store
.validate_scanner_publication_lease_with_purpose(token, movement_generation, true)
.await
.expect("ordinary namespace generation changes must not invalidate an observation");
let error = store
.validate_scanner_publication_lease_with_purpose(token, movement_generation, false)
.await
.expect_err("the same token must not validate as authoritative");
assert!(error.to_string().contains("purpose"));
let operation_gate = store.ctx.data_movement_operation_gate();
assert!(
operation_gate.clone().try_write_owned().is_err(),
"the observational lease must still block data movement"
);
assert!(store.release_scanner_publication_lease(token).await);
assert!(operation_gate.try_write_owned().is_ok());
}
#[tokio::test]
async fn scanner_publication_lease_rejects_a_new_movement_generation() {
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
@@ -2613,8 +2762,11 @@ mod tests {
let error = tokio::time::timeout(Duration::from_secs(1), acquire)
.await
.expect("snapshot admission must finish")
.expect_err("acquisition must preserve its original namespace generation");
assert_eq!(error.to_string(), "Io error: scanner publication lease generation is stale");
.expect_err("a namespace change during admission must not install a lease");
assert_eq!(
error.to_string(),
"Io error: scanner publication lease namespace changed during acquisition"
);
assert!(ctx.data_movement_operation_gate().try_write_owned().is_ok(), "no lease was installed");
}
crates/ecstore/src/store/peer.rs
+69 -2
View File
@@ -232,7 +232,7 @@ impl ECStore {
) -> DiskResult<RenameDataResp> {
let external_guard: Option<Arc<dyn Send + Sync>> = if let Some(token) = scanner_token {
Some(Arc::new(
self.acquire_scanner_publication_lease_guard(token)
self.acquire_scanner_publication_lease_guard_for_rename(token, destination.0, destination.1)
.await
.map_err(|err| DiskError::other(err.to_string()))?,
))
@@ -1191,6 +1191,69 @@ mod tests {
assert!(!ctx.namespace_commits_pending());
}
#[tokio::test]
async fn observational_lease_renames_only_the_observed_usage_object() {
let ctx = Arc::new(InstanceContext::new());
let root = tempfile::tempdir().expect("target root");
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
let store = super::super::tests::build_store_with_ctx(ctx.clone());
let disk_ref = disk.endpoint().to_string();
let volume = crate::disk::RUSTFS_META_BUCKET;
let observed_path = format!(
"{}/{}",
crate::disk::BUCKET_META_PREFIX,
rustfs_data_usage::DATA_USAGE_OBSERVED_OBJECT_NAME
);
let source_path = format!("{}/.staged-observation", crate::disk::BUCKET_META_PREFIX);
let fi = target_file_info(&observed_path, Uuid::new_v4(), b"non-authoritative");
seed_target(&disk, volume, &source_path, fi.clone()).await;
let pending = ctx.begin_namespace_commit();
let (token, generation) = store
.acquire_scanner_publication_lease_with_purpose(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL, true)
.await
.expect("observational lease should permit a pending namespace commit");
let error = store
.rename_local_data(&disk_ref, (volume, &source_path), &fi, (volume, "buckets/.usage.v2.json"), Some(token))
.await
.expect_err("observational token must not authorize the authoritative usage path");
assert!(error.to_string().contains("observed usage publication"));
assert!(root.path().join(volume).join(&source_path).join("xl.meta").exists());
store
.rename_local_data(&disk_ref, (volume, &source_path), &fi, (volume, &observed_path), Some(token))
.await
.expect("observational token may rename only the observed usage key");
assert!(
store
.validate_scanner_publication_lease_with_purpose(token, generation, true)
.await
.is_ok(),
"namespace commits do not invalidate the movement-only observation lease"
);
let wrong_purpose = store
.validate_scanner_publication_lease_with_purpose(token, generation, false)
.await
.expect_err("the same token must not validate as authoritative");
assert!(wrong_purpose.to_string().contains("purpose"));
drop(pending);
assert!(store.release_scanner_publication_lease(token).await);
let observed = disk
.read_version(
volume,
volume,
&observed_path,
&fi.version_id.expect("fixture version").to_string(),
&crate::disk::ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("observed metadata should be present");
assert_eq!(observed.data.as_deref(), Some(&b"non-authoritative"[..]));
}
#[tokio::test]
async fn completed_namespace_commit_rejects_stale_scanner_target_admission() {
use futures::FutureExt;
@@ -1491,7 +1554,11 @@ mod tests {
);
assert_eq!(
ctx.scanner_publication_lease_generations(old_token).await,
Some((movement_generation, namespace_generation)),
Some((
movement_generation,
Some(namespace_generation),
crate::runtime::instance::ScannerPublicationLeasePurpose::Authoritative,
)),
"rejection must not refresh or discard the old lease"
);
let (fresh_token, fresh_generation) = store
crates/ecstore/tests/bucket_config_goldens.rs
+495
View File
@@ -0,0 +1,495 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Byte-level goldens for the 13 persisted bucket-configuration XML families
//! (rustfs/backlog#2744).
//!
//! Responsible for: pinning the exact bytes that `deserialize` followed by
//! `serialize` produces for every family in its three source shapes (AWS
//! documentation shape, MinIO-written shape, RustFS-written shape), and the
//! exact bytes RustFS writes for a typed value. Only bytes are compared; a
//! structural comparison would let a codec replacement change the persisted
//! encoding without failing here.
//!
//! Not responsible for: validating configurations, touching the bucket
//! metadata store, or choosing a canonical shape.
//!
//! Upstream: `rustfs_ecstore::api::bucket::utils::{deserialize, serialize}`
//! and the s3s DTOs they are generic over. Downstream: the DTO/codec
//! replacement (T1.4) and the persisted-bytes comparison (T3.7), which must
//! keep every golden green without regenerating it. Regeneration is reserved
//! to `scripts/gen_bucket_config_goldens.sh`, which sets
//! `BUCKET_CONFIG_GOLDENS_WRITE`: in that mode every test writes its files
//! and then fails on purpose, so a write run never prints a green line.
use rustfs_ecstore::api::bucket::utils::{deserialize, serialize};
use s3s::dto::{
AbortIncompleteMultipartUpload, AccelerateConfiguration, BucketAccelerateStatus, BucketLifecycleConfiguration,
BucketLoggingStatus, BucketVersioningStatus, CORSConfiguration, CORSRule, Condition, DefaultRetention, DelMarkerExpiration,
DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination,
ErrorDocument, Event, ExcludedPrefix, ExistingObjectReplication, ExistingObjectReplicationStatus, ExpirationStatus,
FilterRule, FilterRuleName, IndexDocument, LifecycleExpiration, LifecycleRule, LifecycleRuleAndOperator, LifecycleRuleFilter,
LoggingEnabled, NoncurrentVersionExpiration, NotificationConfiguration, NotificationConfigurationFilter,
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, Payer, PublicAccessBlockConfiguration,
QueueConfiguration, Redirect, ReplicaModifications, ReplicaModificationsStatus, ReplicationConfiguration, ReplicationRule,
ReplicationRuleAndOperator, ReplicationRuleFilter, ReplicationRuleStatus, RequestPaymentConfiguration, RoutingRule,
S3KeyFilter, ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration,
ServerSideEncryptionRule, SourceSelectionCriteria, StorageClass, Tag, Tagging, Transition, TransitionStorageClass,
VersioningConfiguration, WebsiteConfiguration,
};
use s3s::xml;
use std::path::{Path, PathBuf};
/// Set only by `scripts/gen_bucket_config_goldens.sh`. Every pair test then
/// writes its `.out.xml` (and, for the `rustfs` shape, its `.in.xml`) and
/// fails on purpose, so a write run can never be mistaken for a verification.
const WRITE_ENV: &str = "BUCKET_CONFIG_GOLDENS_WRITE";
fn goldens_dir() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("tests")
.join("fixtures")
.join("bucket-config-goldens")
}
fn pair_path(family: &str, shape: &str, side: &str) -> PathBuf {
goldens_dir().join(family).join(format!("{shape}.{side}.xml"))
}
fn read(path: &Path) -> Vec<u8> {
std::fs::read(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display()))
}
fn write(path: &Path, bytes: &[u8]) {
std::fs::write(path, bytes).unwrap_or_else(|err| panic!("write {}: {err}", path.display()));
}
fn contains(haystack: &[u8], needle: &[u8]) -> bool {
haystack.windows(needle.len()).any(|window| window == needle)
}
fn round_trip<T>(input: &[u8], label: &str) -> Vec<u8>
where
T: for<'xml> xml::Deserialize<'xml> + xml::Serialize,
{
let parsed: T = deserialize(input).unwrap_or_else(|err| panic!("{label}: deserialize failed: {err}"));
serialize(&parsed).unwrap_or_else(|err| panic!("{label}: serialize failed: {err}"))
}
/// Byte comparison with a readable report. Parsed values are never compared.
fn assert_bytes_eq(actual: &[u8], expected: &[u8], context: &str) {
if actual != expected {
panic!(
"{context}\n--- expected ({} bytes) ---\n{}\n--- actual ({} bytes) ---\n{}\n",
expected.len(),
String::from_utf8_lossy(expected),
actual.len(),
String::from_utf8_lossy(actual),
);
}
}
/// Checks one `<family>/<shape>` pair. `typed` is the value RustFS writes for
/// the `rustfs` shape; the `aws` and `minio` shapes pass `None`.
fn check_pair<T>(family: &str, shape: &str, typed: Option<&T>)
where
T: for<'xml> xml::Deserialize<'xml> + xml::Serialize,
{
let in_path = pair_path(family, shape, "in");
let out_path = pair_path(family, shape, "out");
let label = format!("{family}/{shape}");
if std::env::var_os(WRITE_ENV).is_some() {
if let Some(value) = typed {
let written = serialize(value).unwrap_or_else(|err| panic!("{label}: typed value failed to serialize: {err}"));
write(&in_path, &written);
}
let input = read(&in_path);
write(&out_path, &round_trip::<T>(&input, &label));
panic!(
"{WRITE_ENV} is set: wrote {} without verifying anything; rerun without {WRITE_ENV}",
out_path.display()
);
}
let input = read(&in_path);
let expected = read(&out_path);
if let Some(value) = typed {
let written = serialize(value).unwrap_or_else(|err| panic!("{label}: typed value failed to serialize: {err}"));
assert_bytes_eq(
&written,
&input,
&format!(
"{label}: RustFS no longer writes the pinned bytes for the typed value ({})",
in_path.display()
),
);
}
let actual = round_trip::<T>(&input, &label);
assert_bytes_eq(
&actual,
&expected,
&format!("{label}: deserialize->serialize bytes drifted from {}", out_path.display()),
);
let again = round_trip::<T>(&expected, &label);
assert_bytes_eq(
&again,
&expected,
&format!("{label}: the golden is not a fixed point of deserialize->serialize"),
);
}
/// The typed values behind every `rustfs.in.xml`: what RustFS writes today
/// through `serialize` when it persists a configuration it built itself.
/// MinIO-only members (`DelMarkerExpiration`, `ExcludeFolders`,
/// `ExcludedPrefixes`, `DeleteReplication`) are included where RustFS accepts
/// them, so the `rustfs` shape is the one that pins their encoding.
mod rustfs_values {
use super::*;
fn tag(key: &str, value: &str) -> Tag {
Tag {
key: Some(key.to_owned()),
value: Some(value.to_owned()),
}
}
pub(super) fn notification() -> NotificationConfiguration {
NotificationConfiguration {
queue_configurations: Some(vec![QueueConfiguration {
events: vec![
Event::from("s3:ObjectCreated:*".to_owned()),
Event::from("s3:ObjectRemoved:*".to_owned()),
],
filter: Some(NotificationConfigurationFilter {
key: Some(S3KeyFilter {
filter_rules: Some(vec![FilterRule {
name: Some(FilterRuleName::from_static(FilterRuleName::PREFIX)),
value: Some("uploads/".to_owned()),
}]),
}),
}),
id: Some("rustfs-webhook".to_owned()),
queue_arn: "arn:rustfs:sqs::primary:webhook".to_owned(),
}]),
..Default::default()
}
}
pub(super) fn lifecycle() -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
rules: vec![
LifecycleRule {
abort_incomplete_multipart_upload: Some(AbortIncompleteMultipartUpload {
days_after_initiation: Some(3),
}),
del_marker_expiration: Some(DelMarkerExpiration { days: Some(7) }),
expiration: Some(LifecycleExpiration {
days: Some(30),
..Default::default()
}),
filter: Some(LifecycleRuleFilter {
prefix: Some("tmp/".to_owned()),
..Default::default()
}),
id: Some("expire-tmp".to_owned()),
noncurrent_version_expiration: Some(NoncurrentVersionExpiration {
newer_noncurrent_versions: Some(2),
noncurrent_days: Some(7),
}),
noncurrent_version_transitions: None,
prefix: None,
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
transitions: None,
},
LifecycleRule {
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
expiration: None,
filter: Some(LifecycleRuleFilter {
and: Some(LifecycleRuleAndOperator {
prefix: Some("archive/".to_owned()),
tags: Some(vec![tag("tier", "cold")]),
..Default::default()
}),
..Default::default()
}),
id: Some("tier-cold".to_owned()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
transitions: Some(vec![Transition {
days: Some(90),
storage_class: Some(TransitionStorageClass::from_static("COLDTIER")),
..Default::default()
}]),
},
],
..Default::default()
}
}
pub(super) fn object_lock() -> ObjectLockConfiguration {
ObjectLockConfiguration {
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
rule: Some(ObjectLockRule {
default_retention: Some(DefaultRetention {
days: Some(30),
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::GOVERNANCE)),
years: None,
}),
}),
}
}
pub(super) fn versioning() -> VersioningConfiguration {
VersioningConfiguration {
exclude_folders: Some(true),
excluded_prefixes: Some(vec![
ExcludedPrefix {
prefix: Some("tmp/".to_owned()),
},
ExcludedPrefix {
prefix: Some("cache/".to_owned()),
},
]),
mfa_delete: None,
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
}
}
pub(super) fn encryption() -> ServerSideEncryptionConfiguration {
ServerSideEncryptionConfiguration {
rules: vec![ServerSideEncryptionRule {
apply_server_side_encryption_by_default: Some(ServerSideEncryptionByDefault {
kms_master_key_id: Some("rustfs-bucket-key".to_owned()),
sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
}),
blocked_encryption_types: None,
bucket_key_enabled: Some(false),
}],
}
}
pub(super) fn tagging() -> Tagging {
Tagging {
tag_set: vec![tag("team", "storage"), tag("env", "staging")],
}
}
pub(super) fn replication() -> ReplicationConfiguration {
ReplicationConfiguration {
role: String::new(),
rules: vec![ReplicationRule {
delete_marker_replication: Some(DeleteMarkerReplication {
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
}),
delete_replication: Some(DeleteReplication {
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
}),
destination: Destination {
bucket: "arn:rustfs:replication::0b6f3c2a-5e6a-4d2b-9c1e-7a8f9d0e1b2c:interop-dr".to_owned(),
storage_class: Some(StorageClass::from_static(StorageClass::STANDARD)),
..Default::default()
},
existing_object_replication: Some(ExistingObjectReplication {
status: ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED),
}),
filter: Some(ReplicationRuleFilter {
and: Some(ReplicationRuleAndOperator {
prefix: Some("docs/".to_owned()),
tags: Some(vec![tag("replicate", "yes")]),
}),
..Default::default()
}),
id: Some("rustfs-dr".to_owned()),
prefix: None,
priority: Some(1),
source_selection_criteria: Some(SourceSelectionCriteria {
replica_modifications: Some(ReplicaModifications {
status: ReplicaModificationsStatus::from_static(ReplicaModificationsStatus::ENABLED),
}),
sse_kms_encrypted_objects: None,
}),
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
}],
}
}
pub(super) fn cors() -> CORSConfiguration {
CORSConfiguration {
cors_rules: vec![CORSRule {
allowed_headers: Some(vec!["Authorization".to_owned(), "Content-Type".to_owned()]),
allowed_methods: vec!["GET".to_owned(), "PUT".to_owned()],
allowed_origins: vec!["https://app.example.internal".to_owned()],
expose_headers: Some(vec!["ETag".to_owned(), "x-amz-request-id".to_owned()]),
id: Some("rustfs-app".to_owned()),
max_age_seconds: Some(3600),
}],
}
}
pub(super) fn logging() -> BucketLoggingStatus {
BucketLoggingStatus {
logging_enabled: Some(LoggingEnabled {
target_bucket: "rustfs-access-logs".to_owned(),
target_grants: None,
target_object_key_format: None,
target_prefix: "buckets/interop/".to_owned(),
}),
}
}
pub(super) fn website() -> WebsiteConfiguration {
WebsiteConfiguration {
error_document: Some(ErrorDocument {
key: "404.html".to_owned(),
}),
index_document: Some(IndexDocument {
suffix: "index.html".to_owned(),
}),
redirect_all_requests_to: None,
routing_rules: Some(vec![RoutingRule {
condition: Some(Condition {
http_error_code_returned_equals: None,
key_prefix_equals: Some("legacy/".to_owned()),
}),
redirect: Redirect {
host_name: None,
http_redirect_code: Some("301".to_owned()),
protocol: None,
replace_key_prefix_with: Some("current/".to_owned()),
replace_key_with: None,
},
}]),
}
}
pub(super) fn accelerate() -> AccelerateConfiguration {
AccelerateConfiguration {
status: Some(BucketAccelerateStatus::from_static(BucketAccelerateStatus::SUSPENDED)),
}
}
pub(super) fn request_payment() -> RequestPaymentConfiguration {
RequestPaymentConfiguration {
payer: Payer::from_static(Payer::BUCKET_OWNER),
}
}
pub(super) fn public_access_block() -> PublicAccessBlockConfiguration {
PublicAccessBlockConfiguration {
block_public_acls: Some(true),
block_public_policy: Some(false),
ignore_public_acls: Some(true),
restrict_public_buckets: Some(false),
}
}
}
/// One test per `<family>/<shape>` pair: 13 families x 3 shapes = 39.
mod bucket_config_goldens {
use super::*;
macro_rules! golden {
($name:ident, $family:literal, $shape:literal, $ty:ty) => {
#[test]
fn $name() {
check_pair::<$ty>($family, $shape, None);
}
};
($name:ident, $family:literal, rustfs = $value:expr) => {
#[test]
fn $name() {
check_pair($family, "rustfs", Some(&$value));
}
};
}
golden!(notification_aws, "notification", "aws", NotificationConfiguration);
golden!(notification_minio, "notification", "minio", NotificationConfiguration);
golden!(notification_rustfs, "notification", rustfs = rustfs_values::notification());
golden!(lifecycle_aws, "lifecycle", "aws", BucketLifecycleConfiguration);
golden!(lifecycle_minio, "lifecycle", "minio", BucketLifecycleConfiguration);
golden!(lifecycle_rustfs, "lifecycle", rustfs = rustfs_values::lifecycle());
golden!(object_lock_aws, "object_lock", "aws", ObjectLockConfiguration);
golden!(object_lock_minio, "object_lock", "minio", ObjectLockConfiguration);
golden!(object_lock_rustfs, "object_lock", rustfs = rustfs_values::object_lock());
golden!(versioning_aws, "versioning", "aws", VersioningConfiguration);
golden!(versioning_minio, "versioning", "minio", VersioningConfiguration);
golden!(versioning_rustfs, "versioning", rustfs = rustfs_values::versioning());
golden!(encryption_aws, "encryption", "aws", ServerSideEncryptionConfiguration);
golden!(encryption_minio, "encryption", "minio", ServerSideEncryptionConfiguration);
golden!(encryption_rustfs, "encryption", rustfs = rustfs_values::encryption());
golden!(tagging_aws, "tagging", "aws", Tagging);
golden!(tagging_minio, "tagging", "minio", Tagging);
golden!(tagging_rustfs, "tagging", rustfs = rustfs_values::tagging());
/// `replication/aws.in.xml` is the unknown-element sample from
/// `crates/replication/src/config.rs`
/// (`s3_xml_parser_discards_unknown_replication_elements_before_validation`).
/// Beyond the byte golden, the pair proves the root-level `FutureTopLevel`
/// element is skipped on read and never re-emitted on write.
#[test]
fn replication_aws() {
check_pair::<ReplicationConfiguration>("replication", "aws", None);
let input = read(&pair_path("replication", "aws", "in"));
let output = read(&pair_path("replication", "aws", "out"));
assert!(
contains(&input, b"<FutureTopLevel>future</FutureTopLevel>"),
"the sample must still carry the unknown element"
);
assert!(
!contains(&output, b"FutureTopLevel"),
"an unknown element must not be re-emitted: {}",
String::from_utf8_lossy(&output)
);
}
golden!(replication_minio, "replication", "minio", ReplicationConfiguration);
golden!(replication_rustfs, "replication", rustfs = rustfs_values::replication());
golden!(cors_aws, "cors", "aws", CORSConfiguration);
golden!(cors_minio, "cors", "minio", CORSConfiguration);
golden!(cors_rustfs, "cors", rustfs = rustfs_values::cors());
golden!(logging_aws, "logging", "aws", BucketLoggingStatus);
golden!(logging_minio, "logging", "minio", BucketLoggingStatus);
golden!(logging_rustfs, "logging", rustfs = rustfs_values::logging());
golden!(website_aws, "website", "aws", WebsiteConfiguration);
golden!(website_minio, "website", "minio", WebsiteConfiguration);
golden!(website_rustfs, "website", rustfs = rustfs_values::website());
golden!(accelerate_aws, "accelerate", "aws", AccelerateConfiguration);
golden!(accelerate_minio, "accelerate", "minio", AccelerateConfiguration);
golden!(accelerate_rustfs, "accelerate", rustfs = rustfs_values::accelerate());
golden!(request_payment_aws, "request_payment", "aws", RequestPaymentConfiguration);
golden!(request_payment_minio, "request_payment", "minio", RequestPaymentConfiguration);
golden!(request_payment_rustfs, "request_payment", rustfs = rustfs_values::request_payment());
golden!(public_access_block_aws, "public_access_block", "aws", PublicAccessBlockConfiguration);
golden!(public_access_block_minio, "public_access_block", "minio", PublicAccessBlockConfiguration);
golden!(
public_access_block_rustfs,
"public_access_block",
rustfs = rustfs_values::public_access_block()
);
}
crates/ecstore/tests/fixtures/bucket-config-goldens/README.md
Vendored
+58
View File
@@ -0,0 +1,58 @@
# Persisted bucket-configuration XML goldens (rustfs/backlog#2744)
Byte-level goldens for the 13 XML families that `BucketMetadata` persists as
`*_config_xml`. They pin what `rustfs_ecstore::api::bucket::utils::deserialize`
followed by `serialize` emits today, so that the DTO/codec replacement (T1.4)
and the persisted-bytes comparison (T3.7) can be proven byte-identical: an
older binary must keep reading what a newer one writes, and the reverse.
Generated at commit: `6b1554003ebf8f2037ffb7da9c9b906527e758da` (2026-10-07T17:40:56Z)
Checked by `crates/ecstore/tests/bucket_config_goldens.rs`
(`cargo nextest run -p rustfs-ecstore bucket_config_goldens`, 39 tests).
Generated once by `scripts/gen_bucket_config_goldens.sh`, which records the
commit above and refuses to run again unless `--force` is passed. Do not
regenerate after a DTO or codec replacement: at that point a diff here is the
finding, not noise.
## Layout
`<family>/<shape>.in.xml` is an input; `<family>/<shape>.out.xml` is the exact
bytes of `serialize(deserialize(in))` on the generating commit. The test
asserts `bytes == out` (never structural equality), that `out` is a fixed point
of the same round trip, and for the `rustfs` shape that `serialize(typed
value)` still equals `rustfs.in.xml`.
| Shape | Source |
| --- | --- |
| `aws` | Written for this corpus from the public S3 XML schema in the shape of the AWS documentation examples: XML declaration, `xmlns` on the root, indented. No AWS prose is copied. |
| `minio` | The bytes a MinIO server persists. Seven families are carved verbatim out of `../minio/bucket_metadata.blob.hex` (MinIO `RELEASE.2025-07-23T15-54-02Z`); the rest are hand-shaped as compact Go `encoding/xml` output with the S3 namespace on the root, because MinIO does not persist those families. |
| `rustfs` | The bytes RustFS writes today through `serialize` for a typed value (`rustfs_values` in the test). This shape carries the MinIO-only members RustFS accepts (`DelMarkerExpiration`, `ExcludeFolders`, `ExcludedPrefixes`, `DeleteReplication`). |
| Family | Root element | `minio.in.xml` provenance |
| --- | --- | --- |
| `notification` | `NotificationConfiguration` | MinIO fixture |
| `lifecycle` | `LifecycleConfiguration` | MinIO fixture (carries the MinIO `ExpiryUpdatedAt` extension) |
| `object_lock` | `ObjectLockConfiguration` | MinIO fixture |
| `versioning` | `VersioningConfiguration` | MinIO fixture |
| `encryption` | `ServerSideEncryptionConfiguration` | MinIO fixture |
| `tagging` | `Tagging` | MinIO fixture |
| `replication` | `ReplicationConfiguration` | MinIO fixture |
| `cors` | `CORSConfiguration` | hand-shaped |
| `logging` | `BucketLoggingStatus` | hand-shaped |
| `website` | `WebsiteConfiguration` | hand-shaped |
| `accelerate` | `AccelerateConfiguration` | hand-shaped |
| `request_payment` | `RequestPaymentConfiguration` | hand-shaped |
| `public_access_block` | `PublicAccessBlockConfiguration` | hand-shaped |
## Unknown elements
`replication/aws.in.xml` is the unknown-element sample from
`crates/replication/src/config.rs`
(`s3_xml_parser_discards_unknown_replication_elements_before_validation`),
re-indented to two spaces. Its golden proves the root-level `FutureTopLevel`
element is skipped on read and never re-emitted on write, which is the
persisted-configuration exception recorded in the root `AGENTS.md`. On the
generating commit every one of the 13 root elements skips unknown children;
nested members (`Rule`, `Filter`, `Destination`, ...) reject them. A
replacement codec must keep that boundary where it is.
crates/ecstore/tests/fixtures/bucket-config-goldens/accelerate/aws.in.xml
Vendored
+4
View File
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<AccelerateConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Status>Enabled</Status>
</AccelerateConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/accelerate/aws.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<AccelerateConfiguration><Status>Enabled</Status></AccelerateConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/accelerate/minio.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<AccelerateConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></AccelerateConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/accelerate/minio.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<AccelerateConfiguration><Status>Suspended</Status></AccelerateConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/accelerate/rustfs.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<AccelerateConfiguration><Status>Suspended</Status></AccelerateConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/accelerate/rustfs.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<AccelerateConfiguration><Status>Suspended</Status></AccelerateConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/cors/aws.in.xml
Vendored
+17
View File
@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<CORSRule>
<ID>AllowWebApp</ID>
<AllowedOrigin>https://www.example.com</AllowedOrigin>
<AllowedMethod>PUT</AllowedMethod>
<AllowedMethod>POST</AllowedMethod>
<AllowedMethod>DELETE</AllowedMethod>
<AllowedHeader>*</AllowedHeader>
<ExposeHeader>x-amz-server-side-encryption</ExposeHeader>
<MaxAgeSeconds>3000</MaxAgeSeconds>
</CORSRule>
<CORSRule>
<AllowedOrigin>*</AllowedOrigin>
<AllowedMethod>GET</AllowedMethod>
</CORSRule>
</CORSConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/cors/aws.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<CORSConfiguration><CORSRule><AllowedHeader>*</AllowedHeader><AllowedMethod>PUT</AllowedMethod><AllowedMethod>POST</AllowedMethod><AllowedMethod>DELETE</AllowedMethod><AllowedOrigin>https://www.example.com</AllowedOrigin><ExposeHeader>x-amz-server-side-encryption</ExposeHeader><ID>AllowWebApp</ID><MaxAgeSeconds>3000</MaxAgeSeconds></CORSRule><CORSRule><AllowedMethod>GET</AllowedMethod><AllowedOrigin>*</AllowedOrigin></CORSRule></CORSConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/cors/minio.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><CORSRule><ID>console</ID><AllowedOrigin>https://console.example.internal</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedMethod>HEAD</AllowedMethod><AllowedHeader>Authorization</AllowedHeader><AllowedHeader>Content-Type</AllowedHeader><ExposeHeader>ETag</ExposeHeader><MaxAgeSeconds>600</MaxAgeSeconds></CORSRule></CORSConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/cors/minio.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<CORSConfiguration><CORSRule><AllowedHeader>Authorization</AllowedHeader><AllowedHeader>Content-Type</AllowedHeader><AllowedMethod>GET</AllowedMethod><AllowedMethod>HEAD</AllowedMethod><AllowedOrigin>https://console.example.internal</AllowedOrigin><ExposeHeader>ETag</ExposeHeader><ID>console</ID><MaxAgeSeconds>600</MaxAgeSeconds></CORSRule></CORSConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/cors/rustfs.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<CORSConfiguration><CORSRule><AllowedHeader>Authorization</AllowedHeader><AllowedHeader>Content-Type</AllowedHeader><AllowedMethod>GET</AllowedMethod><AllowedMethod>PUT</AllowedMethod><AllowedOrigin>https://app.example.internal</AllowedOrigin><ExposeHeader>ETag</ExposeHeader><ExposeHeader>x-amz-request-id</ExposeHeader><ID>rustfs-app</ID><MaxAgeSeconds>3600</MaxAgeSeconds></CORSRule></CORSConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/cors/rustfs.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<CORSConfiguration><CORSRule><AllowedHeader>Authorization</AllowedHeader><AllowedHeader>Content-Type</AllowedHeader><AllowedMethod>GET</AllowedMethod><AllowedMethod>PUT</AllowedMethod><AllowedOrigin>https://app.example.internal</AllowedOrigin><ExposeHeader>ETag</ExposeHeader><ExposeHeader>x-amz-request-id</ExposeHeader><ID>rustfs-app</ID><MaxAgeSeconds>3600</MaxAgeSeconds></CORSRule></CORSConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/encryption/aws.in.xml
Vendored
+10
View File
@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Rule>
<ApplyServerSideEncryptionByDefault>
<SSEAlgorithm>aws:kms</SSEAlgorithm>
<KMSMasterKeyID>arn:aws:kms:us-east-1:123456789012:key/1234abcd-12ab-34cd-56ef-1234567890ab</KMSMasterKeyID>
</ApplyServerSideEncryptionByDefault>
<BucketKeyEnabled>true</BucketKeyEnabled>
</Rule>
</ServerSideEncryptionConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/encryption/aws.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><KMSMasterKeyID>arn:aws:kms:us-east-1:123456789012:key/1234abcd-12ab-34cd-56ef-1234567890ab</KMSMasterKeyID><SSEAlgorithm>aws:kms</SSEAlgorithm></ApplyServerSideEncryptionByDefault><BucketKeyEnabled>true</BucketKeyEnabled></Rule></ServerSideEncryptionConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/encryption/minio.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/encryption/minio.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/encryption/rustfs.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><KMSMasterKeyID>rustfs-bucket-key</KMSMasterKeyID><SSEAlgorithm>aws:kms</SSEAlgorithm></ApplyServerSideEncryptionByDefault><BucketKeyEnabled>false</BucketKeyEnabled></Rule></ServerSideEncryptionConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/encryption/rustfs.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><KMSMasterKeyID>rustfs-bucket-key</KMSMasterKeyID><SSEAlgorithm>aws:kms</SSEAlgorithm></ApplyServerSideEncryptionByDefault><BucketKeyEnabled>false</BucketKeyEnabled></Rule></ServerSideEncryptionConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/lifecycle/aws.in.xml
Vendored
+49
View File
@@ -0,0 +1,49 @@
<?xml version="1.0" encoding="UTF-8"?>
<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Rule>
<ID>Archive and then delete rule</ID>
<Filter>
<And>
<Prefix>projectdocs/</Prefix>
<Tag>
<Key>department</Key>
<Value>finance</Value>
</Tag>
<ObjectSizeGreaterThan>1024</ObjectSizeGreaterThan>
</And>
</Filter>
<Status>Enabled</Status>
<Transition>
<Days>30</Days>
<StorageClass>STANDARD_IA</StorageClass>
</Transition>
<Transition>
<Days>365</Days>
<StorageClass>GLACIER</StorageClass>
</Transition>
<Expiration>
<Days>3650</Days>
</Expiration>
<NoncurrentVersionTransition>
<NoncurrentDays>90</NoncurrentDays>
<StorageClass>GLACIER</StorageClass>
</NoncurrentVersionTransition>
<NoncurrentVersionExpiration>
<NoncurrentDays>400</NoncurrentDays>
<NewerNoncurrentVersions>3</NewerNoncurrentVersions>
</NoncurrentVersionExpiration>
<AbortIncompleteMultipartUpload>
<DaysAfterInitiation>7</DaysAfterInitiation>
</AbortIncompleteMultipartUpload>
</Rule>
<Rule>
<ID>Expire delete markers</ID>
<Filter>
<Prefix>logs/</Prefix>
</Filter>
<Status>Disabled</Status>
<Expiration>
<ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker>
</Expiration>
</Rule>
</LifecycleConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/lifecycle/aws.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<LifecycleConfiguration><Rule><AbortIncompleteMultipartUpload><DaysAfterInitiation>7</DaysAfterInitiation></AbortIncompleteMultipartUpload><Expiration><Days>3650</Days></Expiration><Filter><And><ObjectSizeGreaterThan>1024</ObjectSizeGreaterThan><Prefix>projectdocs/</Prefix><Tag><Key>department</Key><Value>finance</Value></Tag></And></Filter><ID>Archive and then delete rule</ID><NoncurrentVersionExpiration><NewerNoncurrentVersions>3</NewerNoncurrentVersions><NoncurrentDays>400</NoncurrentDays></NoncurrentVersionExpiration><NoncurrentVersionTransition><NoncurrentDays>90</NoncurrentDays><StorageClass>GLACIER</StorageClass></NoncurrentVersionTransition><Status>Enabled</Status><Transition><Days>30</Days><StorageClass>STANDARD_IA</StorageClass></Transition><Transition><Days>365</Days><StorageClass>GLACIER</StorageClass></Transition></Rule><Rule><Expiration><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration><Filter><Prefix>logs/</Prefix></Filter><ID>Expire delete markers</ID><Status>Disabled</Status></Rule></LifecycleConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/lifecycle/minio.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<LifecycleConfiguration><Rule><ID>d96i4g89k8h26a95st60</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><Expiration><Days>30</Days></Expiration><NoncurrentVersionExpiration><NoncurrentDays>7</NoncurrentDays></NoncurrentVersionExpiration></Rule><ExpiryUpdatedAt>2026-07-07T15:58:57.337315Z</ExpiryUpdatedAt></LifecycleConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/lifecycle/minio.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<LifecycleConfiguration><ExpiryUpdatedAt>2026-07-07T15:58:57.337Z</ExpiryUpdatedAt><Rule><Expiration><Days>30</Days></Expiration><Filter><Prefix></Prefix></Filter><ID>d96i4g89k8h26a95st60</ID><NoncurrentVersionExpiration><NoncurrentDays>7</NoncurrentDays></NoncurrentVersionExpiration><Status>Enabled</Status></Rule></LifecycleConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/lifecycle/rustfs.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<LifecycleConfiguration><Rule><AbortIncompleteMultipartUpload><DaysAfterInitiation>3</DaysAfterInitiation></AbortIncompleteMultipartUpload><DelMarkerExpiration><Days>7</Days></DelMarkerExpiration><Expiration><Days>30</Days></Expiration><Filter><Prefix>tmp/</Prefix></Filter><ID>expire-tmp</ID><NoncurrentVersionExpiration><NewerNoncurrentVersions>2</NewerNoncurrentVersions><NoncurrentDays>7</NoncurrentDays></NoncurrentVersionExpiration><Status>Enabled</Status></Rule><Rule><Filter><And><Prefix>archive/</Prefix><Tag><Key>tier</Key><Value>cold</Value></Tag></And></Filter><ID>tier-cold</ID><Status>Enabled</Status><Transition><Days>90</Days><StorageClass>COLDTIER</StorageClass></Transition></Rule></LifecycleConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/lifecycle/rustfs.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<LifecycleConfiguration><Rule><AbortIncompleteMultipartUpload><DaysAfterInitiation>3</DaysAfterInitiation></AbortIncompleteMultipartUpload><DelMarkerExpiration><Days>7</Days></DelMarkerExpiration><Expiration><Days>30</Days></Expiration><Filter><Prefix>tmp/</Prefix></Filter><ID>expire-tmp</ID><NoncurrentVersionExpiration><NewerNoncurrentVersions>2</NewerNoncurrentVersions><NoncurrentDays>7</NoncurrentDays></NoncurrentVersionExpiration><Status>Enabled</Status></Rule><Rule><Filter><And><Prefix>archive/</Prefix><Tag><Key>tier</Key><Value>cold</Value></Tag></And></Filter><ID>tier-cold</ID><Status>Enabled</Status><Transition><Days>90</Days><StorageClass>COLDTIER</StorageClass></Transition></Rule></LifecycleConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/logging/aws.in.xml
Vendored
+20
View File
@@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<BucketLoggingStatus xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<LoggingEnabled>
<TargetBucket>amzn-s3-demo-logging-bucket</TargetBucket>
<TargetPrefix>logs/</TargetPrefix>
<TargetGrants>
<Grant>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="AmazonCustomerByEmail">
<EmailAddress>user@example.com</EmailAddress>
</Grantee>
<Permission>READ</Permission>
</Grant>
</TargetGrants>
<TargetObjectKeyFormat>
<PartitionedPrefix>
<PartitionDateSource>EventTime</PartitionDateSource>
</PartitionedPrefix>
</TargetObjectKeyFormat>
</LoggingEnabled>
</BucketLoggingStatus>
crates/ecstore/tests/fixtures/bucket-config-goldens/logging/aws.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<BucketLoggingStatus><LoggingEnabled><TargetBucket>amzn-s3-demo-logging-bucket</TargetBucket><TargetGrants><Grant><Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="AmazonCustomerByEmail"><EmailAddress>user@example.com</EmailAddress></Grantee><Permission>READ</Permission></Grant></TargetGrants><TargetObjectKeyFormat><PartitionedPrefix><PartitionDateSource>EventTime</PartitionDateSource></PartitionedPrefix></TargetObjectKeyFormat><TargetPrefix>logs/</TargetPrefix></LoggingEnabled></BucketLoggingStatus>
crates/ecstore/tests/fixtures/bucket-config-goldens/logging/minio.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<BucketLoggingStatus xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><LoggingEnabled><TargetBucket>interop-logs</TargetBucket><TargetPrefix>interop/</TargetPrefix></LoggingEnabled></BucketLoggingStatus>
crates/ecstore/tests/fixtures/bucket-config-goldens/logging/minio.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<BucketLoggingStatus><LoggingEnabled><TargetBucket>interop-logs</TargetBucket><TargetPrefix>interop/</TargetPrefix></LoggingEnabled></BucketLoggingStatus>
crates/ecstore/tests/fixtures/bucket-config-goldens/logging/rustfs.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<BucketLoggingStatus><LoggingEnabled><TargetBucket>rustfs-access-logs</TargetBucket><TargetPrefix>buckets/interop/</TargetPrefix></LoggingEnabled></BucketLoggingStatus>
crates/ecstore/tests/fixtures/bucket-config-goldens/logging/rustfs.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<BucketLoggingStatus><LoggingEnabled><TargetBucket>rustfs-access-logs</TargetBucket><TargetPrefix>buckets/interop/</TargetPrefix></LoggingEnabled></BucketLoggingStatus>
crates/ecstore/tests/fixtures/bucket-config-goldens/notification/aws.in.xml
Vendored
+30
View File
@@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<NotificationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TopicConfiguration>
<Id>ImageUploads</Id>
<Topic>arn:aws:sns:us-east-1:123456789012:s3-events</Topic>
<Event>s3:ObjectCreated:*</Event>
<Filter>
<S3Key>
<FilterRule>
<Name>prefix</Name>
<Value>images/</Value>
</FilterRule>
<FilterRule>
<Name>suffix</Name>
<Value>.jpg</Value>
</FilterRule>
</S3Key>
</Filter>
</TopicConfiguration>
<QueueConfiguration>
<Id>Deletions</Id>
<Queue>arn:aws:sqs:us-east-1:123456789012:s3-deletes</Queue>
<Event>s3:ObjectRemoved:*</Event>
</QueueConfiguration>
<CloudFunctionConfiguration>
<Id>Thumbnails</Id>
<CloudFunction>arn:aws:lambda:us-east-1:123456789012:function:thumbnail</CloudFunction>
<Event>s3:ObjectCreated:Put</Event>
</CloudFunctionConfiguration>
</NotificationConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/notification/aws.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<NotificationConfiguration><CloudFunctionConfiguration><Event>s3:ObjectCreated:Put</Event><Id>Thumbnails</Id><CloudFunction>arn:aws:lambda:us-east-1:123456789012:function:thumbnail</CloudFunction></CloudFunctionConfiguration><QueueConfiguration><Event>s3:ObjectRemoved:*</Event><Id>Deletions</Id><Queue>arn:aws:sqs:us-east-1:123456789012:s3-deletes</Queue></QueueConfiguration><TopicConfiguration><Event>s3:ObjectCreated:*</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule><FilterRule><Name>suffix</Name><Value>.jpg</Value></FilterRule></S3Key></Filter><Id>ImageUploads</Id><Topic>arn:aws:sns:us-east-1:123456789012:s3-events</Topic></TopicConfiguration></NotificationConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/notification/minio.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<NotificationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><QueueConfiguration><Id></Id><Event>s3:ObjectCreated:*</Event><Queue>arn:minio:sqs::primary:webhook</Queue></QueueConfiguration></NotificationConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/notification/minio.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:*</Event><Id></Id><Queue>arn:minio:sqs::primary:webhook</Queue></QueueConfiguration></NotificationConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/notification/rustfs.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:*</Event><Event>s3:ObjectRemoved:*</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>uploads/</Value></FilterRule></S3Key></Filter><Id>rustfs-webhook</Id><Queue>arn:rustfs:sqs::primary:webhook</Queue></QueueConfiguration></NotificationConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/notification/rustfs.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:*</Event><Event>s3:ObjectRemoved:*</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>uploads/</Value></FilterRule></S3Key></Filter><Id>rustfs-webhook</Id><Queue>arn:rustfs:sqs::primary:webhook</Queue></QueueConfiguration></NotificationConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/object_lock/aws.in.xml
Vendored
+10
View File
@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<ObjectLockEnabled>Enabled</ObjectLockEnabled>
<Rule>
<DefaultRetention>
<Mode>COMPLIANCE</Mode>
<Years>1</Years>
</DefaultRetention>
</Rule>
</ObjectLockConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/object_lock/aws.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>COMPLIANCE</Mode><Years>1</Years></DefaultRetention></Rule></ObjectLockConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/object_lock/minio.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>7</Days></DefaultRetention></Rule></ObjectLockConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/object_lock/minio.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Days>7</Days><Mode>GOVERNANCE</Mode></DefaultRetention></Rule></ObjectLockConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/object_lock/rustfs.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Days>30</Days><Mode>GOVERNANCE</Mode></DefaultRetention></Rule></ObjectLockConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/object_lock/rustfs.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Days>30</Days><Mode>GOVERNANCE</Mode></DefaultRetention></Rule></ObjectLockConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/public_access_block/aws.in.xml
Vendored
+7
View File
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<PublicAccessBlockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<BlockPublicAcls>TRUE</BlockPublicAcls>
<IgnorePublicAcls>TRUE</IgnorePublicAcls>
<BlockPublicPolicy>TRUE</BlockPublicPolicy>
<RestrictPublicBuckets>TRUE</RestrictPublicBuckets>
</PublicAccessBlockConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/public_access_block/aws.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><IgnorePublicAcls>true</IgnorePublicAcls><RestrictPublicBuckets>true</RestrictPublicBuckets></PublicAccessBlockConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/public_access_block/minio.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<PublicAccessBlockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><BlockPublicAcls>true</BlockPublicAcls><IgnorePublicAcls>false</IgnorePublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><RestrictPublicBuckets>false</RestrictPublicBuckets></PublicAccessBlockConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/public_access_block/minio.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><IgnorePublicAcls>false</IgnorePublicAcls><RestrictPublicBuckets>false</RestrictPublicBuckets></PublicAccessBlockConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/public_access_block/rustfs.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><BlockPublicPolicy>false</BlockPublicPolicy><IgnorePublicAcls>true</IgnorePublicAcls><RestrictPublicBuckets>false</RestrictPublicBuckets></PublicAccessBlockConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/public_access_block/rustfs.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><BlockPublicPolicy>false</BlockPublicPolicy><IgnorePublicAcls>true</IgnorePublicAcls><RestrictPublicBuckets>false</RestrictPublicBuckets></PublicAccessBlockConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/replication/aws.in.xml
Vendored
+12
View File
@@ -0,0 +1,12 @@
<ReplicationConfiguration>
<Role></Role>
<FutureTopLevel>future</FutureTopLevel>
<Rule>
<ID>unknown</ID>
<Status>Enabled</Status>
<Destination>
<Bucket>arn:aws:s3:::destination</Bucket>
</Destination>
</Rule>
</ReplicationConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/replication/aws.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ReplicationConfiguration><Role></Role><Rule><Destination><Bucket>arn:aws:s3:::destination</Bucket></Destination><ID>unknown</ID><Status>Enabled</Status></Rule></ReplicationConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/replication/minio.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ReplicationConfiguration><Rule><ID>d96i4m09k8h2vldifkag</ID><Status>Enabled</Status><Priority>1</Priority><DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Enabled</Status></DeleteReplication><Destination><Bucket>arn:minio:replication::ef5859af-120a-4218-94b5-be23470f3c60:interop-dr</Bucket></Destination><SourceSelectionCriteria><ReplicaModifications><Status>Enabled</Status></ReplicaModifications></SourceSelectionCriteria><Filter><Prefix></Prefix></Filter><ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication></Rule><Role></Role></ReplicationConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/replication/minio.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ReplicationConfiguration><Role></Role><Rule><DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Enabled</Status></DeleteReplication><Destination><Bucket>arn:minio:replication::ef5859af-120a-4218-94b5-be23470f3c60:interop-dr</Bucket></Destination><ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication><Filter><Prefix></Prefix></Filter><ID>d96i4m09k8h2vldifkag</ID><Priority>1</Priority><SourceSelectionCriteria><ReplicaModifications><Status>Enabled</Status></ReplicaModifications></SourceSelectionCriteria><Status>Enabled</Status></Rule></ReplicationConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/replication/rustfs.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ReplicationConfiguration><Role></Role><Rule><DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Enabled</Status></DeleteReplication><Destination><Bucket>arn:rustfs:replication::0b6f3c2a-5e6a-4d2b-9c1e-7a8f9d0e1b2c:interop-dr</Bucket><StorageClass>STANDARD</StorageClass></Destination><ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication><Filter><And><Prefix>docs/</Prefix><Tag><Key>replicate</Key><Value>yes</Value></Tag></And></Filter><ID>rustfs-dr</ID><Priority>1</Priority><SourceSelectionCriteria><ReplicaModifications><Status>Enabled</Status></ReplicaModifications></SourceSelectionCriteria><Status>Enabled</Status></Rule></ReplicationConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/replication/rustfs.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<ReplicationConfiguration><Role></Role><Rule><DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication><DeleteReplication><Status>Enabled</Status></DeleteReplication><Destination><Bucket>arn:rustfs:replication::0b6f3c2a-5e6a-4d2b-9c1e-7a8f9d0e1b2c:interop-dr</Bucket><StorageClass>STANDARD</StorageClass></Destination><ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication><Filter><And><Prefix>docs/</Prefix><Tag><Key>replicate</Key><Value>yes</Value></Tag></And></Filter><ID>rustfs-dr</ID><Priority>1</Priority><SourceSelectionCriteria><ReplicaModifications><Status>Enabled</Status></ReplicaModifications></SourceSelectionCriteria><Status>Enabled</Status></Rule></ReplicationConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/request_payment/aws.in.xml
Vendored
+4
View File
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<RequestPaymentConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Payer>Requester</Payer>
</RequestPaymentConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/request_payment/aws.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<RequestPaymentConfiguration><Payer>Requester</Payer></RequestPaymentConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/request_payment/minio.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<RequestPaymentConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Payer>BucketOwner</Payer></RequestPaymentConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/request_payment/minio.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<RequestPaymentConfiguration><Payer>BucketOwner</Payer></RequestPaymentConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/request_payment/rustfs.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<RequestPaymentConfiguration><Payer>BucketOwner</Payer></RequestPaymentConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/request_payment/rustfs.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<RequestPaymentConfiguration><Payer>BucketOwner</Payer></RequestPaymentConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/tagging/aws.in.xml
Vendored
+13
View File
@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<Tagging xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<TagSet>
<Tag>
<Key>Project</Key>
<Value>Project One</Value>
</Tag>
<Tag>
<Key>User</Key>
<Value>jsmith</Value>
</Tag>
</TagSet>
</Tagging>
crates/ecstore/tests/fixtures/bucket-config-goldens/tagging/aws.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<Tagging><TagSet><Tag><Key>Project</Key><Value>Project One</Value></Tag><Tag><Key>User</Key><Value>jsmith</Value></Tag></TagSet></Tagging>
crates/ecstore/tests/fixtures/bucket-config-goldens/tagging/minio.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<Tagging><TagSet><Tag><Key>team</Key><Value>storage</Value></Tag><Tag><Key>env</Key><Value>prod</Value></Tag></TagSet></Tagging>
crates/ecstore/tests/fixtures/bucket-config-goldens/tagging/minio.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<Tagging><TagSet><Tag><Key>team</Key><Value>storage</Value></Tag><Tag><Key>env</Key><Value>prod</Value></Tag></TagSet></Tagging>
crates/ecstore/tests/fixtures/bucket-config-goldens/tagging/rustfs.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<Tagging><TagSet><Tag><Key>team</Key><Value>storage</Value></Tag><Tag><Key>env</Key><Value>staging</Value></Tag></TagSet></Tagging>
crates/ecstore/tests/fixtures/bucket-config-goldens/tagging/rustfs.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<Tagging><TagSet><Tag><Key>team</Key><Value>storage</Value></Tag><Tag><Key>env</Key><Value>staging</Value></Tag></TagSet></Tagging>
crates/ecstore/tests/fixtures/bucket-config-goldens/versioning/aws.in.xml
Vendored
+5
View File
@@ -0,0 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Status>Enabled</Status>
<MfaDelete>Disabled</MfaDelete>
</VersioningConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/versioning/aws.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<VersioningConfiguration><MfaDelete>Disabled</MfaDelete><Status>Enabled</Status></VersioningConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/versioning/minio.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/versioning/minio.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/versioning/rustfs.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<VersioningConfiguration><ExcludeFolders>true</ExcludeFolders><ExcludedPrefixes><Prefix>tmp/</Prefix></ExcludedPrefixes><ExcludedPrefixes><Prefix>cache/</Prefix></ExcludedPrefixes><Status>Enabled</Status></VersioningConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/versioning/rustfs.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<VersioningConfiguration><ExcludeFolders>true</ExcludeFolders><ExcludedPrefixes><Prefix>tmp/</Prefix></ExcludedPrefixes><ExcludedPrefixes><Prefix>cache/</Prefix></ExcludedPrefixes><Status>Enabled</Status></VersioningConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/website/aws.in.xml
Vendored
+23
View File
@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<WebsiteConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<IndexDocument>
<Suffix>index.html</Suffix>
</IndexDocument>
<ErrorDocument>
<Key>error.html</Key>
</ErrorDocument>
<RoutingRules>
<RoutingRule>
<Condition>
<KeyPrefixEquals>docs/</KeyPrefixEquals>
<HttpErrorCodeReturnedEquals>404</HttpErrorCodeReturnedEquals>
</Condition>
<Redirect>
<Protocol>https</Protocol>
<HostName>www.example.com</HostName>
<ReplaceKeyPrefixWith>documents/</ReplaceKeyPrefixWith>
<HttpRedirectCode>301</HttpRedirectCode>
</Redirect>
</RoutingRule>
</RoutingRules>
</WebsiteConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/website/aws.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<WebsiteConfiguration><ErrorDocument><Key>error.html</Key></ErrorDocument><IndexDocument><Suffix>index.html</Suffix></IndexDocument><RoutingRules><RoutingRule><Condition><HttpErrorCodeReturnedEquals>404</HttpErrorCodeReturnedEquals><KeyPrefixEquals>docs/</KeyPrefixEquals></Condition><Redirect><HostName>www.example.com</HostName><HttpRedirectCode>301</HttpRedirectCode><Protocol>https</Protocol><ReplaceKeyPrefixWith>documents/</ReplaceKeyPrefixWith></Redirect></RoutingRule></RoutingRules></WebsiteConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/website/minio.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<WebsiteConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><IndexDocument><Suffix>index.html</Suffix></IndexDocument><ErrorDocument><Key>404.html</Key></ErrorDocument></WebsiteConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/website/minio.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<WebsiteConfiguration><ErrorDocument><Key>404.html</Key></ErrorDocument><IndexDocument><Suffix>index.html</Suffix></IndexDocument></WebsiteConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/website/rustfs.in.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<WebsiteConfiguration><ErrorDocument><Key>404.html</Key></ErrorDocument><IndexDocument><Suffix>index.html</Suffix></IndexDocument><RoutingRules><RoutingRule><Condition><KeyPrefixEquals>legacy/</KeyPrefixEquals></Condition><Redirect><HttpRedirectCode>301</HttpRedirectCode><ReplaceKeyPrefixWith>current/</ReplaceKeyPrefixWith></Redirect></RoutingRule></RoutingRules></WebsiteConfiguration>
crates/ecstore/tests/fixtures/bucket-config-goldens/website/rustfs.out.xml
Vendored
+1
View File
@@ -0,0 +1 @@
<WebsiteConfiguration><ErrorDocument><Key>404.html</Key></ErrorDocument><IndexDocument><Suffix>index.html</Suffix></IndexDocument><RoutingRules><RoutingRule><Condition><KeyPrefixEquals>legacy/</KeyPrefixEquals></Condition><Redirect><HttpRedirectCode>301</HttpRedirectCode><ReplaceKeyPrefixWith>current/</ReplaceKeyPrefixWith></Redirect></RoutingRule></RoutingRules></WebsiteConfiguration>
crates/obs/src/metrics/scheduler.rs
+35 -201
View File
@@ -110,9 +110,7 @@ use crate::metrics::schema::bucket_replication::{
use crate::metrics::schema::cluster_usage::{
BUCKET_LABEL as USAGE_BUCKET_LABEL, RANGE_LABEL as USAGE_RANGE_LABEL, USAGE_BUCKET_DELETE_MARKERS_COUNT_MD,
USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD, USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD, USAGE_BUCKET_OBJECTS_TOTAL_MD,
USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, USAGE_BUCKET_TOTAL_BYTES_MD, USAGE_BUCKET_VERSIONS_COUNT_MD, USAGE_BUCKETS_COUNT_MD,
USAGE_DELETE_MARKERS_COUNT_MD, USAGE_OBJECTS_COUNT_MD, USAGE_OBJECTS_DISTRIBUTION_MD, USAGE_SINCE_LAST_UPDATE_SECONDS_MD,
USAGE_TOTAL_BYTES_MD, USAGE_VERSIONS_COUNT_MD, USAGE_VERSIONS_DISTRIBUTION_MD,
USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, USAGE_BUCKET_TOTAL_BYTES_MD, USAGE_BUCKET_VERSIONS_COUNT_MD,
};
use crate::metrics::schema::node_bucket::{BUCKET_OBJECTS_TOTAL_MD, BUCKET_QUOTA_BYTES_MD, BUCKET_USAGE_BYTES_MD};
use crate::metrics::schema::notification_target::{
@@ -150,7 +148,7 @@ use crate::metrics::stats_collector::{
};
use crate::metrics::storage_snapshot::StorageSnapshotMetrics;
use crate::node_identity::{SERVER_LABEL, current_local_node_identity};
use crate::telemetry::retire_metric_series;
use crate::telemetry::{process_global_meter, retire_metric_series};
use futures_util::FutureExt;
use rustfs_audit::audit_target_metrics;
use rustfs_config::METER_INTERVAL;
@@ -731,6 +729,17 @@ fn storage_snapshot_metrics(scope: &'static str, collection_interval: Duration)
)
}
fn bucket_metrics_snapshot(collection_interval: Duration) -> StorageSnapshotMetrics {
let export_interval = Duration::from_secs(
get_env_opt_u64(ENV_OBS_METER_INTERVAL)
.filter(|value| *value > 0)
.unwrap_or(METER_INTERVAL),
);
let max_age = collection_interval.max(export_interval).saturating_mul(3);
let meter = process_global_meter().unwrap_or_else(|| opentelemetry::global::meter("rustfs.storage"));
StorageSnapshotMetrics::new_unscoped(meter, max_age)
}
fn metrics_interval(period: Duration, initial_delay: Duration) -> Interval {
let mut interval = tokio::time::interval_at(Instant::now() + initial_delay, period);
interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
@@ -850,26 +859,6 @@ fn bucket_live_keys(stats: &[crate::metrics::collectors::BucketStats]) -> HashSe
stats.iter().map(|stat| stat.name.clone()).collect()
}
fn bucket_observation_live_keys(stats: &[crate::metrics::collectors::BucketStats]) -> HashSet<BucketKey> {
stats
.iter()
.filter(|stat| stat.size_bytes.is_some() || stat.objects_count.is_some())
.map(|stat| stat.name.clone())
.collect()
}
fn bucket_observation_retire_keys(
previous_observations: &HashSet<BucketKey>,
current_buckets: &HashSet<BucketKey>,
current_observations: &HashSet<BucketKey>,
) -> Vec<BucketKey> {
previous_observations
.difference(current_observations)
.filter(|bucket| current_buckets.contains(*bucket))
.cloned()
.collect()
}
fn collect_bucket_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketKey, u8>) -> Vec<PrometheusMetric> {
if zero_tombstones.is_empty() {
return Vec::new();
@@ -892,14 +881,11 @@ fn collect_bucket_zero_tombstone_metrics(zero_tombstones: &HashMap<BucketKey, u8
struct BucketSeriesState {
has_seen_snapshot: bool,
live_keys: HashSet<BucketKey>,
observation_keys: HashSet<BucketKey>,
zero_tombstones: HashMap<BucketKey, u8>,
}
struct BucketSeriesUpdate {
metrics: Vec<PrometheusMetric>,
retire_observations: Vec<BucketKey>,
retire_buckets: Vec<BucketKey>,
}
impl BucketSeriesState {
@@ -910,10 +896,6 @@ impl BucketSeriesState {
) -> Option<BucketSeriesUpdate> {
let stats = stats?;
let current_bucket_keys = bucket_live_keys(stats);
let current_observation_keys = bucket_observation_live_keys(stats);
let retire_observations =
bucket_observation_retire_keys(&self.observation_keys, &current_bucket_keys, &current_observation_keys);
self.observation_keys = current_observation_keys;
update_series_zero_tombstones(
&mut self.has_seen_snapshot,
&mut self.live_keys,
@@ -923,49 +905,11 @@ impl BucketSeriesState {
);
let mut metrics = collect_bucket_metrics(stats);
metrics.extend(collect_bucket_zero_tombstone_metrics(&self.zero_tombstones));
let retire_buckets = expire_series_zero_tombstones(&mut self.zero_tombstones);
Some(BucketSeriesUpdate {
metrics,
retire_observations,
retire_buckets,
})
let _ = expire_series_zero_tombstones(&mut self.zero_tombstones);
Some(BucketSeriesUpdate { metrics })
}
}
fn retire_bucket_metric_series(bucket: &str) -> usize {
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.to_string());
let labels = [("bucket", bucket_label.clone())];
retire_metric_series(&BUCKET_USAGE_BYTES_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&BUCKET_OBJECTS_TOTAL_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&BUCKET_QUOTA_BYTES_MD.get_full_metric_name(), &labels)
}
fn retire_bucket_observation_metric_series(bucket: &str) -> usize {
let labels = [("bucket", Cow::Owned(bucket.to_string()))];
retire_metric_series(&BUCKET_USAGE_BYTES_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&BUCKET_OBJECTS_TOTAL_MD.get_full_metric_name(), &labels)
}
fn retire_cluster_usage_metric_series() -> usize {
let labels: [(&'static str, Cow<'static, str>); 0] = [];
[
USAGE_SINCE_LAST_UPDATE_SECONDS_MD.get_full_metric_name(),
USAGE_TOTAL_BYTES_MD.get_full_metric_name(),
USAGE_OBJECTS_COUNT_MD.get_full_metric_name(),
USAGE_VERSIONS_COUNT_MD.get_full_metric_name(),
USAGE_DELETE_MARKERS_COUNT_MD.get_full_metric_name(),
USAGE_BUCKETS_COUNT_MD.get_full_metric_name(),
]
.iter()
.map(|name| retire_metric_series(name, &labels))
.sum()
}
fn retire_cluster_usage_distribution_series(metric_name: String, range: &str) -> usize {
let labels = [(USAGE_RANGE_LABEL, Cow::Owned(range.to_string()))];
retire_metric_series(&metric_name, &labels)
}
fn bucket_usage_live_keys(stats: &[crate::metrics::collectors::BucketUsageStats]) -> HashSet<BucketKey> {
stats.iter().map(|stat| stat.bucket.clone()).collect()
}
@@ -1043,24 +987,6 @@ fn collect_bucket_usage_zero_tombstone_metrics(
zero_metrics
}
fn retire_bucket_usage_metric_series(bucket: &str) -> usize {
let bucket_label: Cow<'static, str> = Cow::Owned(bucket.to_string());
let labels = [(USAGE_BUCKET_LABEL, bucket_label.clone())];
retire_metric_series(&USAGE_BUCKET_TOTAL_BYTES_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&USAGE_BUCKET_OBJECTS_TOTAL_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&USAGE_BUCKET_VERSIONS_COUNT_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&USAGE_BUCKET_DELETE_MARKERS_COUNT_MD.get_full_metric_name(), &labels)
+ retire_metric_series(&USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD.get_full_metric_name(), &labels)
}
fn retire_bucket_usage_distribution_series(metric_name: String, bucket: &str, range: &str) -> usize {
let labels = [
(USAGE_RANGE_LABEL, Cow::Owned(range.to_string())),
(USAGE_BUCKET_LABEL, Cow::Owned(bucket.to_string())),
];
retire_metric_series(&metric_name, &labels)
}
fn audit_target_live_keys(stats: &[AuditTargetRuntimeStats]) -> HashSet<AuditTargetKey> {
stats
.iter()
@@ -1687,14 +1613,13 @@ pub fn init_metrics_runtime(token: CancellationToken) {
let mut interval = metrics_interval(cluster_interval, stagger_duration(cluster_interval, 1, 3));
let tombstone_cycles = config.replication_bandwidth_zero_tombstone_cycles;
let mut has_seen_bucket_usage_snapshot = false;
let mut bucket_usage_snapshot = bucket_metrics_snapshot(cluster_interval);
let mut prev_bucket_usage_keys: HashSet<BucketKey> = HashSet::new();
let mut bucket_usage_zero_tombstones: HashMap<BucketKey, u8> = HashMap::new();
let mut prev_bucket_usage_object_size_keys: HashSet<BucketRangeKey> = HashSet::new();
let mut bucket_usage_object_size_zero_tombstones: HashMap<BucketRangeKey, u8> = HashMap::new();
let mut prev_bucket_usage_version_keys: HashSet<BucketRangeKey> = HashSet::new();
let mut bucket_usage_version_zero_tombstones: HashMap<BucketRangeKey, u8> = HashMap::new();
let mut prev_cluster_usage_object_size_keys: HashSet<String> = HashSet::new();
let mut prev_cluster_usage_version_keys: HashSet<String> = HashSet::new();
loop {
tokio::select! {
_ = interval.tick() => {
@@ -1704,6 +1629,7 @@ pub fn init_metrics_runtime(token: CancellationToken) {
"supplementary_cluster_stats",
async {
let mut metrics = Vec::new();
let mut usage_metrics_update = None;
if let Some(stats) = collect_cluster_config_stats().await {
metrics.extend(collect_cluster_config_metrics(&stats));
@@ -1713,32 +1639,10 @@ pub fn init_metrics_runtime(token: CancellationToken) {
metrics.extend(collect_iam_metrics(&stats));
}
let usage_collection_started = std::time::Instant::now();
if let Some((cluster_usage, bucket_usage)) = collect_cluster_usage_metric_stats().await {
let current_cluster_usage_object_size_keys = cluster_usage
.object_size_distribution
.iter()
.map(|(range, _)| range.clone())
.collect::<HashSet<_>>();
for range in prev_cluster_usage_object_size_keys.difference(&current_cluster_usage_object_size_keys) {
let _ = retire_cluster_usage_distribution_series(
USAGE_OBJECTS_DISTRIBUTION_MD.get_full_metric_name(),
range,
);
}
prev_cluster_usage_object_size_keys = current_cluster_usage_object_size_keys;
let current_cluster_usage_version_keys = cluster_usage
.versions_distribution
.iter()
.map(|(range, _)| range.clone())
.collect::<HashSet<_>>();
for range in prev_cluster_usage_version_keys.difference(&current_cluster_usage_version_keys) {
let _ = retire_cluster_usage_distribution_series(
USAGE_VERSIONS_DISTRIBUTION_MD.get_full_metric_name(),
range,
);
}
prev_cluster_usage_version_keys = current_cluster_usage_version_keys;
metrics.extend(collect_cluster_usage_metrics(&cluster_usage));
let mut usage_metrics = collect_cluster_usage_metrics(&cluster_usage);
usage_metrics.extend(collect_bucket_usage_metrics(&bucket_usage));
update_series_zero_tombstones(
&mut has_seen_bucket_usage_snapshot,
&mut prev_bucket_usage_keys,
@@ -1760,64 +1664,21 @@ pub fn init_metrics_runtime(token: CancellationToken) {
bucket_usage_version_live_keys(&bucket_usage),
tombstone_cycles,
);
metrics.extend(collect_bucket_usage_metrics(&bucket_usage));
metrics.extend(collect_bucket_usage_zero_tombstone_metrics(
usage_metrics.extend(collect_bucket_usage_zero_tombstone_metrics(
&bucket_usage_zero_tombstones,
&bucket_usage_object_size_zero_tombstones,
&bucket_usage_version_zero_tombstones,
));
for bucket in expire_series_zero_tombstones(&mut bucket_usage_zero_tombstones) {
let _ = retire_bucket_usage_metric_series(&bucket);
}
for (bucket, range) in expire_series_zero_tombstones(&mut bucket_usage_object_size_zero_tombstones) {
let _ = retire_bucket_usage_distribution_series(
USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD.get_full_metric_name(),
&bucket,
&range,
);
}
for (bucket, range) in expire_series_zero_tombstones(&mut bucket_usage_version_zero_tombstones) {
let _ = retire_bucket_usage_distribution_series(
USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD.get_full_metric_name(),
&bucket,
&range,
);
}
} else if has_seen_bucket_usage_snapshot {
let _ = retire_cluster_usage_metric_series();
for range in prev_cluster_usage_object_size_keys.drain() {
let _ = retire_cluster_usage_distribution_series(
USAGE_OBJECTS_DISTRIBUTION_MD.get_full_metric_name(),
&range,
);
}
for range in prev_cluster_usage_version_keys.drain() {
let _ = retire_cluster_usage_distribution_series(
USAGE_VERSIONS_DISTRIBUTION_MD.get_full_metric_name(),
&range,
);
}
for bucket in prev_bucket_usage_keys.drain() {
let _ = retire_bucket_usage_metric_series(&bucket);
}
for (bucket, range) in prev_bucket_usage_object_size_keys.drain() {
let _ = retire_bucket_usage_distribution_series(
USAGE_BUCKET_OBJECT_SIZE_DISTRIBUTION_MD.get_full_metric_name(),
&bucket,
&range,
);
}
for (bucket, range) in prev_bucket_usage_version_keys.drain() {
let _ = retire_bucket_usage_distribution_series(
USAGE_BUCKET_OBJECT_VERSION_COUNT_DISTRIBUTION_MD.get_full_metric_name(),
&bucket,
&range,
);
}
bucket_usage_zero_tombstones.clear();
bucket_usage_object_size_zero_tombstones.clear();
bucket_usage_version_zero_tombstones.clear();
has_seen_bucket_usage_snapshot = false;
let _ = expire_series_zero_tombstones(&mut bucket_usage_zero_tombstones);
let _ = expire_series_zero_tombstones(&mut bucket_usage_object_size_zero_tombstones);
let _ = expire_series_zero_tombstones(&mut bucket_usage_version_zero_tombstones);
usage_metrics_update = Some(usage_metrics);
}
if let Some(usage_metrics) = usage_metrics_update
&& let Err(error) = bucket_usage_snapshot.replace_collected(usage_metrics, usage_collection_started)
{
error!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "supplementary_cluster_stats", result = "invalid_bucket_snapshot", error = %error, "bucket usage metrics snapshot rejected");
}
if !metrics.is_empty() {
@@ -1840,20 +1701,18 @@ pub fn init_metrics_runtime(token: CancellationToken) {
let mut interval = metrics_interval(bucket_interval, Duration::ZERO);
let tombstone_cycles = config.replication_bandwidth_zero_tombstone_cycles;
let mut series_state = BucketSeriesState::default();
let mut snapshot = bucket_metrics_snapshot(bucket_interval);
loop {
tokio::select! {
_ = interval.tick() => {
run_metrics_collector_tick(health, MetricsCollectorTaskId::BucketStats, "bucket_stats", async {
let collection_started = std::time::Instant::now();
let stats = collect_bucket_stats().await;
let Some(update) = series_state.observe(stats.as_deref(), tombstone_cycles) else {
return;
};
for bucket in update.retire_observations {
let _ = retire_bucket_observation_metric_series(&bucket);
}
report_metrics(&update.metrics);
for bucket in update.retire_buckets {
let _ = retire_bucket_metric_series(&bucket);
if let Err(error) = snapshot.replace_collected(update.metrics, collection_started) {
error!(event = EVENT_METRICS_RUNTIME_STATE, component = LOG_COMPONENT_OBS, subsystem = LOG_SUBSYSTEM_METRICS_RUNTIME, collector = "bucket_stats", result = "invalid_snapshot", error = %error, "bucket metrics snapshot rejected");
}
}).await;
}
@@ -3485,29 +3344,6 @@ mod tests {
assert!(zero_tombstones.is_empty());
}
#[test]
fn bucket_observation_retirement_distinguishes_unknown_usage_from_deletion() {
let previous = HashSet::from(["bucket".to_string()]);
let unknown_stats = vec![crate::metrics::collectors::BucketStats {
name: "bucket".to_string(),
size_bytes: None,
objects_count: None,
quota_bytes: 1024,
}];
let current_buckets = bucket_live_keys(&unknown_stats);
let current_observations = bucket_observation_live_keys(&unknown_stats);
assert_eq!(
bucket_observation_retire_keys(&previous, &current_buckets, &current_observations),
vec!["bucket".to_string()],
"an existing bucket with unknown usage must retire its previous usage observations"
);
assert!(
bucket_observation_retire_keys(&previous, &HashSet::new(), &HashSet::new()).is_empty(),
"a deleted bucket remains governed by the zero-tombstone lifecycle"
);
}
#[test]
fn unavailable_bucket_snapshot_preserves_metric_series_state() {
let mut state = BucketSeriesState::default();
@@ -3520,12 +3356,10 @@ mod tests {
assert!(state.observe(Some(&initial), 2).is_some());
let live_keys = state.live_keys.clone();
let observation_keys = state.observation_keys.clone();
let zero_tombstones = state.zero_tombstones.clone();
assert!(state.observe(None, 2).is_none());
assert_eq!(state.live_keys, live_keys);
assert_eq!(state.observation_keys, observation_keys);
assert_eq!(state.zero_tombstones, zero_tombstones);
}
crates/obs/src/metrics/storage_snapshot.rs
+109 -46
View File
@@ -60,34 +60,46 @@ pub(crate) struct StorageSnapshotMetrics {
impl StorageSnapshotMetrics {
pub(crate) fn new(meter: Meter, scope: &'static str, observer: String, max_age: Duration) -> Self {
let snapshot = Arc::new(RwLock::new(Snapshot::default()));
let labels = vec![KeyValue::new(COLLECTION_SCOPE, scope), KeyValue::new(OBSERVER, observer)];
let weak = Arc::downgrade(&snapshot);
let timestamp_labels = labels.clone();
meter
.f64_observable_gauge(LAST_SUCCESS)
.with_description("Unix timestamp of the last successful storage snapshot collection")
.with_callback(move |observer| {
let Some(snapshot) = weak.upgrade() else { return };
let Ok(snapshot) = snapshot.read() else { return };
if snapshot.collected_at.is_some() {
observer.observe(snapshot.timestamp_seconds, &timestamp_labels);
}
})
.build();
let weak = Arc::downgrade(&snapshot);
let age_labels = labels.clone();
meter
.f64_observable_gauge(MAX_AGE)
.with_description("Validity budget in seconds remaining after storage snapshot collection completed")
.with_callback(move |observer| {
let Some(snapshot) = weak.upgrade() else { return };
let Ok(snapshot) = snapshot.read() else { return };
if snapshot.collected_at.is_some() {
observer.observe(snapshot.remaining_age.as_secs_f64(), &age_labels);
}
})
.build();
Self::new_with_labels(meter, labels, max_age, true)
}
/// Creates an unscoped snapshot for dynamic metric series whose membership
/// must match the latest authoritative collection.
pub(crate) fn new_unscoped(meter: Meter, max_age: Duration) -> Self {
Self::new_with_labels(meter, Vec::new(), max_age, false)
}
fn new_with_labels(meter: Meter, labels: Vec<KeyValue>, max_age: Duration, publish_freshness: bool) -> Self {
let snapshot = Arc::new(RwLock::new(Snapshot::default()));
if publish_freshness {
let weak = Arc::downgrade(&snapshot);
let timestamp_labels = labels.clone();
meter
.f64_observable_gauge(LAST_SUCCESS)
.with_description("Unix timestamp of the last successful storage snapshot collection")
.with_callback(move |observer| {
let Some(snapshot) = weak.upgrade() else { return };
let Ok(snapshot) = snapshot.read() else { return };
if snapshot.collected_at.is_some() {
observer.observe(snapshot.timestamp_seconds, &timestamp_labels);
}
})
.build();
let weak = Arc::downgrade(&snapshot);
let age_labels = labels.clone();
meter
.f64_observable_gauge(MAX_AGE)
.with_description("Validity budget in seconds remaining after storage snapshot collection completed")
.with_callback(move |observer| {
let Some(snapshot) = weak.upgrade() else { return };
let Ok(snapshot) = snapshot.read() else { return };
if snapshot.collected_at.is_some() {
observer.observe(snapshot.remaining_age.as_secs_f64(), &age_labels);
}
})
.build();
}
Self {
meter,
snapshot,
@@ -221,6 +233,7 @@ mod tests {
#[derive(Clone, Debug)]
struct ExportedPoint {
scope: String,
name: String,
value: f64,
attributes: Vec<KeyValue>,
@@ -233,28 +246,33 @@ mod tests {
async fn export(&self, metrics: &ResourceMetrics) -> OTelSdkResult {
let mut exported = self.0.lock().unwrap();
exported.clear();
for metric in metrics.scope_metrics().flat_map(|scope| scope.metrics()) {
match metric.data() {
AggregatedMetrics::F64(MetricData::Gauge(gauge)) => {
for point in gauge.data_points() {
exported.push(ExportedPoint {
name: metric.name().to_string(),
value: point.value(),
attributes: point.attributes().cloned().collect(),
});
for scope in metrics.scope_metrics() {
let scope_name = scope.scope().name().to_string();
for metric in scope.metrics() {
match metric.data() {
AggregatedMetrics::F64(MetricData::Gauge(gauge)) => {
for point in gauge.data_points() {
exported.push(ExportedPoint {
scope: scope_name.clone(),
name: metric.name().to_string(),
value: point.value(),
attributes: point.attributes().cloned().collect(),
});
}
}
}
AggregatedMetrics::U64(MetricData::Sum(sum)) => {
assert!(sum.is_monotonic());
for point in sum.data_points() {
exported.push(ExportedPoint {
name: metric.name().to_string(),
value: point.value() as f64,
attributes: point.attributes().cloned().collect(),
});
AggregatedMetrics::U64(MetricData::Sum(sum)) => {
assert!(sum.is_monotonic());
for point in sum.data_points() {
exported.push(ExportedPoint {
scope: scope_name.clone(),
name: metric.name().to_string(),
value: point.value() as f64,
attributes: point.attributes().cloned().collect(),
});
}
}
data => panic!("unexpected storage aggregation: {data:?}"),
}
data => panic!("unexpected storage aggregation: {data:?}"),
}
}
Ok(())
@@ -282,6 +300,16 @@ mod tests {
(provider, exporter, snapshot)
}
fn setup_unscoped() -> (SdkMeterProvider, Exporter, StorageSnapshotMetrics) {
let exporter = Exporter::default();
let reader = PeriodicReader::builder(exporter.clone())
.with_interval(Duration::from_secs(3600))
.build();
let provider = SdkMeterProvider::builder().with_reader(reader).build();
let snapshot = StorageSnapshotMetrics::new_unscoped(provider.meter("service-test"), Duration::from_secs(30));
(provider, exporter, snapshot)
}
fn drive(name: &'static str, kind: MetricType, id: &'static str, value: f64) -> PrometheusMetric {
PrometheusMetric::new(name, kind, "test", value)
.with_label("server", "node1:9000")
@@ -326,6 +354,41 @@ mod tests {
provider.shutdown().unwrap();
}
#[test]
fn actual_sdk_stops_exporting_deleted_bucket_quota_series_after_tombstone() {
let (provider, exporter, mut snapshot) = setup_unscoped();
let bucket = "deleted-bucket";
let quota_metrics = |value| {
vec![
PrometheusMetric::from_descriptor(&super::super::schema::node_bucket::BUCKET_QUOTA_BYTES_MD, value)
.with_label("bucket", bucket),
PrometheusMetric::from_descriptor(&super::super::schema::cluster_usage::USAGE_BUCKET_QUOTA_TOTAL_BYTES_MD, value)
.with_label("bucket", bucket),
]
};
snapshot.replace(quota_metrics(1024.0)).unwrap();
assert_eq!(
flush(&provider, &exporter)
.iter()
.filter(|point| point.value == 1024.0)
.count(),
2
);
assert!(flush(&provider, &exporter).iter().all(|point| point.scope == "service-test"));
snapshot.replace(quota_metrics(0.0)).unwrap();
let tombstones = flush(&provider, &exporter);
assert_eq!(tombstones.iter().filter(|point| point.value == 0.0).count(), 2);
snapshot.replace(Vec::new()).unwrap();
for _ in 0..2 {
let points = flush(&provider, &exporter);
assert!(points.is_empty(), "deleted bucket quota points must not remain in cumulative exports");
}
provider.shutdown().unwrap();
}
#[test]
fn actual_sdk_preserves_counter_resets_instead_of_accumulating_old_absolute_values() {
let (provider, exporter, mut snapshot) = setup();
crates/obs/src/telemetry/mod.rs
+1
View File
@@ -53,6 +53,7 @@ use crate::TelemetryError;
use crate::config::OtelConfig;
pub use filter::HTTP_SERVER_LOG_TARGET;
pub use guard::OtelGuard;
pub(crate) use recorder::process_global_meter;
pub use recorder::{Recorder, retire_metric_series};
use rustfs_config::observability::ENV_OBS_LOG_DIRECTORY;
use rustfs_config::{DEFAULT_LOG_LEVEL, ENVIRONMENT, observability::DEFAULT_OBS_ENVIRONMENT_PRODUCTION};

Some files were not shown because too many files have changed in this diff Show More