gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-16 09:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into cxymds/fix-rebalance-multipart-retry

Browse Source 
# Conflicts:
#	crates/ecstore/src/set_disk/ops/object.rs
#	crates/ecstore/src/set_disk/read.rs
This commit is contained in:
马登山
2026-08-13 08:13:26 +08:00
parent 6cecf7d26f 65091aa6a8
commit fd727a8053
161 changed files with 8100 additions and 5463 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/checksums/src/http.rs
+7
View File
@@ -21,6 +21,13 @@ use crate::{
Xxhash3, Xxhash64, Xxhash128,
};
// DELIBERATE DUPLICATION of the x-amz-checksum-* names that also exist as
// AMZ_CHECKSUM_* in rustfs-utils' headers module (crates/utils/src/http/
// headers.rs): this crate is a zero-internal-dependency leaf, so it cannot
// import them, and it additionally owns the RustFS extension names
// (sha512/xxhash*) that utils does not carry. Values are pinned by the S3
// wire protocol; do not merge without a maintainer decision on the leaf
// boundary (backlog#1833).
pub const CRC_32_HEADER_NAME: &str = "x-amz-checksum-crc32";
pub const CRC_32_C_HEADER_NAME: &str = "x-amz-checksum-crc32c";
pub const SHA_1_HEADER_NAME: &str = "x-amz-checksum-sha1";
crates/checksums/src/lib.rs
+8
View File
@@ -41,6 +41,14 @@ pub const XXHASH_64_NAME: &str = "xxhash64";
pub const XXHASH_128_NAME: &str = "xxhash128";
pub const MD5_NAME: &str = "md5";
/// One of three deliberately separate checksum registries (backlog#1833):
/// this enum owns the **streaming-hash algorithm registry**, including the
/// RustFS extensions (sha512, xxhash3/64/128). The on-disk xl.meta bitset
/// lives in `rustfs_rio::ChecksumType` (crates/rio/src/checksum.rs, varint
/// bits are append-only), and the MinIO-port client keeps its own
/// `ChecksumMode` (crates/ecstore/src/client/checksum.rs). When adding an
/// algorithm, extend all three (or record why not) — they do not derive from
/// each other.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[non_exhaustive]
pub enum ChecksumAlgorithm {
crates/common/src/bucket_stats.rs
-87
View File
@@ -1,87 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::last_minute::{self};
use std::collections::HashMap;
pub struct ReplicationLatency {
// Delays for single and multipart PUT requests
upload_histogram: last_minute::LastMinuteHistogram,
}
impl ReplicationLatency {
// Merge two ReplicationLatency
pub fn merge(&mut self, other: &mut ReplicationLatency) -> &ReplicationLatency {
self.upload_histogram.merge(&other.upload_histogram);
self
}
// Get upload delay (categorized by object size interval)
pub fn get_upload_latency(&mut self) -> HashMap<String, u64> {
let mut ret = HashMap::new();
let avg = self.upload_histogram.get_avg_data();
for (i, v) in avg.iter().enumerate() {
let avg_duration = v.avg();
ret.insert(self.size_tag_to_string(i), avg_duration.as_millis() as u64);
}
ret
}
pub fn update(&mut self, size: i64, during: std::time::Duration) {
self.upload_histogram.add(size, during);
}
// Simulate the conversion from size tag to string
fn size_tag_to_string(&self, tag: usize) -> String {
match tag {
0 => String::from("Size < 1 KiB"),
1 => String::from("Size < 1 MiB"),
2 => String::from("Size < 10 MiB"),
3 => String::from("Size < 100 MiB"),
4 => String::from("Size < 1 GiB"),
_ => String::from("Size > 1 GiB"),
}
}
}
// #[derive(Debug, Clone, Default)]
// pub struct ReplicationLastMinute {
// pub last_minute: LastMinuteLatency,
// }
// impl ReplicationLastMinute {
// pub fn merge(&mut self, other: ReplicationLastMinute) -> ReplicationLastMinute {
// let mut nl = ReplicationLastMinute::default();
// nl.last_minute = self.last_minute.merge(&mut other.last_minute);
// nl
// }
// pub fn add_size(&mut self, n: i64) {
// let t = SystemTime::now()
// .duration_since(UNIX_EPOCH)
// .expect("Time went backwards")
// .as_secs();
// self.last_minute.add_all(t - 1, &AccElem { total: t - 1, size: n as u64, n: 1 });
// }
// pub fn get_total(&self) -> AccElem {
// self.last_minute.get_total()
// }
// }
// impl fmt::Display for ReplicationLastMinute {
// fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
// let t = self.last_minute.get_total();
// write!(f, "ReplicationLastMinute sz= {}, n= {}, dur= {}", t.size, t.n, t.total)
// }
// }
crates/common/src/last_minute.rs
-41
View File
@@ -572,44 +572,3 @@ mod tests {
assert_eq!(total.n, 6);
}
}
const SIZE_LAST_ELEM_MARKER: usize = 10; // Assumed marker size is 10, modify according to actual situation
#[allow(dead_code)]
#[derive(Debug, Default)]
pub struct LastMinuteHistogram {
histogram: Vec<LastMinuteLatency>,
size: u32,
}
impl LastMinuteHistogram {
pub fn merge(&mut self, other: &LastMinuteHistogram) {
for i in 0..self.histogram.len() {
self.histogram[i].merge(&other.histogram[i]);
}
}
pub fn add(&mut self, size: i64, t: Duration) {
let index = size_to_tag(size);
self.histogram[index].add(&t);
}
pub fn get_avg_data(&mut self) -> [AccElem; SIZE_LAST_ELEM_MARKER] {
let mut res = [AccElem::default(); SIZE_LAST_ELEM_MARKER];
for (i, elem) in self.histogram.iter_mut().enumerate() {
res[i] = elem.get_total();
}
res
}
}
fn size_to_tag(size: i64) -> usize {
match size {
_ if size < 1024 => 0, // sizeLessThan1KiB
_ if size < 1024 * 1024 => 1, // sizeLessThan1MiB
_ if size < 10 * 1024 * 1024 => 2, // sizeLessThan10MiB
_ if size < 100 * 1024 * 1024 => 3, // sizeLessThan100MiB
_ if size < 1024 * 1024 * 1024 => 4, // sizeLessThan1GiB
_ => 5, // sizeGreaterThan1GiB
}
}
crates/common/src/lib.rs
-1
View File
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod bucket_stats;
// pub mod error;
pub mod globals;
pub mod heal_channel;
crates/common/src/metrics.rs
+34
View File
@@ -915,11 +915,13 @@ const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1;
const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3;
const SCAN_CYCLE_RESULT_SUPERSEDED: u8 = 4;
const SCAN_CYCLE_RESULT_DEFERRED: u8 = 5;
const SCAN_CYCLE_RESULT_UNKNOWN_LABEL: &str = "unknown";
const SCAN_CYCLE_RESULT_SUCCESS_LABEL: &str = "success";
const SCAN_CYCLE_RESULT_ERROR_LABEL: &str = "error";
const SCAN_CYCLE_RESULT_PARTIAL_LABEL: &str = "partial";
const SCAN_CYCLE_RESULT_SUPERSEDED_LABEL: &str = "superseded";
const SCAN_CYCLE_RESULT_DEFERRED_LABEL: &str = "deferred";
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum ScanCyclePartialReason {
@@ -1424,6 +1426,7 @@ fn scan_cycle_result_label(result: u8) -> &'static str {
SCAN_CYCLE_RESULT_ERROR => SCAN_CYCLE_RESULT_ERROR_LABEL,
SCAN_CYCLE_RESULT_PARTIAL => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
SCAN_CYCLE_RESULT_SUPERSEDED => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL,
SCAN_CYCLE_RESULT_DEFERRED => SCAN_CYCLE_RESULT_DEFERRED_LABEL,
_ => SCAN_CYCLE_RESULT_UNKNOWN_LABEL,
}
}
@@ -1752,6 +1755,11 @@ pub fn emit_scan_cycle_superseded(duration: Duration) {
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_SUPERSEDED_LABEL).increment(1);
}
pub fn emit_scan_cycle_deferred(duration: Duration) {
global_metrics().record_scan_cycle_deferred(duration);
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_DEFERRED_LABEL).increment(1);
}
pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
let result = if success { "success" } else { "error" };
global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
@@ -2549,6 +2557,17 @@ impl Metrics {
.store(duration_millis_saturated(duration), Ordering::Relaxed);
}
pub fn record_scan_cycle_deferred(&self, duration: Duration) {
self.record_scanner_cycle_end_time();
self.last_scan_cycle_result
.store(SCAN_CYCLE_RESULT_DEFERRED, Ordering::Relaxed);
self.last_scan_cycle_partial_reason
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
self.last_scan_cycle_duration_millis
.store(duration_millis_saturated(duration), Ordering::Relaxed);
}
pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) {
self.record_scan_cycle_partial_with_source(duration, reason, None);
}
@@ -4264,6 +4283,21 @@ mod tests {
assert_eq!(report.partial_cycles, 0);
}
#[tokio::test]
async fn report_tracks_deferred_cycle_without_failed_increment() {
let metrics = Metrics::new();
metrics.record_scan_cycle_deferred(Duration::from_millis(250));
let report = metrics.report().await;
assert_eq!(report.last_cycle_result, SCAN_CYCLE_RESULT_DEFERRED_LABEL);
assert_eq!(report.last_cycle_result_code, u64::from(SCAN_CYCLE_RESULT_DEFERRED));
assert_eq!(report.last_cycle_duration_seconds, 0.25);
assert_eq!(report.failed_cycles, 0);
assert_eq!(report.superseded_cycles, 0);
assert_eq!(report.partial_cycles, 0);
}
#[tokio::test]
async fn report_tracks_successful_scan_cycle_without_failed_increment() {
let metrics = Metrics::new();
crates/config/src/constants/runtime.rs
+3
View File
@@ -81,6 +81,9 @@ pub const ENV_TEST_IAM_FAIL_INIT_ATTEMPTS: &str = "RUSTFS_TEST_IAM_FAIL_INIT_ATT
pub const ENV_TEST_IAM_RETRY_INTERVAL_MS: &str = "RUSTFS_TEST_IAM_RETRY_INTERVAL_MS";
/// Runtime env var controlling the transition worker count.
pub const ENV_TRANSITION_WORKERS: &str = "RUSTFS_MAX_TRANSITION_WORKERS";
/// Runtime env var controlling the ILM expiry worker count. A set, parsable,
/// non-zero value wins; anything else falls back to `min(cpus, 16)`.
pub const ENV_MAX_EXPIRY_WORKERS: &str = "RUSTFS_MAX_EXPIRY_WORKERS";
/// Runtime env var controlling the absolute maximum transition workers.
pub const ENV_TRANSITION_WORKERS_ABSOLUTE_MAX: &str = "RUSTFS_ABSOLUTE_MAX_WORKERS";
/// Runtime env var controlling the transition queue capacity.
crates/e2e_test/src/get_codec_streaming_compat_test.rs
+7 -2
View File
@@ -189,8 +189,6 @@ mod tests {
("RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT", "100"),
("RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED", "true"),
("RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED", "true"),
// Lower the min-size floor so every non-inline object below is eligible.
("RUSTFS_GET_CODEC_STREAMING_MIN_SIZE", "4096"),
// Route multipart objects through per-part codec streaming too.
("RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE", "true"),
// Lock optimization is on by default, but pin it so the gate's
@@ -315,6 +313,13 @@ mod tests {
},
payload(64 * 1024, 2),
),
(
Shape {
key: "small-non-inline-256kib-plus",
expect_large: true,
},
payload(256 * 1024 + 1, 6),
),
(
Shape {
key: "mid-1_5mib",
crates/e2e_test/src/group_delete_test.rs
+51 -1
View File
@@ -14,7 +14,7 @@
//! E2E tests for group management (fixes #2028).
use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_get, awscurl_put, init_logging};
use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging};
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::{Client, Config};
use serial_test::serial;
@@ -32,6 +32,56 @@ fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_k
Client::from_conf(config)
}
#[tokio::test(flavor = "multi_thread")]
async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let invalid_groups = [
("test group", "group name contains whitespace"),
("test=group", "group name contains reserved characters =,"),
("test,group", "group name contains reserved characters =,"),
];
for (group, expected_message) in invalid_groups {
let body = serde_json::json!({
"group": group,
"members": [],
"isRemove": false,
"groupStatus": "enabled"
})
.to_string();
let (status, response_body) = admin_request(
&env.url,
http::Method::PUT,
"/rustfs/admin/v3/update-group-members",
Some(body),
&env.access_key,
&env.secret_key,
)
.await?;
assert_eq!(
status,
reqwest::StatusCode::BAD_REQUEST,
"invalid group {group:?} must return HTTP 400, body: {response_body}"
);
assert!(
response_body.contains("<Code>InvalidArgument</Code>"),
"invalid group {group:?} must return InvalidArgument, body: {response_body}"
);
assert!(
response_body.contains(&format!("<Message>{expected_message}</Message>")),
"invalid group {group:?} returned an unexpected message: {response_body}"
);
}
env.stop_server();
Ok(())
}
/// Test that deleting a group with members fails, and deleting an empty group succeeds.
#[tokio::test(flavor = "multi_thread")]
#[serial]
crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs
+612
View File
@@ -0,0 +1,612 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! ILM on SSE-KMS buckets while per-key SSE authorization is enforced (backlog#1582).
//!
//! Per-key KMS authorization (`RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY=true`) scopes the
//! SSE-KMS data path to the requesting principal's `kms:GenerateDataKey` /
//! `kms:Decrypt` grants. Internal callers — the lifecycle scanner's expiry deletes
//! and the tier transition worker's reads — carry no request principal, and
//! `authorize_sse_kms_key` (rustfs/src/storage/sse.rs) exempts a `None` principal
//! so background maintenance keeps working on encrypted buckets.
//!
//! These tests pin that exemption end to end. If enforcement ever starts applying
//! to the scanner's internal operations, expiry stops happening on SSE-KMS buckets
//! and [`ilm_expiration_on_sse_kms_bucket_under_enforcement`] times out; if it
//! starts applying to the transition worker or the read-through path,
//! [`ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back`] fails at the
//! transition wait or the plaintext round-trip.
//!
//! The replication half of the same acceptance item lives in
//! `crates/e2e_test/src/replication_extension_test.rs`
//! (`test_bucket_replication_sse_kms_failure_contract`); ILM had no coverage
//! before this file.
//!
//! Deployment constraint pinned by the transition test's setup: the RustFS warm
//! backend forwards the object's stored `x-amz-server-side-encryption*` metadata
//! as raw headers on the tier data PUT (`build_transition_put_options` +
//! `api_put_object.rs` header mapping), so a RustFS tier target must itself have
//! KMS enabled and hold the named key or it rejects every transition upload with
//! 400 InvalidRequest. That rejection is independent of the enforcement switch;
//! the cold server here therefore runs its own Local KMS with the same key id.
use super::common::{LocalKMSTestEnvironment, create_key_with_specific_id};
use crate::common::{RustFSTestEnvironment, admin_request, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, RestoreRequest,
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, Transition,
TransitionStorageClass,
};
use serde::Deserialize;
use serial_test::serial;
use std::time::{Duration as StdDuration, Instant};
use tracing::info;
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
const SSE_KEY: &str = "kms-ilm-sse-key";
const PAYLOAD: &[u8] = b"kms ilm sse payload: survives enforcement, expires and transitions on schedule";
const EXPIRY_BUCKET: &str = "kms-ilm-expiry";
const EXPIRE_KEY: &str = "expire/object.bin";
const SURVIVOR_KEY: &str = "keep/object.bin";
const TIER_NAME: &str = "KMSCOLD";
const TIER_BUCKET: &str = "kms-ilm-cold-tier";
const TIER_PREFIX: &str = "tiered";
const TRANSITION_BUCKET: &str = "kms-ilm-transition";
const TRANSITION_KEY: &str = "tier/object.bin";
/// Generous CI safety net; with a 1s scanner cycle and 2s lifecycle days the
/// terminal state normally lands within a few seconds.
const ILM_DEADLINE: StdDuration = StdDuration::from_secs(90);
/// Start a Local-KMS server with per-key SSE authorization enforced and the
/// lifecycle clock accelerated.
///
/// KMS wiring matches `kms_authorization_negative_matrix_test.rs` (local backend,
/// `--kms-default-key-id`, insecure dev defaults). The lifecycle env matches
/// `reliant/lifecycle.rs::fast_lifecycle_env` plus `RUSTFS_ILM_DEBUG_DAY_SECS=2`,
/// so a `Days=1` rule is due about two seconds after the write.
async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment) -> TestResult {
create_key_with_specific_id(&env.kms_keys_dir, SSE_KEY).await?;
let key_dir = env.kms_keys_dir.clone();
let args = vec![
"--kms-enable",
"--kms-backend",
"local",
"--kms-key-dir",
key_dir.as_str(),
"--kms-default-key-id",
SSE_KEY,
];
let envs = [
("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "false"),
("RUSTFS_SCANNER_CYCLE", "1"),
("RUSTFS_ILM_PROCESS_TIME", "1"),
("RUSTFS_ILM_DEBUG_DAY_SECS", "2"),
];
env.base_env.start_rustfs_server_with_env(args, &envs).await?;
Ok(())
}
/// Set the bucket's default encryption to SSE-KMS under [`SSE_KEY`], so plain
/// PUTs (and internal rewrites) are encrypted without per-request SSE headers.
async fn set_bucket_default_sse_kms(client: &Client, bucket: &str) -> TestResult {
let encryption_config = ServerSideEncryptionConfiguration::builder()
.rules(
ServerSideEncryptionRule::builder()
.apply_server_side_encryption_by_default(
ServerSideEncryptionByDefault::builder()
.sse_algorithm(ServerSideEncryption::AwsKms)
.kms_master_key_id(SSE_KEY)
.build()?,
)
.build(),
)
.build()?;
client
.put_bucket_encryption()
.bucket(bucket)
.server_side_encryption_configuration(encryption_config)
.send()
.await?;
Ok(())
}
/// Assert via `HeadObject` that the stored object is SSE-KMS encrypted under
/// [`SSE_KEY`]. Without this, a bucket-default misconfiguration would let the
/// tests pass on an unencrypted object and prove nothing about KMS.
async fn assert_head_sse_kms(client: &Client, bucket: &str, key: &str) -> TestResult {
let head = client.head_object().bucket(bucket).key(key).send().await?;
assert_eq!(
head.server_side_encryption(),
Some(&ServerSideEncryption::AwsKms),
"{bucket}/{key} must be SSE-KMS encrypted via the bucket default"
);
assert_eq!(
head.ssekms_key_id(),
Some(SSE_KEY),
"{bucket}/{key} must be wrapped under the configured KMS key"
);
Ok(())
}
/// Returns `true` once `GET bucket/key` fails with `NoSuchKey`, `false` while it
/// still succeeds. Any other error is surfaced. (Copied from
/// `reliant/lifecycle.rs`; that helper is private to the reliant module.)
async fn object_is_gone(client: &Client, bucket: &str, key: &str) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
match client.get_object().bucket(bucket).key(key).send().await {
Ok(output) => {
output.body.collect().await?;
Ok(false)
}
Err(e) => {
if let Some(service_error) = e.as_service_error() {
if service_error.is_no_such_key() {
return Ok(true);
}
return Err(format!("expected NoSuchKey, got: {e:?}").into());
}
Err(format!("expected a service error, got: {e:?}").into())
}
}
}
/// Poll until `GET bucket/key` returns `NoSuchKey`, or fail after `deadline`.
async fn wait_for_object_expired(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult {
let start = Instant::now();
loop {
if object_is_gone(client, bucket, key).await? {
return Ok(());
}
if start.elapsed() >= deadline {
return Err(format!(
"object {bucket}/{key} was not expired by the lifecycle scanner within {}s; \
SSE key-policy enforcement may have started blocking the scanner's internal deletes",
deadline.as_secs()
)
.into());
}
tokio::time::sleep(StdDuration::from_millis(500)).await;
}
}
/// Install a prefix-scoped `Days`-based expiration rule.
async fn put_expiration_rule(client: &Client, bucket: &str, id: &str, prefix: &str, days: i32) -> TestResult {
let rule = LifecycleRule::builder()
.id(id)
.filter(LifecycleRuleFilter::builder().prefix(prefix).build())
.expiration(LifecycleExpiration::builder().days(days).build())
.status(ExpirationStatus::Enabled)
.build()?;
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
client
.put_bucket_lifecycle_configuration()
.bucket(bucket)
.lifecycle_configuration(lifecycle)
.send()
.await?;
Ok(())
}
/// Install a prefix-scoped `Days`-based transition rule targeting [`TIER_NAME`].
async fn put_transition_rule(client: &Client, bucket: &str, id: &str, prefix: &str, days: i32) -> TestResult {
let rule = LifecycleRule::builder()
.id(id)
.filter(LifecycleRuleFilter::builder().prefix(prefix).build())
.transitions(
Transition::builder()
.days(days)
.storage_class(TransitionStorageClass::from(TIER_NAME))
.build(),
)
.status(ExpirationStatus::Enabled)
.build()?;
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
client
.put_bucket_lifecycle_configuration()
.bucket(bucket)
.lifecycle_configuration(lifecycle)
.send()
.await?;
Ok(())
}
/// Start a plain Local-KMS server (no enforcement, no lifecycle acceleration)
/// holding [`SSE_KEY`], to serve as the cold tier target.
///
/// The RustFS warm backend forwards the stored SSE-KMS headers on the tier data
/// PUT, so the target re-applies managed SSE-KMS under the named key and must
/// be able to resolve it; without KMS it answers 400 InvalidRequest and the
/// transition can never complete. Enforcement stays off here: the tier writes
/// arrive under `cold`'s root credentials, and one enforcing side is enough to
/// pin the exemption.
async fn start_cold_tier_kms_server(env: &mut LocalKMSTestEnvironment) -> TestResult {
create_key_with_specific_id(&env.kms_keys_dir, SSE_KEY).await?;
let key_dir = env.kms_keys_dir.clone();
let args = vec![
"--kms-enable",
"--kms-backend",
"local",
"--kms-key-dir",
key_dir.as_str(),
"--kms-default-key-id",
SSE_KEY,
];
env.base_env
.start_rustfs_server_with_env(args, &[("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true")])
.await?;
Ok(())
}
/// The subset of the manual transition run report these tests assert on.
///
/// Unknown fields are ignored, so this stays compatible with report growth; the
/// full shape is pinned by `reliant/tiering.rs`.
#[derive(Debug, Deserialize)]
struct ManualTransitionRunReport {
#[serde(default)]
scanned: u64,
#[serde(default)]
enqueued: u64,
#[serde(default)]
skipped_already_in_flight: u64,
#[serde(default)]
skipped_tier: u64,
}
#[derive(Debug, Deserialize)]
struct ManualTransitionRunResponse {
state: String,
report: ManualTransitionRunReport,
}
/// One synchronous (enqueue-only) manual transition run over `bucket/prefix`,
/// via the same admin endpoint `reliant/tiering.rs` drives.
async fn manual_transition_run(
hot: &RustFSTestEnvironment,
bucket: &str,
prefix: &str,
) -> Result<ManualTransitionRunResponse, Box<dyn std::error::Error + Send + Sync>> {
let bucket = urlencoding::encode(bucket);
let prefix = urlencoding::encode(prefix);
let tier = urlencoding::encode(TIER_NAME);
let path =
format!("/rustfs/admin/v3/ilm/transition/run?bucket={bucket}&prefix={prefix}&tier={tier}&dryRun=false&maxObjects=10");
let (status, body) = admin_request(&hot.url, http::Method::POST, &path, None, &hot.access_key, &hot.secret_key).await?;
if !status.is_success() {
return Err(format!("manual transition run failed: status={status}, body={body}").into());
}
Ok(serde_json::from_str(&body)?)
}
/// Drive manual transition runs until one reports the object as processed.
///
/// The `Days=1` rule becomes due about two seconds after the write
/// (`RUSTFS_ILM_DEBUG_DAY_SECS=2`), so early runs may legitimately report the
/// object as not yet eligible; the loop keeps running the endpoint until it
/// either enqueues the transition, sees it already in flight (the 1s scanner
/// backstop got there first), or finds it already on the tier.
async fn run_manual_transition_until_processed(
hot: &RustFSTestEnvironment,
bucket: &str,
prefix: &str,
deadline: StdDuration,
) -> TestResult {
let start = Instant::now();
loop {
let run = manual_transition_run(hot, bucket, prefix).await?;
assert_eq!(run.report.scanned, 1, "manual transition run must scan the object: {run:#?}");
if run.report.enqueued + run.report.skipped_already_in_flight + run.report.skipped_tier >= 1 {
info!(state = %run.state, report = ?run.report, "manual transition run processed the SSE-KMS object");
return Ok(());
}
if start.elapsed() >= deadline {
return Err(format!(
"manual transition runs never processed {bucket}/{prefix} within {}s; last report: {run:#?}",
deadline.as_secs()
)
.into());
}
tokio::time::sleep(StdDuration::from_millis(500)).await;
}
}
/// Wire `hot` -> `cold` as a `TierType::RustFS` remote tier via `AddTier`.
///
/// No `force`, so the server runs the real connectivity probe against `cold`
/// (the tier bucket must already exist there). Mirrors
/// `reliant/tiering.rs::add_rustfs_tier`, which is private to that module.
async fn add_rustfs_tier(hot: &RustFSTestEnvironment, cold: &RustFSTestEnvironment) -> TestResult {
let body = serde_json::json!({
"type": "rustfs",
"rustfs": {
"name": TIER_NAME,
"endpoint": cold.url.as_str(),
"accessKey": cold.access_key.as_str(),
"secretKey": cold.secret_key.as_str(),
"bucket": TIER_BUCKET,
"prefix": TIER_PREFIX,
"region": "us-east-1",
"storageClass": ""
}
})
.to_string();
let (status, resp) = admin_request(
&hot.url,
http::Method::PUT,
"/rustfs/admin/v3/tier",
Some(body),
&hot.access_key,
&hot.secret_key,
)
.await?;
if !status.is_success() {
return Err(format!("AddTier(RustFS) failed: status={status}, body={resp}").into());
}
Ok(())
}
/// Poll `HEAD` until the object's storage class is the tier name (transition
/// complete), or fail after `deadline`. (From `reliant/tiering.rs`.)
async fn wait_for_transition(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult {
let start = Instant::now();
loop {
let head = client.head_object().bucket(bucket).key(key).send().await?;
if head.storage_class().map(|sc| sc.as_str()) == Some(TIER_NAME) {
return Ok(());
}
if start.elapsed() >= deadline {
return Err(format!(
"object {bucket}/{key} was not transitioned to {TIER_NAME} within {}s (storage_class={:?}); \
SSE key-policy enforcement may have started blocking the transition worker's internal reads",
deadline.as_secs(),
head.storage_class()
)
.into());
}
tokio::time::sleep(StdDuration::from_millis(500)).await;
}
}
/// Poll `HEAD` until `x-amz-restore` reports a finished restore
/// (`ongoing-request="false"`), or fail after `deadline`.
async fn wait_for_restore_complete(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult {
let start = Instant::now();
loop {
let head = client.head_object().bucket(bucket).key(key).send().await?;
if head.restore().is_some_and(|r| r.contains("ongoing-request=\"false\"")) {
return Ok(());
}
if start.elapsed() >= deadline {
return Err(format!(
"object {bucket}/{key} restore did not complete within {}s (restore={:?}); \
SSE key-policy enforcement may have started blocking the restore copy-back's internal reads",
deadline.as_secs(),
head.restore()
)
.into());
}
tokio::time::sleep(StdDuration::from_millis(500)).await;
}
}
/// ILM expiration keeps working on an SSE-KMS bucket while per-key SSE
/// authorization is enforced.
///
/// The lifecycle scanner deletes expired objects with an internal (no-principal)
/// identity that holds no `kms` grant. If enforcement ever starts applying to
/// those internal deletes (or to the scanner's metadata reads) on encrypted
/// buckets, expiry stops happening and this test times out.
///
/// A survivor object under a non-matching prefix isolates the rule's prefix
/// filter as the cause of the deletion and proves the encrypted bucket stays
/// readable end to end after the scanner has run.
#[tokio::test]
#[serial]
async fn ilm_expiration_on_sse_kms_bucket_under_enforcement() -> TestResult {
init_logging();
let mut env = LocalKMSTestEnvironment::new().await?;
start_enforcing_ilm_server(&mut env).await?;
env.base_env.create_test_bucket(EXPIRY_BUCKET).await?;
let client = env.base_env.create_s3_client();
set_bucket_default_sse_kms(&client, EXPIRY_BUCKET).await?;
for key in [EXPIRE_KEY, SURVIVOR_KEY] {
client
.put_object()
.bucket(EXPIRY_BUCKET)
.key(key)
.body(ByteStream::from_static(PAYLOAD))
.send()
.await?;
assert_head_sse_kms(&client, EXPIRY_BUCKET, key).await?;
}
info!("both objects stored SSE-KMS encrypted under enforcement");
put_expiration_rule(&client, EXPIRY_BUCKET, "kms-ilm-expire", "expire/", 1).await?;
// The regression this pins: the scanner's internal delete must stay exempt
// from per-key SSE authorization, so the encrypted object actually expires.
wait_for_object_expired(&client, EXPIRY_BUCKET, EXPIRE_KEY, ILM_DEADLINE).await?;
info!("SSE-KMS object expired by the lifecycle scanner under enforcement");
// Negative control: same bucket, same encryption, non-matching prefix. It
// must survive the scanner and still decrypt for the requesting principal.
assert!(
!object_is_gone(&client, EXPIRY_BUCKET, SURVIVOR_KEY).await?,
"non-matching-prefix object must not be expired by a prefix-scoped rule"
);
let survivor = client.get_object().bucket(EXPIRY_BUCKET).key(SURVIVOR_KEY).send().await?;
assert_eq!(
survivor.body.collect().await?.into_bytes().as_ref(),
PAYLOAD,
"surviving SSE-KMS object must still decrypt after the scanner has run"
);
Ok(())
}
/// ILM transition to a remote tier keeps working on an SSE-KMS bucket while
/// per-key SSE authorization is enforced, and the transitioned object reads
/// back as plaintext.
///
/// The transition worker moves the stored (encrypted) bytes to the cold tier
/// with an internal (no-principal) identity; the read-through `GET` then
/// decrypts the envelope for the requesting principal. If enforcement ever
/// starts applying to the worker's internal reads, the transition wait times
/// out; if the stored envelope is mishandled across the tier round trip, the
/// plaintext comparison fails.
///
/// The transition is driven through the manual transition-run admin endpoint
/// (the mechanism `reliant/tiering.rs` established), so the test does not
/// depend on scanner scheduling; the 1s scanner cycle stays on as a backstop.
#[tokio::test]
#[serial]
#[ignore = "pins rustfs/rustfs#6025: GET on a transitioned managed-SSE object silently returns corrupt bytes (fails with enforcement on AND off, so it is not an authorization regression); un-ignore with the fix"]
async fn ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back() -> TestResult {
init_logging();
// Cold-tier server: independent credentials, its own Local KMS holding the
// same key id (see the module docs for why the tier target needs KMS).
// Started first; each server's startup cleanup only matches its own unique
// address and temp dir, so the two instances coexist.
let mut cold = LocalKMSTestEnvironment::new().await?;
cold.base_env.access_key = "kmscoldtieradmin".to_string();
cold.base_env.secret_key = "kmscoldtiersecret".to_string();
start_cold_tier_kms_server(&mut cold).await?;
let cold_client = cold.base_env.create_s3_client();
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
// Hot server: Local KMS + enforcement + accelerated lifecycle clock.
let mut env = LocalKMSTestEnvironment::new().await?;
start_enforcing_ilm_server(&mut env).await?;
let hot_client = env.base_env.create_s3_client();
add_rustfs_tier(&env.base_env, &cold.base_env).await?;
env.base_env.create_test_bucket(TRANSITION_BUCKET).await?;
set_bucket_default_sse_kms(&hot_client, TRANSITION_BUCKET).await?;
hot_client
.put_object()
.bucket(TRANSITION_BUCKET)
.key(TRANSITION_KEY)
.body(ByteStream::from_static(PAYLOAD))
.send()
.await?;
assert_head_sse_kms(&hot_client, TRANSITION_BUCKET, TRANSITION_KEY).await?;
info!("object stored SSE-KMS encrypted under enforcement");
// Days=1 is due ~2s after the write with RUSTFS_ILM_DEBUG_DAY_SECS=2.
put_transition_rule(&hot_client, TRANSITION_BUCKET, "kms-ilm-transition", "tier/", 1).await?;
// Drive the transition deterministically via the manual run endpoint, then
// wait for HEAD to report the tier as the object's storage class.
run_manual_transition_until_processed(&env.base_env, TRANSITION_BUCKET, "tier/", ILM_DEADLINE).await?;
wait_for_transition(&hot_client, TRANSITION_BUCKET, TRANSITION_KEY, ILM_DEADLINE).await?;
info!("SSE-KMS object transitioned to the remote tier under enforcement");
let head = hot_client
.head_object()
.bucket(TRANSITION_BUCKET)
.key(TRANSITION_KEY)
.send()
.await?;
assert!(
head.restore().is_none(),
"a freshly transitioned object must not advertise x-amz-restore, got {:?}",
head.restore()
);
// The remote copy exists on the cold tier. The payload the tier holds is the
// hot server's stored ciphertext, wrapped once more under the cold server's
// own managed SSE-KMS layer (the forwarded headers re-request encryption).
let remote = cold_client.list_objects_v2().bucket(TIER_BUCKET).send().await?;
assert!(!remote.contents().is_empty(), "cold-tier bucket must hold the transitioned object's data");
// Read-through GET under enforcement must succeed (not AccessDenied) and
// keep advertising SSE-KMS. Its BODY is deliberately not compared here:
// the transitioned read path skips managed-SSE decryption — a product gap
// unrelated to enforcement — so a direct GET streams the stored ciphertext
// (`new_getobjectreader` in crates/ecstore/src/client/object_api_utils.rs
// hardcodes `is_encrypted = false` and never applies the
// `ReadTransform::Encrypted` wrapping the hot-read path builds in
// crates/ecstore/src/object_api/readers.rs). Plaintext recovery is pinned
// through restore semantics below; when the read-through gap is fixed, a
// byte assertion can be added here too.
let read_through = hot_client
.get_object()
.bucket(TRANSITION_BUCKET)
.key(TRANSITION_KEY)
.send()
.await?;
assert_eq!(
read_through.server_side_encryption(),
Some(&ServerSideEncryption::AwsKms),
"transitioned object must still report SSE-KMS on read-through"
);
let read_through_body = read_through.body.collect().await?.into_bytes();
assert_eq!(
read_through_body.len(),
PAYLOAD.len(),
"read-through GET must stream the object's full logical size under enforcement"
);
// RestoreObject copies the ciphertext back from the tier under the original
// envelope metadata; the restored copy is then served by the normal
// decrypting read path. The copy-back runs with an internal (no-principal)
// identity, so this also pins the exemption on the restore path. Days=300
// because RUSTFS_ILM_DEBUG_DAY_SECS=2 accelerates the restored copy's
// expiry as well (300 accelerated days == 600s of validity).
hot_client
.restore_object()
.bucket(TRANSITION_BUCKET)
.key(TRANSITION_KEY)
.restore_request(RestoreRequest::builder().days(300).build())
.send()
.await?;
wait_for_restore_complete(&hot_client, TRANSITION_BUCKET, TRANSITION_KEY, ILM_DEADLINE).await?;
info!("SSE-KMS object restored from the remote tier under enforcement");
// The KMS-relevant half: the restored envelope decrypts back to the exact
// plaintext for the requesting principal.
let restored = hot_client
.get_object()
.bucket(TRANSITION_BUCKET)
.key(TRANSITION_KEY)
.send()
.await?;
assert_eq!(
restored.server_side_encryption(),
Some(&ServerSideEncryption::AwsKms),
"restored object must still report SSE-KMS"
);
let body = restored.body.collect().await?.into_bytes();
assert_eq!(body.as_ref(), PAYLOAD, "restored SSE-KMS object must round-trip byte-identical plaintext");
Ok(())
}
crates/e2e_test/src/kms/mod.rs
+3
View File
@@ -59,3 +59,6 @@ mod configured_roundtrip_test;
#[cfg(test)]
mod kms_authorization_negative_matrix_test;
#[cfg(test)]
mod kms_ilm_sse_kms_test;
crates/e2e_test/src/multipart_auth_test.rs
+608 -1718
View File
File diff suppressed because it is too large Load Diff
crates/e2e_test/src/protocols/sftp_compliance_tests.rs
+1 -1
View File
@@ -2854,7 +2854,7 @@ pub(crate) mod cmptst_30 {
result
}
#[ignore]
#[ignore = "timing-sensitive backend-pressure latency probe; run explicitly with --ignored"]
#[tokio::test]
async fn regression() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
crate::common::init_logging();
crates/e2e_test/src/replication_extension_test.rs
+54 -33
View File
@@ -2401,15 +2401,20 @@ async fn wait_for_site_replication_info<F>(
where
F: Fn(&SiteReplicationInfo) -> bool,
{
for _ in 0..40 {
// 30s to match wait_for_replication_state: the three-node site tests run
// several full rustfs processes on one runner, so peer-state propagation
// can take well over 10s under CI load.
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
loop {
let info = site_replication_info(env).await?;
if predicate(&info) {
return Ok(info);
}
if tokio::time::Instant::now() >= deadline {
return Err(format!("site replication info did not reach expected state on {}", env.address).into());
}
sleep(Duration::from_millis(250)).await;
}
Err(format!("site replication info did not reach expected state on {}", env.address).into())
}
async fn wait_for_site_replication_status<F>(
@@ -2420,15 +2425,19 @@ async fn wait_for_site_replication_status<F>(
where
F: Fn(&SRStatusInfo) -> bool,
{
for _ in 0..40 {
// Same 30s ceiling as wait_for_site_replication_info: the status probes
// fan out to every peer, so they see the same multi-process CI load.
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
loop {
let status = site_replication_status(env, query).await?;
if predicate(&status) {
return Ok(status);
}
if tokio::time::Instant::now() >= deadline {
return Err(format!("site replication status did not reach expected state on {}", env.address).into());
}
sleep(Duration::from_millis(250)).await;
}
Err(format!("site replication status did not reach expected state on {}", env.address).into())
}
async fn wait_for_replication_reset_target<F>(
@@ -4235,37 +4244,49 @@ async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestR
"tag rule with disabled delete-marker replication created a marker: {tagged_state:?}"
);
set_bucket_versioning(&source_env, source_bucket, BucketVersioningStatus::Suspended).await?;
set_bucket_versioning(&target_env_a, target_bucket_a, BucketVersioningStatus::Suspended).await?;
let null_put = source_client
// AWS S3 and MinIO both reject suspending versioning on a bucket that
// carries a replication configuration (InvalidBucketState): suspension
// would mint null versions that versioned replication can never converge.
let suspend_err = source_client
.put_bucket_versioning()
.bucket(source_bucket)
.versioning_configuration(
VersioningConfiguration::builder()
.status(BucketVersioningStatus::Suspended)
.build(),
)
.send()
.await
.expect_err("suspending versioning on a replication source must be rejected");
assert_eq!(
suspend_err.as_service_error().and_then(|error| error.code()),
Some("InvalidBucketState"),
"suspension on a replication source must fail with InvalidBucketState: {suspend_err:?}"
);
// The rejected suspension must leave the versioning + replication state
// fully intact: a fresh matched PUT still replicates with a real version.
let post_reject_put = source_client
.put_object()
.bucket(source_bucket)
.key("prefix/null.txt")
.body(ByteStream::from_static(b"null version"))
.key("prefix/after-rejected-suspend.txt")
.body(ByteStream::from_static(b"still replicating"))
.send()
.await?;
assert!(null_put.version_id().is_none(), "suspended source PUT must create a null version");
wait_for_replication_state(&target_client_a, target_bucket_a, "null version did not replicate", |state| {
state
.iter()
.any(|entry| entry.key == "prefix/null.txt" && entry.version_id == "null" && !entry.delete_marker)
})
.await?;
let null_delete = source_client
.delete_object()
.bucket(source_bucket)
.key("prefix/null.txt")
.send()
.await?;
assert!(
null_delete.version_id().is_none(),
"suspended source DELETE must create a null delete marker"
);
wait_for_replication_state(&target_client_a, target_bucket_a, "null delete marker did not replicate", |state| {
state
.iter()
.any(|entry| entry.key == "prefix/null.txt" && entry.version_id == "null" && entry.delete_marker)
})
let post_reject_version_id = post_reject_put
.version_id()
.ok_or("PUT after rejected suspension omitted version ID")?
.to_string();
wait_for_replication_state(
&target_client_a,
target_bucket_a,
"replication stopped after rejected versioning suspension",
|state| {
state
.iter()
.any(|entry| entry.key == "prefix/after-rejected-suspend.txt" && entry.version_id == post_reject_version_id)
},
)
.await?;
Ok(())
crates/ecstore/Cargo.toml
+5
View File
@@ -32,6 +32,11 @@ workspace = true
[features]
default = []
# Compiles the controlled list-objects namespace-journal chaos injector into a
# production binary (it is always available to tests). Off by default so the
# RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_* env vars cannot rewrite journal
# state in a stock build (backlog#1832).
list-chaos = []
rio-v2 = ["dep:rustfs-rio-v2"]
hotpath = [
"hotpath/hotpath",
crates/ecstore/benches/single_block_non_inline_benchmark.rs
+7 -1
View File
@@ -69,6 +69,7 @@ fn build_non_inline_writers(config: &BenchConfig) -> Vec<Option<BitrotWriterWrap
fn bench_single_block_non_inline_fast_path(c: &mut Criterion) {
let configs = vec![
BenchConfig::new(4 * 1024, 4, 2, 128 * 1024),
BenchConfig::new(16 * 1024, 4, 2, 128 * 1024),
BenchConfig::new(64 * 1024, 4, 2, 128 * 1024),
BenchConfig::new(128 * 1024, 4, 2, 128 * 1024),
];
@@ -112,7 +113,12 @@ fn bench_single_block_non_inline_fast_path(c: &mut Criterion) {
rt.block_on(async {
erasure
.clone()
.encode_single_block_non_inline(reader, &mut writers, config.data_shards)
.encode_single_block_non_inline_with_size_hint(
reader,
&mut writers,
config.data_shards,
config.payload_size,
)
.await
.expect("single block candidate benchmark");
});
crates/ecstore/src/api/mod.rs
+6 -4
View File
@@ -61,9 +61,11 @@ pub mod bucket {
delete_manual_transition_scope_admission_if_current, load_manual_transition_job_record,
load_manual_transition_job_record_with_etag, load_manual_transition_scope_admission,
manual_transition_job_lease_expired, manual_transition_scope_admission_lease_expired,
manual_transition_scope_key, persist_manual_transition_job_progress, renew_manual_transition_job_lease,
request_manual_transition_job_cancel, save_manual_transition_job_record,
save_manual_transition_job_record_if_current, save_manual_transition_scope_admission_if_absent,
manual_transition_scope_key, persist_manual_transition_job_progress,
persist_manual_transition_job_progress_if_owned, renew_manual_transition_job_lease,
renew_manual_transition_job_lease_if_owned, request_manual_transition_job_cancel,
save_manual_transition_job_record, save_manual_transition_job_record_if_current,
save_manual_transition_scope_admission_if_absent, update_manual_transition_job_record,
};
}
@@ -344,7 +346,7 @@ pub mod disk {
}
pub mod error {
pub use crate::disk::error::{BitrotErrorType, DiskError, Error, FileAccessDeniedWithContext, Result};
pub use crate::disk::error::{DiskError, Error, FileAccessDeniedWithContext, Result};
}
pub mod error_reduce {
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+412 -90
View File
@@ -27,9 +27,10 @@ use crate::bucket::lifecycle::manual_transition_job::{
ManualTransitionWorkerResult, claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
load_manual_transition_job_record, load_manual_transition_job_record_with_etag, load_manual_transition_pending_task_records,
manual_transition_job_id_from_record_object_name, manual_transition_job_lease_expired,
manual_transition_worker_result_task_key, persist_manual_transition_job_progress, reconcile_manual_transition_worker_results,
record_manual_transition_worker_result, record_manual_transition_worker_result_with_reason,
renew_manual_transition_job_lease, save_manual_transition_job_record_if_current, save_manual_transition_task_if_absent,
manual_transition_worker_result_task_key, persist_manual_transition_job_progress_if_owned,
reconcile_manual_transition_worker_results_if_owned, record_manual_transition_worker_result,
record_manual_transition_worker_result_with_reason, renew_manual_transition_job_lease_if_owned,
save_manual_transition_job_record_if_current, save_manual_transition_task_if_absent, update_manual_transition_job_record,
};
use crate::bucket::lifecycle::replication_sink;
use crate::bucket::lifecycle::replication_sink::{
@@ -78,8 +79,8 @@ use rustfs_common::metrics::{
};
use rustfs_config::{
DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_QUEUE_SEND_TIMEOUT_MS, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX,
DEFAULT_TRANSITION_WORKERS_CAP, ENV_TRANSITION_QUEUE_CAPACITY, ENV_TRANSITION_QUEUE_SEND_TIMEOUT_MS, ENV_TRANSITION_WORKERS,
ENV_TRANSITION_WORKERS_ABSOLUTE_MAX,
DEFAULT_TRANSITION_WORKERS_CAP, ENV_MAX_EXPIRY_WORKERS, ENV_TRANSITION_QUEUE_CAPACITY, ENV_TRANSITION_QUEUE_SEND_TIMEOUT_MS,
ENV_TRANSITION_WORKERS, ENV_TRANSITION_WORKERS_ABSOLUTE_MAX,
};
use rustfs_data_usage::TierStats;
use rustfs_filemeta::{
@@ -2016,18 +2017,25 @@ fn is_slow_down(err: &Error) -> bool {
matches!(err, Error::SlowDown)
}
pub async fn init_background_expiry(api: Arc<ECStore>) {
let mut workers = get_env_usize("RUSTFS_MAX_EXPIRY_WORKERS", std::cmp::min(num_cpus::get(), 16));
//globalILMConfig.getExpirationWorkers()
if let Ok(env_expiration_workers) = env::var("_RUSTFS_ILM_EXPIRATION_WORKERS")
&& let Ok(num_expirations) = env_expiration_workers.parse::<usize>()
{
workers = num_expirations;
/// Resolves the expiry worker count from the single documented knob,
/// `RUSTFS_MAX_EXPIRY_WORKERS`: a set, parsable, non-zero value wins;
/// anything else falls back to `min(cpus, 16)`. The historical
/// `_RUSTFS_ILM_EXPIRATION_WORKERS` silent override and the
/// `RUSTFS_DEFAULT_EXPIRY_WORKERS` zero-fallback were undocumented, unset in
/// every known deployment, and are removed (backlog#1832).
fn expiry_worker_count() -> usize {
let default = std::cmp::min(num_cpus::get(), 16);
match env::var(ENV_MAX_EXPIRY_WORKERS) {
Ok(value) => match value.parse::<usize>() {
Ok(workers) if workers > 0 => workers,
_ => default,
},
Err(_) => default,
}
}
if workers == 0 {
workers = get_env_usize("RUSTFS_DEFAULT_EXPIRY_WORKERS", 8);
}
pub async fn init_background_expiry(api: Arc<ECStore>) {
let workers = expiry_worker_count();
ExpiryState::resize_workers(workers, api.clone()).await;
let _ = spawn_tier_free_version_recovery_once(api.clone(), &TIER_FREE_VERSION_RECOVERY_STARTED);
@@ -2212,7 +2220,18 @@ async fn recover_manual_transition_job(
let recovery_unknown_snapshot = ManualTransitionQueueSnapshot::default();
if record.scan_completed {
let reconciled = reconcile_manual_transition_worker_results(api.clone(), job_id, recovery_unknown_snapshot).await?;
let reconciled = match reconcile_manual_transition_worker_results_if_owned(
api.clone(),
job_id,
record.lease_id,
recovery_unknown_snapshot,
)
.await
{
Ok(record) => record,
Err(Error::PreconditionFailed) => return Ok(ManualTransitionJobRecoveryOutcome::Skipped),
Err(err) => return Err(err),
};
if reconciled.is_terminal() {
release_manual_transition_recovery_admission(api, &reconciled).await;
return match reconciled.state {
@@ -2265,34 +2284,41 @@ async fn recover_manual_transition_job(
replay,
ManualTransitionPendingTaskReplay::Queued | ManualTransitionPendingTaskReplay::Deferred
) {
spawn_manual_transition_recovery_heartbeat(api, job_id);
spawn_manual_transition_recovery_heartbeat(api, job_id, recovery_lease_id);
return Ok(ManualTransitionJobRecoveryOutcome::Resumed);
}
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
if record.mark_unknown_if_worker_results_lost(recovery_unknown_snapshot)
|| record.mark_unknown_if_recovery_would_skip_pending_page(recovery_unknown_snapshot)
let mut marked_unknown = false;
let record = match update_manual_transition_job_record(api.clone(), job_id, Some(recovery_lease_id), |record| {
marked_unknown = record.mark_unknown_if_worker_results_lost(recovery_unknown_snapshot)
|| record.mark_unknown_if_recovery_would_skip_pending_page(recovery_unknown_snapshot);
marked_unknown
})
.await
{
return match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
Ok(()) => {
release_manual_transition_recovery_admission(api, &record).await;
Ok(ManualTransitionJobRecoveryOutcome::Unknown)
}
Err(Error::PreconditionFailed) => Ok(ManualTransitionJobRecoveryOutcome::Skipped),
Err(err) => Err(err),
};
Ok(record) => record,
Err(Error::PreconditionFailed) => return Ok(ManualTransitionJobRecoveryOutcome::Skipped),
Err(err) => return Err(err),
};
if marked_unknown {
release_manual_transition_recovery_admission(api, &record).await;
return Ok(ManualTransitionJobRecoveryOutcome::Unknown);
}
let mut options = record.resume_options();
options.job_id = Some(job_id);
options.cancel_check = Some(manual_transition_recovery_cancel_check(api.clone(), job_id));
options.progress_sink = Some(manual_transition_recovery_progress_sink(api.clone(), job_id));
options.progress_sink = Some(manual_transition_recovery_progress_sink(api.clone(), job_id, recovery_lease_id));
let result = enqueue_transition_for_existing_objects_scoped(api.clone(), &record.bucket, options).await;
let final_record = finalize_recovered_manual_transition_job(api.clone(), job_id, result).await?;
let final_record = match finalize_recovered_manual_transition_job(api.clone(), job_id, recovery_lease_id, result).await {
Ok(record) => record,
Err(Error::PreconditionFailed) => return Ok(ManualTransitionJobRecoveryOutcome::Skipped),
Err(err) => return Err(err),
};
if final_record.is_terminal() {
release_manual_transition_recovery_admission(api, &final_record).await;
} else {
spawn_manual_transition_recovery_heartbeat(api, job_id);
spawn_manual_transition_recovery_heartbeat(api, job_id, recovery_lease_id);
}
Ok(ManualTransitionJobRecoveryOutcome::Resumed)
}
@@ -2376,11 +2402,11 @@ fn manual_transition_recovery_cancel_check(api: Arc<ECStore>, job_id: Uuid) -> M
})
}
fn manual_transition_recovery_progress_sink(api: Arc<ECStore>, job_id: Uuid) -> ManualTransitionProgressSink {
fn manual_transition_recovery_progress_sink(api: Arc<ECStore>, job_id: Uuid, lease_id: Uuid) -> ManualTransitionProgressSink {
Arc::new(move |report| {
let api = api.clone();
Box::pin(async move {
persist_manual_transition_job_progress(api, job_id, &report, manual_transition_queue_snapshot())
persist_manual_transition_job_progress_if_owned(api, job_id, lease_id, &report, manual_transition_queue_snapshot())
.await
.map(|_| ())
})
@@ -2390,24 +2416,20 @@ fn manual_transition_recovery_progress_sink(api: Arc<ECStore>, job_id: Uuid) ->
async fn finalize_recovered_manual_transition_job(
api: Arc<ECStore>,
job_id: Uuid,
expected_lease_id: Uuid,
result: Result<ManualTransitionRunReport, Error>,
) -> Result<ManualTransitionJobRecord, Error> {
for _ in 0..4 {
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
update_manual_transition_job_record(api, job_id, Some(expected_lease_id), |record| {
if record.is_terminal() {
return Ok(record);
return false;
}
match &result {
Ok(report) => record.complete(report.clone(), manual_transition_queue_snapshot()),
Err(err) => record.fail(format!("manual transition recovery failed: {err}")),
}
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
Ok(()) => return Ok(record),
Err(Error::PreconditionFailed) => continue,
Err(err) => return Err(err),
}
}
Err(Error::PreconditionFailed)
true
})
.await
}
async fn release_manual_transition_recovery_admission(api: Arc<ECStore>, record: &ManualTransitionJobRecord) {
@@ -2426,18 +2448,20 @@ async fn release_manual_transition_recovery_admission(api: Arc<ECStore>, record:
}
}
fn spawn_manual_transition_recovery_heartbeat(api: Arc<ECStore>, job_id: Uuid) {
fn spawn_manual_transition_recovery_heartbeat(api: Arc<ECStore>, job_id: Uuid, lease_id: Uuid) {
tokio::spawn(async move {
let mut interval = tokio::time::interval(std::time::Duration::from_secs(5));
loop {
interval.tick().await;
match renew_manual_transition_job_lease(api.clone(), job_id, manual_transition_queue_snapshot()).await {
match renew_manual_transition_job_lease_if_owned(api.clone(), job_id, lease_id, manual_transition_queue_snapshot())
.await
{
Ok(record) if record.is_terminal() => {
release_manual_transition_recovery_admission(api, &record).await;
return;
}
Ok(_) => {}
Err(Error::ConfigNotFound) => return,
Err(Error::ConfigNotFound | Error::PreconditionFailed) => return,
Err(err) => {
warn!(
event = EVENT_LIFECYCLE_WORKER_STATE,
@@ -2455,23 +2479,18 @@ fn spawn_manual_transition_recovery_heartbeat(api: Arc<ECStore>, job_id: Uuid) {
}
async fn abandon_manual_transition_recovery_lease(api: Arc<ECStore>, job_id: Uuid, lease_id: Uuid) -> Result<(), Error> {
for _ in 0..4 {
let (mut record, etag) = match load_manual_transition_job_record_with_etag(api.clone(), job_id).await {
Ok(record) => record,
Err(Error::ConfigNotFound) => return Ok(()),
Err(err) => return Err(err),
};
if record.lease_id != lease_id || record.is_terminal() {
return Ok(());
match update_manual_transition_job_record(api, job_id, Some(lease_id), |record| {
if record.is_terminal() {
return false;
}
record.abandon_recovery_lease(lease_id);
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
Ok(()) => return Ok(()),
Err(Error::PreconditionFailed) => continue,
Err(err) => return Err(err),
}
true
})
.await
{
Ok(_) | Err(Error::ConfigNotFound | Error::PreconditionFailed) => Ok(()),
Err(err) => Err(err),
}
Ok(())
}
fn tier_free_version_recovery_enabled() -> bool {
@@ -5075,6 +5094,7 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc,
#[cfg(test)]
mod tests {
use super::expiry_worker_count;
use super::{
DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX,
DEFAULT_TRANSITION_WORKERS_CAP, EVENT_LIFECYCLE_EVALUATION_FAILED, EVENT_LIFECYCLE_EXPIRED_DETECTED,
@@ -5090,12 +5110,13 @@ mod tests {
lifecycle_rule_has_date_expiration, manual_transition_duration_elapsed, manual_transition_has_more_after_limit,
manual_transition_recovery_progress_sink, manual_transition_version_marker, manual_transition_worker_failure_reason,
mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate,
persist_manual_transition_job_progress, persist_manual_transition_page_checkpoint, recover_manual_transition_job,
recover_manual_transition_jobs, resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity,
resolve_transition_queue_send_timeout, resolve_transition_worker_count, resolve_transition_workers_absolute_max,
run_tier_free_version_recovery_loop, select_restore_s3_location, set_lifecycle_observability_observer,
set_recovered_free_version_enqueue_observer, should_defer_date_expiry_for_recent_config_update,
transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery,
persist_manual_transition_job_progress_if_owned, persist_manual_transition_page_checkpoint,
recover_manual_transition_job, recover_manual_transition_jobs, resolve_tier_free_version_recovery_enabled,
resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, resolve_transition_worker_count,
resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop, select_restore_s3_location,
set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer,
should_defer_date_expiry_for_recent_config_update, transitioned_cleanup_tuple, transitioned_object_delete_opts,
wait_for_tier_free_version_recovery,
};
#[cfg(feature = "test-util")]
use super::{delete_free_version_remote_object_then, encode_dir_object, get_transitioned_object_reader_with_tier_manager};
@@ -5105,18 +5126,19 @@ mod tests {
};
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::manual_transition_job::{
ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionScopeAdmission, ManualTransitionScopeAdmissionClaim,
ManualTransitionTaskRecord, ManualTransitionWorkerFailureReason, ManualTransitionWorkerResult,
ManualTransitionWorkerResultRecord, claim_manual_transition_scope_admission,
ManualTransitionJobCasBarrier, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionScopeAdmission,
ManualTransitionScopeAdmissionClaim, ManualTransitionTaskRecord, ManualTransitionWorkerFailureReason,
ManualTransitionWorkerResult, ManualTransitionWorkerResultRecord, claim_manual_transition_scope_admission,
delete_manual_transition_scope_admission_if_current, legacy_manual_transition_scope_key,
load_manual_transition_job_record, load_manual_transition_scope_admission,
load_manual_transition_job_record, load_manual_transition_job_record_with_etag, load_manual_transition_scope_admission,
load_manual_transition_scope_admission_with_etag, load_manual_transition_task_record,
manual_transition_scope_record_object_name, manual_transition_worker_result_object_name,
manual_transition_worker_result_task_key, reconcile_manual_transition_worker_results,
record_manual_transition_worker_result, record_manual_transition_worker_result_with_reason,
renew_manual_transition_job_lease, request_manual_transition_job_cancel, save_manual_transition_job_record,
save_manual_transition_scope_admission_if_absent, save_manual_transition_scope_admission_if_current,
save_manual_transition_task_if_absent, save_manual_transition_worker_result_if_absent,
renew_manual_transition_job_lease_if_owned, request_manual_transition_job_cancel, save_manual_transition_job_record,
save_manual_transition_job_record_if_current, save_manual_transition_scope_admission_if_absent,
save_manual_transition_scope_admission_if_current, save_manual_transition_task_if_absent,
save_manual_transition_worker_result_if_absent,
};
use crate::bucket::lifecycle::replication_sink::{ReplicationStatusType, VersionPurgeStatusType};
use crate::bucket::lifecycle::runtime_boundary as runtime_sources;
@@ -5156,6 +5178,7 @@ mod tests {
#[cfg(feature = "test-util")]
use http::HeaderMap;
use rustfs_common::metrics::{IlmAction, global_metrics};
use rustfs_config::ENV_MAX_EXPIRY_WORKERS;
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
use rustfs_data_usage::TierStats;
use rustfs_filemeta::{FileInfo, FileMeta};
@@ -7154,6 +7177,63 @@ mod tests {
}
}
// SAFETY: same contract as with_transition_worker_env — only used from
// `#[serial]` tests, so no concurrent reader/writer can access the process
// environment while `env::set_var`/`env::remove_var` is active.
#[allow(unsafe_code)]
fn with_expiry_worker_env<F>(value: Option<&str>, test_fn: F)
where
F: FnOnce(),
{
let original = env::var_os(ENV_MAX_EXPIRY_WORKERS);
match value {
Some(v) => unsafe {
env::set_var(ENV_MAX_EXPIRY_WORKERS, v);
},
None => unsafe {
env::remove_var(ENV_MAX_EXPIRY_WORKERS);
},
}
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(test_fn));
match original {
Some(v) => unsafe {
env::set_var(ENV_MAX_EXPIRY_WORKERS, v);
},
None => unsafe {
env::remove_var(ENV_MAX_EXPIRY_WORKERS);
},
}
if let Err(e) = result {
std::panic::resume_unwind(e);
}
}
/// backlog#1832: the single expiry knob must resolve all four env states
/// (unset / zero / valid / garbage); the removed `_RUSTFS_ILM_EXPIRATION_WORKERS`
/// override and `RUSTFS_DEFAULT_EXPIRY_WORKERS` fallback must stay gone.
#[test]
#[serial]
fn expiry_worker_count_resolves_all_env_states() {
let default = std::cmp::min(num_cpus::get(), 16);
with_expiry_worker_env(None, || {
assert_eq!(expiry_worker_count(), default, "unset env must fall back to min(cpus, 16)");
});
with_expiry_worker_env(Some("0"), || {
assert_eq!(expiry_worker_count(), default, "zero must fall back instead of spawning zero workers");
});
with_expiry_worker_env(Some("4"), || {
assert_eq!(expiry_worker_count(), 4, "a valid positive value must win");
});
with_expiry_worker_env(Some("not-a-number"), || {
assert_eq!(expiry_worker_count(), default, "garbage must fall back to the default");
});
}
// SAFETY: this helper is only used from `#[serial]` tests and those tests run under a
// single-thread runtime (`worker_threads = 1`), so no concurrent reader/writer can access
// process environment while `env::set_var`/`env::remove_var` is active.
@@ -8554,9 +8634,10 @@ mod tests {
..Default::default()
};
let persisted = persist_manual_transition_job_progress(ecstore.clone(), job_id, &report, queue_snapshot)
.await
.expect("page checkpoint should persist to the job record");
let persisted =
persist_manual_transition_job_progress_if_owned(ecstore.clone(), job_id, record.lease_id, &report, queue_snapshot)
.await
.expect("page checkpoint should persist to the job record");
assert_eq!(persisted.state, ManualTransitionJobState::Running);
assert_eq!(persisted.report.scanned, 1000);
@@ -8575,6 +8656,232 @@ mod tests {
assert_eq!(admission.updated_at_unix_nanos, loaded.updated_at_unix_nanos);
}
#[tokio::test]
#[serial]
async fn manual_transition_progress_retries_heartbeat_cas_without_losing_checkpoint() {
let (_paths, ecstore) = setup_test_env().await;
let job_id = Uuid::new_v4();
let options = ManualTransitionRunOptions {
prefix: "logs/".to_string(),
..Default::default()
};
let record = ManualTransitionJobRecord::new(job_id, "manual-progress-cas-bucket", &options, "owner-a");
save_manual_transition_job_record(ecstore.clone(), &record)
.await
.expect("running job record should save");
save_manual_transition_scope_admission_if_absent(ecstore.clone(), &ManualTransitionScopeAdmission::from_job(&record))
.await
.expect("running scope admission should save");
let lease_id = record.lease_id;
let barrier = ManualTransitionJobCasBarrier::install(job_id);
let progress_store = ecstore.clone();
let progress = tokio::spawn(async move {
persist_manual_transition_job_progress_if_owned(
progress_store,
job_id,
lease_id,
&ManualTransitionRunReport {
bucket: "manual-progress-cas-bucket".to_string(),
prefix: "logs/".to_string(),
scanned: 1000,
eligible: 900,
enqueued: 800,
continuation_token: Some("opaque-page-cursor".to_string()),
..Default::default()
},
ManualTransitionQueueSnapshot {
queued: 7,
active: 3,
..Default::default()
},
)
.await
});
barrier.wait_until_paused().await;
let heartbeat = renew_manual_transition_job_lease_if_owned(
ecstore.clone(),
job_id,
lease_id,
ManualTransitionQueueSnapshot {
queued: 2,
active: 1,
..Default::default()
},
)
.await
.expect("heartbeat should win the first CAS write");
barrier.release();
let checkpointed = progress
.await
.expect("progress task should join")
.expect("progress should retry its stale ETag");
assert_eq!(checkpointed.lease_id, heartbeat.lease_id);
assert_eq!(checkpointed.report.scanned, 1000);
assert_eq!(checkpointed.report.eligible, 900);
assert_eq!(checkpointed.report.enqueued, 800);
assert_eq!(checkpointed.report.continuation_token.as_deref(), Some("opaque-page-cursor"));
assert_eq!(checkpointed.queue_snapshot.queued, 7);
assert_eq!(checkpointed.queue_snapshot.active, 3);
}
#[tokio::test]
#[serial]
async fn manual_transition_progress_rejects_stale_recovery_lease() {
let (_paths, ecstore) = setup_test_env().await;
let job_id = Uuid::new_v4();
let record = ManualTransitionJobRecord::new(
job_id,
"manual-progress-stale-lease-bucket",
&ManualTransitionRunOptions::default(),
"owner-a",
);
let stale_lease_id = record.lease_id;
save_manual_transition_job_record(ecstore.clone(), &record)
.await
.expect("running job record should save");
let (mut recovered, etag) = load_manual_transition_job_record_with_etag(ecstore.clone(), job_id)
.await
.expect("running job record should load");
recovered.lease_id = Uuid::new_v4();
recovered.owner_id = "owner-b".to_string();
save_manual_transition_job_record_if_current(ecstore.clone(), &recovered, &etag)
.await
.expect("recovery owner should replace the lease");
let error = persist_manual_transition_job_progress_if_owned(
ecstore.clone(),
job_id,
stale_lease_id,
&ManualTransitionRunReport {
scanned: 1000,
continuation_token: Some("stale-owner-cursor".to_string()),
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
)
.await
.expect_err("the stale owner must not update the recovered job");
let heartbeat_error = renew_manual_transition_job_lease_if_owned(
ecstore.clone(),
job_id,
stale_lease_id,
ManualTransitionQueueSnapshot::default(),
)
.await
.expect_err("the stale owner must not renew the recovered job");
assert_eq!(error, Error::PreconditionFailed);
assert_eq!(heartbeat_error, Error::PreconditionFailed);
let loaded = load_manual_transition_job_record(ecstore, job_id)
.await
.expect("recovered job record should load");
assert_eq!(loaded.lease_id, recovered.lease_id);
assert_eq!(loaded.owner_id, "owner-b");
assert_eq!(loaded.report.scanned, 0);
assert!(loaded.report.continuation_token.is_none());
}
#[tokio::test]
#[serial]
async fn manual_transition_reconcile_rejects_lease_takeover_during_cas() {
let (_paths, ecstore) = setup_test_env().await;
let job_id = Uuid::new_v4();
let bucket = format!("manual-reconcile-lease-race-{}", job_id.simple());
let mut record = ManualTransitionJobRecord::new(job_id, &bucket, &ManualTransitionRunOptions::default(), "owner-a");
record.scan_completed = true;
let stale_lease_id = record.lease_id;
save_manual_transition_job_record(ecstore.clone(), &record)
.await
.expect("running job record should save");
let task_key = manual_transition_worker_result_task_key(&bucket, "logs/a", None);
let task = ManualTransitionTaskRecord::new(job_id, &task_key, &bucket, "logs/a", None, "WARM");
assert!(
save_manual_transition_task_if_absent(ecstore.clone(), &task)
.await
.expect("task journal marker should save")
);
let barrier = ManualTransitionJobCasBarrier::install(job_id);
let heartbeat_store = ecstore.clone();
let heartbeat = tokio::spawn(async move {
renew_manual_transition_job_lease_if_owned(
heartbeat_store,
job_id,
stale_lease_id,
ManualTransitionQueueSnapshot::default(),
)
.await
});
barrier.wait_until_paused().await;
let (mut recovered, etag) = load_manual_transition_job_record_with_etag(ecstore.clone(), job_id)
.await
.expect("running job record should load during reconciliation");
recovered.lease_id = Uuid::new_v4();
recovered.owner_id = "owner-b".to_string();
save_manual_transition_job_record_if_current(ecstore.clone(), &recovered, &etag)
.await
.expect("recovery owner should replace the lease");
barrier.release();
let error = heartbeat
.await
.expect("heartbeat task should join")
.expect_err("stale reconciliation must reject the recovery lease");
assert_eq!(error, Error::PreconditionFailed);
let loaded = load_manual_transition_job_record(ecstore, job_id)
.await
.expect("recovered job record should load");
assert_eq!(loaded.lease_id, recovered.lease_id);
assert_eq!(loaded.owner_id, "owner-b");
assert_eq!(loaded.state, ManualTransitionJobState::Running);
assert_eq!(loaded.report.enqueued, 0);
}
#[tokio::test]
#[serial]
async fn manual_transition_progress_does_not_regress_newer_admission_lease() {
let (_paths, ecstore) = setup_test_env().await;
let job_id = Uuid::new_v4();
let record = ManualTransitionJobRecord::new(
job_id,
"manual-progress-admission-order-bucket",
&ManualTransitionRunOptions::default(),
"owner-a",
);
save_manual_transition_job_record(ecstore.clone(), &record)
.await
.expect("running job record should save");
let mut newer_admission = ManualTransitionScopeAdmission::from_job(&record);
newer_admission.lease_expires_at_unix_nanos = newer_admission.lease_expires_at_unix_nanos.saturating_add(60_000_000_000);
newer_admission.updated_at_unix_nanos = newer_admission.updated_at_unix_nanos.saturating_add(60_000_000_000);
save_manual_transition_scope_admission_if_absent(ecstore.clone(), &newer_admission)
.await
.expect("newer scope admission should save");
persist_manual_transition_job_progress_if_owned(
ecstore.clone(),
job_id,
record.lease_id,
&ManualTransitionRunReport {
scanned: 1000,
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
)
.await
.expect("progress should preserve the newer admission lease");
let admission = load_manual_transition_scope_admission(ecstore, &record.scope_key)
.await
.expect("scope admission should load");
assert_eq!(admission.lease_expires_at_unix_nanos, newer_admission.lease_expires_at_unix_nanos);
assert_eq!(admission.updated_at_unix_nanos, newer_admission.updated_at_unix_nanos);
}
#[tokio::test]
async fn manual_transition_page_checkpoint_persists_resume_cursor() {
let observed = Arc::new(StdMutex::new(Vec::new()));
@@ -8639,7 +8946,7 @@ mod tests {
.await
.expect("expired scope admission should save");
let checkpoint_options = ManualTransitionRunOptions {
progress_sink: Some(manual_transition_recovery_progress_sink(ecstore.clone(), job_id)),
progress_sink: Some(manual_transition_recovery_progress_sink(ecstore.clone(), job_id, record.lease_id)),
..options
};
let report = ManualTransitionRunReport {
@@ -8728,7 +9035,7 @@ mod tests {
prefix: prefix.to_string(),
tier: Some("WARM".to_string()),
dry_run: true,
progress_sink: Some(manual_transition_recovery_progress_sink(ecstore.clone(), job_id)),
progress_sink: Some(manual_transition_recovery_progress_sink(ecstore.clone(), job_id, record.lease_id)),
..Default::default()
};
let final_report = enqueue_transition_for_existing_objects_scoped(ecstore.clone(), &bucket, production_path_options)
@@ -9428,9 +9735,14 @@ mod tests {
"new worker result marker must be created"
);
let renewed = renew_manual_transition_job_lease(ecstore.clone(), job_id, ManualTransitionQueueSnapshot::default())
.await
.expect("heartbeat should reconcile marker before unknown fallback");
let renewed = renew_manual_transition_job_lease_if_owned(
ecstore.clone(),
job_id,
record.lease_id,
ManualTransitionQueueSnapshot::default(),
)
.await
.expect("heartbeat should reconcile marker before unknown fallback");
assert_eq!(renewed.state, ManualTransitionJobState::Completed);
assert_eq!(renewed.report.transition_completed, 1);
@@ -9473,9 +9785,14 @@ mod tests {
"new worker result marker must be created"
);
let renewed = renew_manual_transition_job_lease(ecstore.clone(), job_id, ManualTransitionQueueSnapshot::default())
.await
.expect("heartbeat should reconcile task and result journals");
let renewed = renew_manual_transition_job_lease_if_owned(
ecstore.clone(),
job_id,
record.lease_id,
ManualTransitionQueueSnapshot::default(),
)
.await
.expect("heartbeat should reconcile task and result journals");
assert_eq!(renewed.state, ManualTransitionJobState::Completed);
assert_eq!(renewed.report.enqueued, 1);
@@ -9790,9 +10107,10 @@ mod tests {
.await
.expect("running scope admission should save");
let checkpointed = persist_manual_transition_job_progress(
let checkpointed = persist_manual_transition_job_progress_if_owned(
ecstore.clone(),
job_id,
record.lease_id,
&ManualTransitionRunReport {
bucket: bucket.to_string(),
prefix: "logs/".to_string(),
@@ -9881,7 +10199,7 @@ mod tests {
compensation_running: 1,
};
let renewed = renew_manual_transition_job_lease(ecstore.clone(), job_id, queue_snapshot)
let renewed = renew_manual_transition_job_lease_if_owned(ecstore.clone(), job_id, record.lease_id, queue_snapshot)
.await
.expect("running job heartbeat should persist queue pressure status");
@@ -9932,9 +10250,14 @@ mod tests {
.await
.expect("running job admission should save");
let renewed = renew_manual_transition_job_lease(ecstore.clone(), job_id, ManualTransitionQueueSnapshot::default())
.await
.expect("lost worker result should persist unknown state");
let renewed = renew_manual_transition_job_lease_if_owned(
ecstore.clone(),
job_id,
record.lease_id,
ManualTransitionQueueSnapshot::default(),
)
.await
.expect("lost worker result should persist unknown state");
assert_eq!(renewed.state, ManualTransitionJobState::Unknown);
assert!(renewed.completed_at_unix_nanos.is_some());
@@ -11524,7 +11847,6 @@ mod tests {
}
#[tokio::test]
#[ignore = "requires isolated global object layer state"]
#[serial]
async fn ecstore_new_succeeds_on_fresh_local_volumes() {
let test_base_dir = format!("/tmp/rustfs_ecstore_empty_boot_{}", Uuid::new_v4());
crates/ecstore/src/bucket/lifecycle/config_boundary.rs
+15
View File
@@ -86,6 +86,21 @@ where
com::save_config_with_opts(api, file, data, opts).await
}
pub(crate) async fn save_config_with_opts_quiet<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
com::save_config_with_opts_quiet(api, file, data, opts).await
}
pub(crate) async fn delete_config<S>(api: Arc<S>, file: &str) -> Result<()>
where
S: ObjectOperations<
crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs
+332 -100
View File
@@ -45,6 +45,104 @@ const MANUAL_TRANSITION_JOB_LEASE_SECONDS: i128 = 60;
const MANUAL_TRANSITION_LEGACY_SCOPE_SCAN_LIMIT: i32 = 1000;
const MANUAL_TRANSITION_TASK_SCAN_LIMIT: i32 = 1000;
const MANUAL_TRANSITION_WORKER_RESULT_SCAN_LIMIT: i32 = 1000;
const MANUAL_TRANSITION_JOB_CAS_RETRIES: usize = 4;
#[cfg(test)]
struct ManualTransitionJobCasBarrierState {
job_id: Uuid,
paused: std::sync::atomic::AtomicBool,
arrived: tokio::sync::Notify,
release: tokio::sync::Semaphore,
}
#[cfg(test)]
pub(crate) struct ManualTransitionJobCasBarrier {
state: Arc<ManualTransitionJobCasBarrierState>,
}
#[cfg(test)]
static MANUAL_TRANSITION_JOB_CAS_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<ManualTransitionJobCasBarrierState>>>> =
std::sync::OnceLock::new();
#[cfg(test)]
impl ManualTransitionJobCasBarrier {
pub(crate) fn install(job_id: Uuid) -> Self {
let state = Arc::new(ManualTransitionJobCasBarrierState {
job_id,
paused: std::sync::atomic::AtomicBool::new(false),
arrived: tokio::sync::Notify::new(),
release: tokio::sync::Semaphore::new(0),
});
let mut slot = MANUAL_TRANSITION_JOB_CAS_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("manual transition progress CAS barrier mutex should not poison");
assert!(
slot.is_none(),
"manual transition job CAS barrier must be installed by one test at a time"
);
*slot = Some(Arc::clone(&state));
drop(slot);
Self { state }
}
pub(crate) async fn wait_until_paused(&self) {
tokio::time::timeout(std::time::Duration::from_secs(30), async {
loop {
let arrived = self.state.arrived.notified();
if self.state.paused.load(std::sync::atomic::Ordering::Acquire) {
return;
}
arrived.await;
}
})
.await
.expect("manual transition job update should reach the deterministic CAS barrier");
}
pub(crate) fn release(&self) {
self.state.release.add_permits(1);
}
}
#[cfg(test)]
impl Drop for ManualTransitionJobCasBarrier {
fn drop(&mut self) {
self.release();
let mut slot = MANUAL_TRANSITION_JOB_CAS_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("manual transition progress CAS barrier mutex should not poison");
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
*slot = None;
}
}
}
#[cfg(test)]
async fn pause_manual_transition_job_before_first_cas(job_id: Uuid) {
let barrier = MANUAL_TRANSITION_JOB_CAS_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("manual transition progress CAS barrier mutex should not poison")
.as_ref()
.filter(|barrier| barrier.job_id == job_id)
.cloned();
if let Some(barrier) = barrier
&& barrier
.paused
.compare_exchange(false, true, std::sync::atomic::Ordering::AcqRel, std::sync::atomic::Ordering::Acquire)
.is_ok()
{
barrier.arrived.notify_one();
barrier
.release
.acquire()
.await
.expect("manual transition job CAS barrier should remain open")
.forget();
}
}
fn is_false(value: &bool) -> bool {
!*value
@@ -148,7 +246,6 @@ impl ManualTransitionJobRecord {
pub fn fail(&mut self, error: impl Into<String>) {
self.state = ManualTransitionJobState::Failed;
self.report.tier_failure = self.report.tier_failure.saturating_add(1);
self.error = Some(error.into());
self.mark_updated_terminal();
}
@@ -1040,7 +1137,7 @@ pub async fn save_manual_transition_job_record_if_current(
}
let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
let data = job.encode().map_err(manual_transition_job_store_error)?;
config_boundary::save_config_with_opts(
config_boundary::save_config_with_opts_quiet(
api,
&object,
data,
@@ -1056,6 +1153,54 @@ pub async fn save_manual_transition_job_record_if_current(
.await
}
/// Applies a job-record mutation with optimistic concurrency control.
///
/// The mutation returns whether the record needs to be persisted. When a lease
/// is supplied, ownership is checked again after every conflicting write.
pub async fn update_manual_transition_job_record<F>(
api: Arc<ECStore>,
job_id: Uuid,
expected_lease_id: Option<Uuid>,
update: F,
) -> EcstoreResult<ManualTransitionJobRecord>
where
F: FnMut(&mut ManualTransitionJobRecord) -> bool,
{
update_manual_transition_job_record_from(api, job_id, expected_lease_id, None, update).await
}
async fn update_manual_transition_job_record_from<F>(
api: Arc<ECStore>,
job_id: Uuid,
expected_lease_id: Option<Uuid>,
mut current: Option<(ManualTransitionJobRecord, String)>,
mut update: F,
) -> EcstoreResult<ManualTransitionJobRecord>
where
F: FnMut(&mut ManualTransitionJobRecord) -> bool,
{
for _ in 0..MANUAL_TRANSITION_JOB_CAS_RETRIES {
let (mut record, etag) = match current.take() {
Some(current) => current,
None => load_manual_transition_job_record_with_etag(api.clone(), job_id).await?,
};
if expected_lease_id.is_some_and(|lease_id| record.lease_id != lease_id) {
return Err(Error::PreconditionFailed);
}
if !update(&mut record) {
return Ok(record);
}
#[cfg(test)]
pause_manual_transition_job_before_first_cas(job_id).await;
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
Ok(()) => return Ok(record),
Err(Error::PreconditionFailed) => continue,
Err(err) => return Err(err),
}
}
Err(Error::PreconditionFailed)
}
pub(crate) async fn save_manual_transition_worker_result_if_absent(
api: Arc<ECStore>,
record: &ManualTransitionWorkerResultRecord,
@@ -1314,99 +1459,113 @@ pub async fn reconcile_manual_transition_worker_results(
api: Arc<ECStore>,
job_id: Uuid,
queue_snapshot: ManualTransitionQueueSnapshot,
) -> EcstoreResult<ManualTransitionJobRecord> {
reconcile_manual_transition_worker_results_inner(api, job_id, None, queue_snapshot, false).await
}
pub(crate) async fn reconcile_manual_transition_worker_results_if_owned(
api: Arc<ECStore>,
job_id: Uuid,
expected_lease_id: Uuid,
queue_snapshot: ManualTransitionQueueSnapshot,
) -> EcstoreResult<ManualTransitionJobRecord> {
reconcile_manual_transition_worker_results_inner(api, job_id, Some(expected_lease_id), queue_snapshot, false).await
}
async fn reconcile_manual_transition_worker_results_inner(
api: Arc<ECStore>,
job_id: Uuid,
expected_lease_id: Option<Uuid>,
queue_snapshot: ManualTransitionQueueSnapshot,
mark_missing_results_unknown: bool,
) -> EcstoreResult<ManualTransitionJobRecord> {
let task_stats = match scan_manual_transition_task_journal(api.clone(), job_id).await? {
ManualTransitionTaskJournal::Stats(stats) => stats,
ManualTransitionTaskJournal::Corrupt(error) => {
return mark_manual_transition_job_unknown_for_task_journal_error(api, job_id, error, queue_snapshot).await;
return mark_manual_transition_job_unknown_for_task_journal_error(
api,
job_id,
expected_lease_id,
error,
queue_snapshot,
)
.await;
}
};
let stats = match scan_manual_transition_worker_result_journal(api.clone(), job_id).await? {
ManualTransitionWorkerResultJournal::Stats(stats) => stats,
ManualTransitionWorkerResultJournal::Corrupt(error) => {
return mark_manual_transition_job_unknown_for_worker_result_journal_error(api, job_id, error, queue_snapshot).await;
return mark_manual_transition_job_unknown_for_worker_result_journal_error(
api,
job_id,
expected_lease_id,
error,
queue_snapshot,
)
.await;
}
};
for _ in 0..4 {
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
let changed = record.apply_worker_result_counts(
let mut changed = false;
let record = update_manual_transition_job_record(api.clone(), job_id, expected_lease_id, |record| {
let counts_changed = record.apply_worker_result_counts(
stats.stats.completed,
stats.stats.failed,
&stats.stats.tier_failure_by_reason,
task_stats.queued,
queue_snapshot,
);
if !changed {
return Ok(record);
}
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
Ok(()) => {
if record.is_terminal() {
delete_manual_transition_scope_admission_if_current(
api.clone(),
&record.scope_key,
record.job_id,
record.lease_id,
)
.await?;
} else {
renew_manual_transition_scope_admission_from_job(api, &record).await?;
}
return Ok(record);
}
Err(Error::PreconditionFailed) => continue,
Err(err) => return Err(err),
}
let became_unknown = mark_missing_results_unknown && record.mark_unknown_if_worker_results_lost(queue_snapshot);
changed = counts_changed || became_unknown;
changed
})
.await?;
if !changed {
return Ok(record);
}
Err(Error::PreconditionFailed)
if record.is_terminal() {
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
} else {
renew_manual_transition_scope_admission_from_job(api, &record).await?;
}
Ok(record)
}
async fn mark_manual_transition_job_unknown_for_task_journal_error(
api: Arc<ECStore>,
job_id: Uuid,
expected_lease_id: Option<Uuid>,
error: String,
queue_snapshot: ManualTransitionQueueSnapshot,
) -> EcstoreResult<ManualTransitionJobRecord> {
for _ in 0..4 {
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
if !record.mark_unknown_for_task_journal_error(error.clone(), queue_snapshot) {
return Ok(record);
}
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
Ok(()) => {
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id)
.await?;
return Ok(record);
}
Err(Error::PreconditionFailed) => continue,
Err(err) => return Err(err),
}
let mut changed = false;
let record = update_manual_transition_job_record(api.clone(), job_id, expected_lease_id, |record| {
changed = record.mark_unknown_for_task_journal_error(error.clone(), queue_snapshot);
changed
})
.await?;
if changed && record.is_terminal() {
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
}
Err(Error::PreconditionFailed)
Ok(record)
}
async fn mark_manual_transition_job_unknown_for_worker_result_journal_error(
api: Arc<ECStore>,
job_id: Uuid,
expected_lease_id: Option<Uuid>,
error: String,
queue_snapshot: ManualTransitionQueueSnapshot,
) -> EcstoreResult<ManualTransitionJobRecord> {
for _ in 0..4 {
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
if !record.mark_unknown_for_worker_result_journal_error(error.clone(), queue_snapshot) {
return Ok(record);
}
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
Ok(()) => {
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id)
.await?;
return Ok(record);
}
Err(Error::PreconditionFailed) => continue,
Err(err) => return Err(err),
}
let mut changed = false;
let record = update_manual_transition_job_record(api.clone(), job_id, expected_lease_id, |record| {
changed = record.mark_unknown_for_worker_result_journal_error(error.clone(), queue_snapshot);
changed
})
.await?;
if changed && record.is_terminal() {
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
}
Err(Error::PreconditionFailed)
Ok(record)
}
pub async fn save_manual_transition_scope_admission_if_absent(
@@ -1603,19 +1762,14 @@ async fn find_active_legacy_manual_transition_scope_conflict(
}
pub async fn request_manual_transition_job_cancel(api: Arc<ECStore>, job_id: Uuid) -> EcstoreResult<ManualTransitionJobRecord> {
for _ in 0..4 {
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
update_manual_transition_job_record(api, job_id, None, |record| {
if record.is_terminal() || record.cancel_requested {
return Ok(record);
return false;
}
record.mark_cancel_requested();
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
Ok(()) => return Ok(record),
Err(Error::PreconditionFailed) => continue,
Err(err) => return Err(err),
}
}
Err(Error::PreconditionFailed)
true
})
.await
}
pub async fn persist_manual_transition_job_progress(
@@ -1624,10 +1778,39 @@ pub async fn persist_manual_transition_job_progress(
report: &ManualTransitionRunReport,
queue_snapshot: ManualTransitionQueueSnapshot,
) -> EcstoreResult<ManualTransitionJobRecord> {
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
record.update_running_progress(report.clone(), queue_snapshot);
save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await?;
renew_manual_transition_scope_admission_from_job(api, &record).await?;
let current = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
persist_manual_transition_job_progress_inner(api, job_id, current.0.lease_id, Some(current), report, queue_snapshot).await
}
pub async fn persist_manual_transition_job_progress_if_owned(
api: Arc<ECStore>,
job_id: Uuid,
expected_lease_id: Uuid,
report: &ManualTransitionRunReport,
queue_snapshot: ManualTransitionQueueSnapshot,
) -> EcstoreResult<ManualTransitionJobRecord> {
persist_manual_transition_job_progress_inner(api, job_id, expected_lease_id, None, report, queue_snapshot).await
}
async fn persist_manual_transition_job_progress_inner(
api: Arc<ECStore>,
job_id: Uuid,
expected_lease_id: Uuid,
current: Option<(ManualTransitionJobRecord, String)>,
report: &ManualTransitionRunReport,
queue_snapshot: ManualTransitionQueueSnapshot,
) -> EcstoreResult<ManualTransitionJobRecord> {
let record = update_manual_transition_job_record_from(api.clone(), job_id, Some(expected_lease_id), current, |record| {
if record.state != ManualTransitionJobState::Running {
return false;
}
record.update_running_progress(report.clone(), queue_snapshot);
true
})
.await?;
if record.state == ManualTransitionJobState::Running {
renew_manual_transition_scope_admission_from_job(api, &record).await?;
}
Ok(record)
}
@@ -1661,25 +1844,58 @@ pub async fn renew_manual_transition_job_lease(
job_id: Uuid,
queue_snapshot: ManualTransitionQueueSnapshot,
) -> EcstoreResult<ManualTransitionJobRecord> {
let (mut record, mut etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
if record.state == ManualTransitionJobState::Running {
if record.scan_completed && queue_snapshot.queued == 0 && queue_snapshot.active == 0 {
record = reconcile_manual_transition_worker_results(api.clone(), job_id, queue_snapshot).await?;
if record.is_terminal() || !record.report.worker_transition_pending() {
return Ok(record);
let current = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
renew_manual_transition_job_lease_inner(api, job_id, current.0.lease_id, Some(current), queue_snapshot).await
}
pub async fn renew_manual_transition_job_lease_if_owned(
api: Arc<ECStore>,
job_id: Uuid,
expected_lease_id: Uuid,
queue_snapshot: ManualTransitionQueueSnapshot,
) -> EcstoreResult<ManualTransitionJobRecord> {
renew_manual_transition_job_lease_inner(api, job_id, expected_lease_id, None, queue_snapshot).await
}
async fn renew_manual_transition_job_lease_inner(
api: Arc<ECStore>,
job_id: Uuid,
expected_lease_id: Uuid,
current: Option<(ManualTransitionJobRecord, String)>,
queue_snapshot: ManualTransitionQueueSnapshot,
) -> EcstoreResult<ManualTransitionJobRecord> {
let (current, current_etag) = match current {
Some(current) => current,
None => load_manual_transition_job_record_with_etag(api.clone(), job_id).await?,
};
if current.lease_id != expected_lease_id {
return Err(Error::PreconditionFailed);
}
if current.state != ManualTransitionJobState::Running {
return Ok(current);
}
if current.scan_completed && queue_snapshot.queued == 0 && queue_snapshot.active == 0 {
return reconcile_manual_transition_worker_results_inner(api, job_id, Some(expected_lease_id), queue_snapshot, true)
.await;
}
let record = update_manual_transition_job_record_from(
api.clone(),
job_id,
Some(expected_lease_id),
Some((current, current_etag)),
|record| {
if record.state != ManualTransitionJobState::Running {
return false;
}
(record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
}
let became_terminal = record.mark_unknown_if_worker_results_lost(queue_snapshot);
if !became_terminal {
record.renew_lease(queue_snapshot);
}
save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await?;
if became_terminal {
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
} else {
renew_manual_transition_scope_admission_from_job(api, &record).await?;
}
true
},
)
.await?;
if record.is_terminal() {
delete_manual_transition_scope_admission_if_current(api, &record.scope_key, record.job_id, record.lease_id).await?;
} else if record.state == ManualTransitionJobState::Running {
renew_manual_transition_scope_admission_from_job(api, &record).await?;
}
Ok(record)
}
@@ -1688,15 +1904,31 @@ async fn renew_manual_transition_scope_admission_from_job(
api: Arc<ECStore>,
record: &ManualTransitionJobRecord,
) -> EcstoreResult<()> {
if let Ok((admission, admission_etag)) =
load_manual_transition_scope_admission_with_etag(api.clone(), &record.scope_key).await
&& admission.job_id == record.job_id
&& admission.lease_id == record.lease_id
{
let renewed_admission = ManualTransitionScopeAdmission::from_job(record);
save_manual_transition_scope_admission_if_current(api, &renewed_admission, &admission_etag).await?;
for _ in 0..MANUAL_TRANSITION_JOB_CAS_RETRIES {
let (admission, admission_etag) =
match load_manual_transition_scope_admission_with_etag(api.clone(), &record.scope_key).await {
Ok(admission) => admission,
Err(Error::ConfigNotFound) => return Ok(()),
Err(err) => return Err(err),
};
if admission.job_id != record.job_id || admission.lease_id != record.lease_id {
return Err(Error::PreconditionFailed);
}
let mut renewed_admission = ManualTransitionScopeAdmission::from_job(record);
renewed_admission.lease_expires_at_unix_nanos = renewed_admission
.lease_expires_at_unix_nanos
.max(admission.lease_expires_at_unix_nanos);
renewed_admission.updated_at_unix_nanos = renewed_admission.updated_at_unix_nanos.max(admission.updated_at_unix_nanos);
if renewed_admission == admission {
return Ok(());
}
match save_manual_transition_scope_admission_if_current(api.clone(), &renewed_admission, &admission_etag).await {
Ok(()) => return Ok(()),
Err(Error::PreconditionFailed) => continue,
Err(err) => return Err(err),
}
}
Ok(())
Err(Error::PreconditionFailed)
}
pub async fn delete_manual_transition_scope_admission_if_current(
@@ -2386,14 +2618,14 @@ mod tests {
}
#[test]
fn manual_transition_job_record_failure_counts_tier_failure() {
fn manual_transition_job_record_control_plane_failure_does_not_count_tier_failure() {
let options = ManualTransitionRunOptions::default();
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
record.fail("missing tier");
assert_eq!(record.state, ManualTransitionJobState::Failed);
assert_eq!(record.report.tier_failure, 1);
assert_eq!(record.report.tier_failure, 0);
assert_eq!(record.error.as_deref(), Some("missing tier"));
}
crates/ecstore/src/bucket/metadata.rs
+2 -2
View File
@@ -1311,7 +1311,7 @@ mod test {
assert!(bm.object_locking(), "object lock active via parsed config");
}
/// backlog#580: KNOWN GAP (weisd 2026-03-06 "inline_data 前缀不同"). RustFS's
/// backlog#580: KNOWN GAP (flagged 2026-03-06: "inline_data 前缀不同"). RustFS's
/// inline-data extraction does not yet recover the object body from a
/// MinIO-written bucket-metadata object: `into_fileinfo(read_data=true).data`
/// returns bytes that are not the `.metadata.bin` blob (no `format|version`
@@ -1319,7 +1319,7 @@ mod test {
/// inline-data framing is handled on the read path.
/// backlog#580: prove RustFS reads a MinIO-written **inlined** bucket-metadata
/// object end-to-end. MinIO stores inline data as `[bitrot hash][object body]`
/// (the "`inline_data` 前缀不同" that weisd flagged on 2026-03-06 is that
/// (the "`inline_data` 前缀不同" gap flagged on 2026-03-06 is that
/// bitrot prefix, not a format incompatibility). Running the raw inline shard
/// through RustFS's `BitrotReader` with the default `HighwayHash256S` must
/// verify the checksum and yield the exact `.metadata.bin` blob.
crates/ecstore/src/client/api_bucket_policy.rs
-171
View File
@@ -1,171 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use http::{HeaderMap, StatusCode};
use http_body_util::BodyExt;
use hyper::body::Body;
use hyper::body::Bytes;
use std::collections::HashMap;
use crate::client::{
api_error_response::http_resp_to_error_response,
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
impl TransitionClient {
pub async fn set_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
if policy == "" {
return self.remove_bucket_policy(bucket_name).await;
}
self.put_bucket_policy(bucket_name, policy).await
}
pub async fn put_bucket_policy(&self, bucket_name: &str, policy: &str) -> Result<(), std::io::Error> {
let mut url_values = HashMap::new();
url_values.insert("policy".to_string(), "".to_string());
let mut req_metadata = RequestMetadata {
bucket_name: bucket_name.to_string(),
query_values: url_values,
content_body: ReaderImpl::Body(Bytes::from(policy.as_bytes().to_vec())),
content_length: policy.len() as i64,
object_name: "".to_string(),
custom_header: HeaderMap::new(),
content_md5_base64: "".to_string(),
content_sha256_hex: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
};
let resp = self.execute_method(http::Method::PUT, &mut req_metadata).await?;
//defer closeResponse(resp)
let resp_status = resp.status();
let h = resp.headers().clone();
//if resp != nil {
if resp_status != StatusCode::NO_CONTENT && resp.status() != StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(
resp_status,
&h,
vec![],
bucket_name,
"",
)));
}
//}
Ok(())
}
pub async fn remove_bucket_policy(&self, bucket_name: &str) -> Result<(), std::io::Error> {
let mut url_values = HashMap::new();
url_values.insert("policy".to_string(), "".to_string());
let resp = self
.execute_method(
http::Method::DELETE,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(),
query_values: url_values,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
object_name: "".to_string(),
custom_header: HeaderMap::new(),
content_body: ReaderImpl::Body(Bytes::new()),
content_length: 0,
content_md5_base64: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
},
)
.await?;
//defer closeResponse(resp)
let resp_status = resp.status();
let h = resp.headers().clone();
if resp_status != StatusCode::NO_CONTENT {
return Err(std::io::Error::other(http_resp_to_error_response(
resp_status,
&h,
vec![],
bucket_name,
"",
)));
}
Ok(())
}
pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result<String, std::io::Error> {
let bucket_policy = self.get_bucket_policy_inner(bucket_name).await?;
Ok(bucket_policy)
}
pub async fn get_bucket_policy_inner(&self, bucket_name: &str) -> Result<String, std::io::Error> {
let mut url_values = HashMap::new();
url_values.insert("policy".to_string(), "".to_string());
let resp = self
.execute_method(
http::Method::GET,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(),
query_values: url_values,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
object_name: "".to_string(),
custom_header: HeaderMap::new(),
content_body: ReaderImpl::Body(Bytes::new()),
content_length: 0,
content_md5_base64: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
},
)
.await?;
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
let policy = String::from_utf8_lossy(&body_vec).to_string();
Ok(policy)
}
}
crates/ecstore/src/client/api_get_object_acl.rs
-199
View File
@@ -1,199 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use crate::client::{
api_error_response::http_resp_to_error_response,
api_get_options::GetObjectOptions,
transition_api::{ObjectInfo, ReaderImpl, RequestMetadata, TransitionClient},
};
use bytes::Bytes;
use http::{HeaderMap, HeaderValue};
use http_body_util::BodyExt;
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
use s3s::dto::Owner;
use std::collections::HashMap;
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
pub struct Grantee {
pub id: String,
pub display_name: String,
pub uri: String,
}
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
pub struct Grant {
pub grantee: Grantee,
pub permission: String,
}
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
pub struct AccessControlList {
pub grant: Vec<Grant>,
pub permission: String,
}
#[derive(Debug, Default, serde::Deserialize)]
pub struct AccessControlPolicy {
#[serde(skip)]
owner: Owner,
pub access_control_list: AccessControlList,
}
impl TransitionClient {
pub async fn get_object_acl(&self, bucket_name: &str, object_name: &str) -> Result<ObjectInfo, std::io::Error> {
let mut url_values = HashMap::new();
url_values.insert("acl".to_string(), "".to_string());
let mut resp = self
.execute_method(
http::Method::GET,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(),
query_values: url_values,
custom_header: HeaderMap::new(),
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
content_body: ReaderImpl::Body(Bytes::new()),
content_length: 0,
content_md5_base64: "".to_string(),
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
},
)
.await?;
let resp_status = resp.status();
let h = resp.headers().clone();
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
if resp_status != http::StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(
resp_status,
&h,
body_vec,
bucket_name,
object_name,
)));
}
let mut res = match quick_xml::de::from_str::<AccessControlPolicy>(&String::from_utf8(body_vec).unwrap()) {
Ok(result) => result,
Err(err) => {
return Err(std::io::Error::other(err.to_string()));
}
};
let mut obj_info = self
.stat_object(bucket_name, object_name, &GetObjectOptions::default())
.await?;
obj_info.owner.display_name = res.owner.display_name.clone();
obj_info.owner.id = res.owner.id.clone();
//obj_info.grant.extend(res.access_control_list.grant);
let canned_acl = get_canned_acl(&res);
if canned_acl != "" {
obj_info
.metadata
.insert("X-Amz-Acl", HeaderValue::from_str(&canned_acl).unwrap());
return Ok(obj_info);
}
let grant_acl = get_amz_grant_acl(&res);
/*for (k, v) in grant_acl {
obj_info.metadata.insert(HeaderName::from_bytes(k.as_bytes()).unwrap(), HeaderValue::from_str(&v.to_string()).unwrap());
}*/
Ok(obj_info)
}
}
fn get_canned_acl(ac_policy: &AccessControlPolicy) -> String {
let grants = ac_policy.access_control_list.grant.clone();
if grants.len() == 1 {
if grants[0].grantee.uri == "" && grants[0].permission == "FULL_CONTROL" {
return "private".to_string();
}
} else if grants.len() == 2 {
for g in grants {
if g.grantee.uri == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" && &g.permission == "READ" {
return "authenticated-read".to_string();
}
if g.grantee.uri == "http://acs.amazonaws.com/groups/global/AllUsers" && &g.permission == "READ" {
return "public-read".to_string();
}
if g.permission == "READ" && g.grantee.id == ac_policy.owner.id.clone().unwrap() {
return "bucket-owner-read".to_string();
}
}
} else if grants.len() == 3 {
for g in grants {
if g.grantee.uri == "http://acs.amazonaws.com/groups/global/AllUsers" && g.permission == "WRITE" {
return "public-read-write".to_string();
}
}
}
"".to_string()
}
pub fn get_amz_grant_acl(ac_policy: &AccessControlPolicy) -> HashMap<String, Vec<String>> {
let grants = ac_policy.access_control_list.grant.clone();
let mut res = HashMap::<String, Vec<String>>::new();
for g in grants {
let mut id = "id=".to_string();
id.push_str(&g.grantee.id);
let permission: &str = &g.permission;
match permission {
"READ" => {
res.entry("X-Amz-Grant-Read".to_string()).or_insert(vec![]).push(id);
}
"WRITE" => {
res.entry("X-Amz-Grant-Write".to_string()).or_insert(vec![]).push(id);
}
"READ_ACP" => {
res.entry("X-Amz-Grant-Read-Acp".to_string()).or_insert(vec![]).push(id);
}
"WRITE_ACP" => {
res.entry("X-Amz-Grant-Write-Acp".to_string()).or_insert(vec![]).push(id);
}
"FULL_CONTROL" => {
res.entry("X-Amz-Grant-Full-Control".to_string()).or_insert(vec![]).push(id);
}
_ => (),
}
}
res
}
crates/ecstore/src/client/api_get_object_attributes.rs
-266
View File
@@ -1,266 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use http::{HeaderMap, HeaderValue};
use std::collections::HashMap;
use time::OffsetDateTime;
use crate::client::constants::{GET_OBJECT_ATTRIBUTES_MAX_PARTS, GET_OBJECT_ATTRIBUTES_TAGS, ISO8601_DATEFORMAT};
use crate::client::{
api_get_object_acl::AccessControlPolicy,
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
};
use http_body_util::BodyExt;
use hyper::body::Body;
use hyper::body::Bytes;
use hyper::body::Incoming;
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
use rustfs_utils::EMPTY_STRING_SHA256_HASH;
use s3s::header::{X_AMZ_MAX_PARTS, X_AMZ_OBJECT_ATTRIBUTES, X_AMZ_PART_NUMBER_MARKER, X_AMZ_VERSION_ID};
pub struct ObjectAttributesOptions {
pub max_parts: i64,
pub version_id: String,
pub part_number_marker: i64,
//server_side_encryption: encrypt::ServerSide,
}
pub struct ObjectAttributes {
pub version_id: String,
pub last_modified: OffsetDateTime,
pub object_attributes_response: ObjectAttributesResponse,
}
impl ObjectAttributes {
fn new() -> Self {
Self {
version_id: "".to_string(),
last_modified: OffsetDateTime::now_utc(),
object_attributes_response: ObjectAttributesResponse::new(),
}
}
}
#[derive(Debug, Default, serde::Deserialize)]
pub struct Checksum {
checksum_crc32: String,
checksum_crc32c: String,
checksum_sha1: String,
checksum_sha256: String,
}
impl Checksum {
fn new() -> Self {
Self {
checksum_crc32: "".to_string(),
checksum_crc32c: "".to_string(),
checksum_sha1: "".to_string(),
checksum_sha256: "".to_string(),
}
}
}
#[derive(Debug, Default, serde::Deserialize)]
pub struct ObjectParts {
pub parts_count: i64,
pub part_number_marker: i64,
pub next_part_number_marker: i64,
pub max_parts: i64,
is_truncated: bool,
parts: Vec<ObjectAttributePart>,
}
impl ObjectParts {
fn new() -> Self {
Self {
parts_count: 0,
part_number_marker: 0,
next_part_number_marker: 0,
max_parts: 0,
is_truncated: false,
parts: Vec::new(),
}
}
}
#[derive(Debug, Default, serde::Deserialize)]
pub struct ObjectAttributesResponse {
pub etag: String,
pub storage_class: String,
pub object_size: i64,
pub checksum: Checksum,
pub object_parts: ObjectParts,
}
impl ObjectAttributesResponse {
fn new() -> Self {
Self {
etag: "".to_string(),
storage_class: "".to_string(),
object_size: 0,
checksum: Checksum::new(),
object_parts: ObjectParts::new(),
}
}
}
#[derive(Debug, Default, serde::Deserialize)]
struct ObjectAttributePart {
checksum_crc32: String,
checksum_crc32c: String,
checksum_sha1: String,
checksum_sha256: String,
part_number: i64,
size: i64,
}
impl ObjectAttributes {
pub async fn parse_response(&mut self, h: &HeaderMap, body_vec: Vec<u8>) -> Result<(), std::io::Error> {
let last_modified = h
.get("Last-Modified")
.ok_or_else(|| std::io::Error::other("missing Last-Modified header"))?
.to_str()
.map_err(|e| std::io::Error::other(format!("invalid Last-Modified header: {e}")))?;
let mod_time = OffsetDateTime::parse(last_modified, ISO8601_DATEFORMAT)
.map_err(|e| std::io::Error::other(format!("invalid Last-Modified date: {e}")))?;
self.last_modified = mod_time;
let version_id = h
.get(X_AMZ_VERSION_ID)
.ok_or_else(|| std::io::Error::other("missing version ID header"))?
.to_str()
.map_err(|e| std::io::Error::other(format!("invalid version ID header: {e}")))?;
self.version_id = version_id.to_string();
let body_str = String::from_utf8(body_vec).map_err(|e| std::io::Error::other(format!("invalid UTF-8 body: {e}")))?;
let mut response = match quick_xml::de::from_str::<ObjectAttributesResponse>(&body_str) {
Ok(result) => result,
Err(err) => {
return Err(std::io::Error::other(err.to_string()));
}
};
self.object_attributes_response = response;
Ok(())
}
}
impl TransitionClient {
pub async fn get_object_attributes(
&self,
bucket_name: &str,
object_name: &str,
opts: ObjectAttributesOptions,
) -> Result<ObjectAttributes, std::io::Error> {
let mut url_values = HashMap::new();
url_values.insert("attributes".to_string(), "".to_string());
if opts.version_id != "" {
url_values.insert("versionId".to_string(), opts.version_id);
}
let mut headers = HeaderMap::new();
headers.insert(
X_AMZ_OBJECT_ATTRIBUTES,
HeaderValue::from_str(GET_OBJECT_ATTRIBUTES_TAGS).expect("valid header value"),
);
if opts.part_number_marker > 0 {
headers.insert(
X_AMZ_PART_NUMBER_MARKER,
HeaderValue::from_str(&opts.part_number_marker.to_string()).expect("valid header value"),
);
}
if opts.max_parts > 0 {
headers.insert(
X_AMZ_MAX_PARTS,
HeaderValue::from_str(&opts.max_parts.to_string()).expect("valid header value"),
);
} else {
headers.insert(
X_AMZ_MAX_PARTS,
HeaderValue::from_str(&GET_OBJECT_ATTRIBUTES_MAX_PARTS.to_string()).expect("valid header value"),
);
}
/*if opts.server_side_encryption.is_some() {
opts.server_side_encryption.Marshal(headers);
}*/
let mut resp = self
.execute_method(
http::Method::HEAD,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(),
query_values: url_values,
custom_header: headers,
content_sha256_hex: EMPTY_STRING_SHA256_HASH.to_string(),
content_md5_base64: "".to_string(),
content_body: ReaderImpl::Body(Bytes::new()),
content_length: 0,
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
},
)
.await?;
let resp_status = resp.status();
let h = resp.headers().clone();
let has_etag = h.get("ETag").and_then(|v| v.to_str().ok()).unwrap_or("");
if !has_etag.is_empty() {
return Err(std::io::Error::other(
"get_object_attributes is not supported by the current endpoint version",
));
}
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
if resp_status != http::StatusCode::OK {
let err_body =
String::from_utf8(body_vec).map_err(|e| std::io::Error::other(format!("invalid UTF-8 error body: {e}")))?;
let mut er = match quick_xml::de::from_str::<AccessControlPolicy>(&err_body) {
Ok(result) => result,
Err(err) => {
return Err(std::io::Error::other(err.to_string()));
}
};
return Err(std::io::Error::other(er.access_control_list.permission));
}
let mut oa = ObjectAttributes::new();
oa.parse_response(&h, body_vec).await?;
Ok(oa)
}
}
crates/ecstore/src/client/api_get_object_file.rs
-159
View File
@@ -1,159 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::io;
use std::path::{Path, PathBuf};
#[cfg(not(windows))]
use std::os::unix::fs::PermissionsExt;
use tokio::fs::{self, OpenOptions};
use tokio::io::{AsyncSeekExt, AsyncWriteExt, SeekFrom};
use crate::client::{
api_error_response::err_invalid_argument, api_get_options::GetObjectOptions, transition_api::TransitionClient,
};
async fn prepare_download_target(file_path: &Path) -> io::Result<()> {
match fs::metadata(file_path).await {
Ok(metadata) if metadata.is_dir() => {
return Err(io::Error::other(err_invalid_argument("filename is a directory.")));
}
Ok(_) => {}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => return Err(err),
}
if let Some(parent) = file_path.parent()
&& !parent.as_os_str().is_empty()
{
fs::create_dir_all(parent).await?;
#[cfg(not(windows))]
{
let mut permissions = fs::metadata(parent).await?.permissions();
permissions.set_mode(0o700);
fs::set_permissions(parent, permissions).await?;
}
}
Ok(())
}
fn build_part_path(file_path: &Path) -> PathBuf {
PathBuf::from(format!("{}.part.rustfs", file_path.display()))
}
async fn open_download_part_file(file_part_path: &Path) -> io::Result<tokio::fs::File> {
let mut options = OpenOptions::new();
options.create(true).truncate(false).read(true).write(true);
#[cfg(not(windows))]
options.mode(0o600);
options.open(file_part_path).await
}
async fn cleanup_part_file(file_part_path: &Path) {
let _ = fs::remove_file(file_part_path).await;
}
impl TransitionClient {
pub async fn fget_object(
&self,
bucket_name: &str,
object_name: &str,
file_path: &str,
mut opts: GetObjectOptions,
) -> Result<(), io::Error> {
let file_path = Path::new(file_path);
prepare_download_target(file_path).await?;
let file_part_path = build_part_path(file_path);
let mut file_part = open_download_part_file(&file_part_path).await?;
let existing_len = file_part.metadata().await?.len();
if existing_len > 0 {
opts.set_range(existing_len as i64, 0)?;
file_part.seek(SeekFrom::Start(existing_len)).await?;
}
let (_object_info, _headers, mut object_reader) = self.get_object_inner(bucket_name, object_name, &opts).await?;
if let Err(err) = tokio::io::copy(&mut object_reader, &mut file_part).await {
cleanup_part_file(&file_part_path).await;
return Err(err);
}
if let Err(err) = file_part.flush().await {
cleanup_part_file(&file_part_path).await;
return Err(err);
}
drop(file_part);
if let Err(err) = fs::rename(&file_part_path, file_path).await {
cleanup_part_file(&file_part_path).await;
return Err(err);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
#[tokio::test]
async fn prepare_download_target_allows_missing_file_and_creates_parent_dirs() {
let dir = tempdir().expect("temp dir");
let target = dir.path().join("nested").join("object.bin");
prepare_download_target(&target)
.await
.expect("missing target should be accepted");
assert!(target.parent().expect("parent").exists(), "parent directory should be created");
assert!(
fs::metadata(&target).await.is_err(),
"preparing the target should not create the final file eagerly"
);
}
#[tokio::test]
async fn prepare_download_target_rejects_directory_paths() {
let dir = tempdir().expect("temp dir");
let target_dir = dir.path().join("download-dir");
fs::create_dir_all(&target_dir).await.expect("target dir");
let err = prepare_download_target(&target_dir)
.await
.expect_err("directory targets must be rejected");
assert!(err.to_string().contains("directory"), "unexpected error for directory target: {err}");
}
#[tokio::test]
async fn open_download_part_file_creates_part_file() {
let dir = tempdir().expect("temp dir");
let target = dir.path().join("object.bin");
let part_path = build_part_path(&target);
let file = open_download_part_file(&part_path)
.await
.expect("part file should be created");
drop(file);
assert!(part_path.exists(), "part file should exist after creation");
}
}
crates/ecstore/src/client/api_restore.rs
-134
View File
@@ -1,134 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use crate::client::{
api_error_response::{err_invalid_argument, http_resp_to_error_response},
api_get_object_acl::AccessControlList,
api_get_options::GetObjectOptions,
transition_api::{ObjectInfo, ReadCloser, ReaderImpl, RequestMetadata, TransitionClient, to_object_info},
};
use http::HeaderMap;
use http_body_util::BodyExt;
use hyper::body::Body;
use hyper::body::Bytes;
use s3s::dto::RestoreRequest;
use std::collections::HashMap;
use std::io::Cursor;
use tokio::io::BufReader;
const TIER_STANDARD: &str = "Standard";
const TIER_BULK: &str = "Bulk";
const TIER_EXPEDITED: &str = "Expedited";
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
pub struct Encryption {
pub encryption_type: String,
pub kms_context: String,
pub kms_key_id: String,
}
#[derive(Debug, Default, serde::Serialize, serde::Deserialize)]
pub struct MetadataEntry {
pub name: String,
pub value: String,
}
#[derive(Debug, Default, serde::Serialize)]
pub struct S3 {
pub access_control_list: AccessControlList,
pub bucket_name: String,
pub prefix: String,
pub canned_acl: String,
pub encryption: Encryption,
pub storage_class: String,
//tagging: Tags,
pub user_metadata: MetadataEntry,
}
impl TransitionClient {
pub async fn restore_object(
&self,
bucket_name: &str,
object_name: &str,
version_id: &str,
restore_req: &RestoreRequest,
) -> Result<(), std::io::Error> {
/*let restore_request = match quick_xml::se::to_string(restore_req) {
Ok(buf) => buf,
Err(e) => {
return Err(std::io::Error::other(e));
}
};*/
let restore_request = "".to_string();
let restore_request_bytes = restore_request.as_bytes().to_vec();
let mut url_values = HashMap::new();
url_values.insert("restore".to_string(), "".to_string());
if version_id != "" {
url_values.insert("versionId".to_string(), version_id.to_string());
}
let restore_request_buffer = Bytes::from(restore_request_bytes.clone());
let resp = self
.execute_method(
http::Method::HEAD,
&mut RequestMetadata {
bucket_name: bucket_name.to_string(),
object_name: object_name.to_string(),
query_values: url_values,
custom_header: HeaderMap::new(),
content_sha256_hex: "".to_string(), //sum_sha256_hex(&restore_request_bytes),
content_md5_base64: "".to_string(), //sum_md5_base64(&restore_request_bytes),
content_body: ReaderImpl::Body(restore_request_buffer),
content_length: restore_request_bytes.len() as i64,
stream_sha256: false,
trailer: HeaderMap::new(),
pre_sign_url: Default::default(),
add_crc: Default::default(),
extra_pre_sign_header: Default::default(),
bucket_location: Default::default(),
expires: Default::default(),
},
)
.await?;
let resp_status = resp.status();
let h = resp.headers().clone();
let mut body_vec = Vec::new();
let mut body = resp.into_body();
while let Some(frame) = body.frame().await {
let frame = frame.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
if let Some(data) = frame.data_ref() {
body_vec.extend_from_slice(data);
}
}
if resp_status != http::StatusCode::ACCEPTED && resp_status != http::StatusCode::OK {
return Err(std::io::Error::other(http_resp_to_error_response(
resp_status,
&h,
body_vec,
bucket_name,
"",
)));
}
Ok(())
}
}
crates/ecstore/src/client/checksum.rs
+12 -2
View File
@@ -27,12 +27,24 @@ use crate::client::utils::base64_decode;
use crate::client::utils::base64_encode;
use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
use crate::{disk::DiskAPI, object_api::GetObjectReader};
// s3s::header has no CRC64NVME constant yet; the canonical RustFS copy lives
// in rustfs-utils' headers module.
use rustfs_utils::http::headers::AMZ_CHECKSUM_CRC64NVME;
use s3s::header::{
X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
};
use enumset::{EnumSet, EnumSetType, enum_set};
/// One of three deliberately separate checksum registries (backlog#1833):
/// this enum is the MinIO-port client's wire vocabulary and stops at the
/// standard S3 set (CRC64NVME is its newest member; the RustFS extensions do
/// not exist on this client path). The streaming-hash registry lives in
/// `rustfs_checksums::ChecksumAlgorithm` (crates/checksums/src/lib.rs) and
/// the on-disk xl.meta bitset in `rustfs_rio::ChecksumType`
/// (crates/rio/src/checksum.rs, varint bits are append-only). When adding an
/// algorithm, extend all three (or record why not) — they do not derive from
/// each other.
#[derive(Debug, EnumSetType, Default)]
#[enumset(repr = "u8")]
pub enum ChecksumMode {
@@ -57,8 +69,6 @@ lazy_static! {
static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> =
enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject);
}
const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme";
impl ChecksumMode {
//pub const CRC64_NVME_POLYNOMIAL: i64 = 0xad93d23594c93659;
crates/ecstore/src/client/constants.rs
-3
View File
@@ -37,6 +37,3 @@ pub const TOTAL_WORKERS: i64 = 4;
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
pub const GET_OBJECT_ATTRIBUTES_TAGS: &str = "ETag,Checksum,StorageClass,ObjectSize,ObjectParts";
pub const GET_OBJECT_ATTRIBUTES_MAX_PARTS: i64 = 1000;
crates/ecstore/src/client/mod.rs
-5
View File
@@ -16,12 +16,8 @@
#![allow(dead_code)]
pub mod admin_handler_utils;
pub mod api_bucket_policy;
pub mod api_error_response;
pub mod api_get_object;
pub mod api_get_object_acl;
pub mod api_get_object_attributes;
pub mod api_get_object_file;
pub mod api_get_options;
pub mod api_list;
pub mod api_put_object;
@@ -29,7 +25,6 @@ pub mod api_put_object_common;
pub mod api_put_object_multipart;
pub mod api_put_object_streaming;
pub mod api_remove;
pub mod api_restore;
pub mod api_s3_datatypes;
pub mod api_stat;
pub mod bucket_cache;
crates/ecstore/src/client/transition_api.rs
-10
View File
@@ -1006,16 +1006,6 @@ impl TransitionCore {
client.abort_multipart_upload(bucket_name, object, upload_id).await
}
pub async fn get_bucket_policy(&self, bucket_name: &str) -> Result<String, std::io::Error> {
let client = self.0.clone();
client.get_bucket_policy(bucket_name).await
}
pub async fn put_bucket_policy(&self, bucket_name: &str, bucket_policy: &str) -> Result<(), std::io::Error> {
let client = self.0.clone();
client.put_bucket_policy(bucket_name, bucket_policy).await
}
pub async fn get_object(
&self,
bucket_name: &str,
crates/ecstore/src/diagnostics/admin_server_info.rs
+102 -4
View File
@@ -12,7 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::cluster::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
use crate::cluster::rpc::{
ScannerBucketListing, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
};
use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend_cached};
use crate::error::{Error, Result};
use crate::{
@@ -23,6 +25,7 @@ use crate::{
use crate::data_usage::load_data_usage_cache;
use crate::storage_api_contracts::admin::StorageAdminApi;
use crate::storage_api_contracts::bucket::BucketOptions;
use rustfs_common::heal_channel::DriveState;
use rustfs_madmin::{
BackendDisks, Disk, ErasureSetInfo, ITEM_INITIALIZING, ITEM_OFFLINE, ITEM_ONLINE, ITEM_UNKNOWN, InfoMessage, MemStats,
@@ -74,6 +77,19 @@ fn apply_data_usage_result(
}
}
fn apply_bucket_namespace_count(result: Result<ScannerBucketListing>, buckets: &mut rustfs_madmin::Buckets) {
if let Ok(listing) = result
&& listing.topology_complete
{
let count = listing.buckets.iter().filter(|bucket| !bucket.name.starts_with('.')).count();
let Ok(count) = u64::try_from(count) else {
return;
};
buckets.count = count;
buckets.error = None;
}
}
// pub const ITEM_OFFLINE: &str = "offline";
// pub const ITEM_INITIALIZING: &str = "initializing";
// pub const ITEM_ONLINE: &str = "online";
@@ -285,6 +301,18 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
&mut delete_markers,
&mut usage,
);
if buckets.error.is_some() {
apply_bucket_namespace_count(
store
.list_bucket_for_scanner(&BucketOptions {
cached: true,
no_metadata: true,
..Default::default()
})
.await,
&mut buckets,
);
}
let after3 = OffsetDateTime::now_utc();
@@ -705,12 +733,13 @@ mod tests {
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
};
use crate::runtime::sources as runtime_sources;
use crate::storage_api_contracts::bucket::BucketInfo;
use rustfs_madmin::{Disk, ITEM_OFFLINE, ITEM_ONLINE, ITEM_UNKNOWN, ServerProperties};
use super::{
DATA_USAGE_ROOT, DATA_USAGE_UNAVAILABLE_ERROR, apply_data_usage_result, apply_erasure_set_usage,
get_local_server_property, get_online_offline_disks_stats, get_server_info, reconcile_servers_with_endpoint_topology,
server_topology_completeness_report,
DATA_USAGE_ROOT, DATA_USAGE_UNAVAILABLE_ERROR, apply_bucket_namespace_count, apply_data_usage_result,
apply_erasure_set_usage, get_local_server_property, get_online_offline_disks_stats, get_server_info,
reconcile_servers_with_endpoint_topology, server_topology_completeness_report,
};
fn disk_with_state(endpoint: &str, state: &str) -> Disk {
@@ -960,6 +989,75 @@ mod tests {
assert_eq!(usage.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
}
#[test]
fn live_bucket_namespace_count_survives_unavailable_data_usage() {
let mut buckets = rustfs_madmin::Buckets {
count: 0,
error: Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
};
apply_bucket_namespace_count(
Ok(crate::cluster::rpc::ScannerBucketListing {
buckets: vec![
BucketInfo {
name: "bucket-a".to_string(),
..Default::default()
},
BucketInfo {
name: ".rustfs.sys".to_string(),
..Default::default()
},
BucketInfo {
name: "bucket-b".to_string(),
..Default::default()
},
],
set_buckets: Vec::new(),
topology_complete: true,
}),
&mut buckets,
);
assert_eq!(buckets.count, 2);
assert_eq!(buckets.error, None);
}
#[test]
fn incomplete_bucket_namespace_lookup_preserves_usage_state() {
let mut buckets = rustfs_madmin::Buckets {
count: 7,
error: Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
};
apply_bucket_namespace_count(
Ok(crate::cluster::rpc::ScannerBucketListing {
buckets: vec![BucketInfo {
name: "bucket-a".to_string(),
..Default::default()
}],
set_buckets: Vec::new(),
topology_complete: false,
}),
&mut buckets,
);
assert_eq!(buckets.count, 7);
assert_eq!(buckets.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
}
#[test]
fn failed_bucket_namespace_lookup_preserves_usage_state() {
let mut buckets = rustfs_madmin::Buckets {
count: 7,
error: Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
};
apply_bucket_namespace_count(Err(crate::error::Error::DiskNotFound), &mut buckets);
assert_eq!(buckets.count, 7);
assert_eq!(buckets.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
}
#[test]
fn incomplete_erasure_set_cache_is_not_reported_as_zero() {
let mut cache = rustfs_data_usage::DataUsageCache::default();
crates/ecstore/src/disk/error.rs
+6 -26
View File
@@ -113,6 +113,9 @@ pub enum DiskError {
#[error("bit-rot hash algorithm is invalid")]
BitrotHashAlgoInvalid,
/// Never constructed locally by RustFS (only reachable through wire
/// decoding, and no current node sends it). The wire code is kept for
/// cross-version compatibility — do not renumber or remove (backlog#1831).
#[error("Rename across devices not allowed, please fix your backend configuration")]
CrossDeviceLink,
@@ -143,6 +146,9 @@ pub enum DiskError {
#[error("io error {0}")]
Io(#[source] io::Error),
/// Never constructed locally by RustFS (only reachable through wire
/// decoding, and no current node sends it). The wire code is kept for
/// cross-version compatibility — do not renumber or remove (backlog#1831).
#[error("source stalled")]
SourceStalled,
@@ -642,19 +648,6 @@ impl Hash for DiskError {
// is currently commented out to avoid complexity. These can be re-enabled
// when needed for specific disk quorum checking and error aggregation logic.
/// Bitrot errors
#[derive(Debug, thiserror::Error)]
pub enum BitrotErrorType {
#[error("bitrot checksum verification failed")]
BitrotChecksumMismatch { expected: String, got: String },
}
impl From<BitrotErrorType> for DiskError {
fn from(e: BitrotErrorType) -> Self {
DiskError::other(e)
}
}
/// Context wrapper for file access errors
#[derive(Debug, thiserror::Error)]
pub struct FileAccessDeniedWithContext {
@@ -869,19 +862,6 @@ mod tests {
let _disk_error: DiskError = json_error.into();
}
#[test]
fn test_bitrot_error_type() {
let bitrot_error = BitrotErrorType::BitrotChecksumMismatch {
expected: "abc123".to_string(),
got: "def456".to_string(),
};
assert!(bitrot_error.to_string().contains("bitrot checksum verification failed"));
let disk_error: DiskError = bitrot_error.into();
assert!(matches!(disk_error, DiskError::Io(_)));
}
#[test]
fn test_file_access_denied_with_context() {
let path = PathBuf::from("/test/path");
crates/ecstore/src/erasure/coding/bitrot.rs
+29 -17
View File
@@ -18,7 +18,11 @@ use std::io::IoSlice;
use std::time::Duration;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use tracing::error;
use uuid::Uuid;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_ERASURE: &str = "erasure";
const EVENT_BITROT_SHORT_SHARD_READ: &str = "bitrot_short_shard_read";
const EVENT_BITROT_HASH_MISMATCH: &str = "bitrot_hash_mismatch";
/// A shard source that may already hold its bytes in memory.
///
@@ -73,7 +77,6 @@ pin_project! {
buf: Vec<u8>,
skip_verify: bool,
last_verify_duration: Duration,
id: Uuid,
}
}
@@ -90,7 +93,6 @@ where
buf: Vec::new(),
skip_verify,
last_verify_duration: Duration::ZERO,
id: Uuid::new_v4(),
}
}
@@ -118,7 +120,7 @@ where
let need = self.hash_algo.size() + want;
self.read_scratch_block(need, want).await?;
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need], &self.id)?;
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need])?;
out.copy_from_slice(data);
self.last_verify_duration = verify;
Ok(want)
@@ -157,7 +159,7 @@ where
}
let filled = fill(&mut self.inner, &mut self.buf[..need]).await?;
if filled < need {
return Err(short_shard_read(&self.id, filled.saturating_sub(self.hash_algo.size()), want));
return Err(short_shard_read(filled.saturating_sub(self.hash_algo.size()), want));
}
Ok(())
}
@@ -166,15 +168,23 @@ where
/// buffer returns its length, a short read is UnexpectedEof (backlog#799 B2).
fn finish_len(&self, data_len: usize, want: usize) -> std::io::Result<usize> {
if data_len < want {
return Err(short_shard_read(&self.id, data_len, want));
return Err(short_shard_read(data_len, want));
}
Ok(data_len)
}
}
/// A truncated shard is `UnexpectedEof`, not a short success (backlog#799 B2).
fn short_shard_read(id: &Uuid, got: usize, want: usize) -> std::io::Error {
error!("bitrot reader short shard read: id={id} got {got} of {want} bytes");
fn short_shard_read(got: usize, want: usize) -> std::io::Error {
error!(
event = EVENT_BITROT_SHORT_SHARD_READ,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_ERASURE,
state = "failed",
got,
want,
"short shard read: got {got} of {want} bytes"
);
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, format!("short shard read: got {got} of {want} bytes"))
}
@@ -184,12 +194,7 @@ fn short_shard_read(id: &Uuid, got: usize, want: usize) -> std::io::Error {
/// hash never reaches the caller's buffer. The verify duration is returned
/// rather than stored so this stays a free function usable while `self` is
/// borrowed for the block.
fn split_and_verify<'a>(
hash_algo: &HashAlgorithm,
skip_verify: bool,
block: &'a [u8],
id: &Uuid,
) -> std::io::Result<(&'a [u8], Duration)> {
fn split_and_verify<'a>(hash_algo: &HashAlgorithm, skip_verify: bool, block: &'a [u8]) -> std::io::Result<(&'a [u8], Duration)> {
let (hash, data) = block.split_at(hash_algo.size());
if skip_verify {
return Ok((data, Duration::ZERO));
@@ -198,7 +203,14 @@ fn split_and_verify<'a>(
let actual_hash = hash_algo.hash_encode(data);
let verify = verify_start.elapsed();
if actual_hash.as_ref() != hash {
error!("bitrot reader hash mismatch, id={id} data_len={}", data.len());
error!(
event = EVENT_BITROT_HASH_MISMATCH,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_ERASURE,
state = "failed",
data_len = data.len(),
"bitrot hash mismatch"
);
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch"));
}
Ok((data, verify))
@@ -254,7 +266,7 @@ where
// `need` bytes returns `None` and falls through to the scratch path,
// keeping the short-read contract.
if let Some(block) = self.inner.try_take_block(need) {
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &block, &self.id)?;
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &block)?;
out.extend_from_slice(data);
self.last_verify_duration = verify;
return Ok(want);
@@ -264,7 +276,7 @@ where
// the sink differs (`extend_from_slice` into `out` instead of
// `copy_from_slice` into a pre-zeroed buffer).
self.read_scratch_block(need, want).await?;
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need], &self.id)?;
let (data, verify) = split_and_verify(&self.hash_algo, self.skip_verify, &self.buf[..need])?;
out.extend_from_slice(data);
self.last_verify_duration = verify;
Ok(want)
crates/ecstore/src/erasure/coding/decode.rs
+63 -20
View File
@@ -29,6 +29,7 @@ use crate::set_disk::shard_source::{ShardReadCost, ShardStripeSource, StripeRead
use futures::FutureExt;
use futures::stream::{FuturesUnordered, StreamExt};
use pin_project_lite::pin_project;
use smallvec::{SmallVec, smallvec};
use std::future::Future;
use std::io;
use std::io::ErrorKind;
@@ -40,9 +41,15 @@ use tracing::{debug, error, warn};
type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, bool)> + Send + 'a>>;
const INLINE_SHARD_SLOTS: usize = 32;
type ShardBuffers = SmallVec<[Option<Vec<u8>>; INLINE_SHARD_SLOTS]>;
type ShardErrors = SmallVec<[Option<Error>; INLINE_SHARD_SLOTS]>;
type ShardIndexes = SmallVec<[usize; INLINE_SHARD_SLOTS]>;
type ActiveReaders = SmallVec<[bool; INLINE_SHARD_SLOTS]>;
/// One stripe's worth of shard buffers plus the per-shard read errors, as
/// returned by `ParallelReader::read` / `read_stripe_timed`.
type StripeReadOutput = (Vec<Option<Vec<u8>>>, Vec<Option<Error>>);
type StripeReadOutput = (ShardBuffers, ShardErrors);
const ENV_RUSTFS_SHARD_LOCALITY_SCHEDULING: &str = "RUSTFS_SHARD_LOCALITY_SCHEDULING";
const ENV_RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE: &str = "RUSTFS_GET_SHARD_LOCALITY_PREFERENCE_ENABLE";
@@ -390,7 +397,7 @@ pub(crate) struct ParallelReader<R> {
// start, parity slots only once a data shard is missing/dead. Unengaged
// parity stays an unopened deferred reader; `deferred_handles[i]` realigns
// it to the current stripe when it is engaged mid-object (backlog#923).
engaged: Vec<bool>,
engaged: SmallVec<[bool; INLINE_SHARD_SLOTS]>,
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
stripe_index: usize,
}
@@ -573,7 +580,7 @@ where
// behavior. With the gate on, only data slots start engaged; parity is
// engaged on demand, stripe-aligned through its deferred handle.
let data_shards_only = get_lockstep_data_shards_only_enabled();
let engaged = (0..readers.len())
let engaged: SmallVec<_> = (0..readers.len())
.map(|index| !data_shards_only || index < e.data_shards)
.collect();
ParallelReader {
@@ -612,7 +619,7 @@ where
fn record_shard_read_result(
shards: &mut [Option<Vec<u8>>],
errs: &mut [Option<Error>],
retire_readers: &mut Vec<usize>,
retire_readers: &mut ShardIndexes,
success: &mut usize,
successful_costs: &mut ShardReadCostCounts,
i: usize,
@@ -637,7 +644,7 @@ fn record_shard_read_result(
}
}
fn retire_abandoned_readers(errs: &mut [Option<Error>], retire_readers: &mut Vec<usize>, active_readers: &[bool]) {
fn retire_abandoned_readers(errs: &mut [Option<Error>], retire_readers: &mut ShardIndexes, active_readers: &[bool]) {
for (i, active) in active_readers.iter().enumerate() {
if !*active {
continue;
@@ -692,7 +699,7 @@ where
R: crate::erasure::coding::ShardSource,
{
#[hotpath::measure(impl_type = "ParallelReader")]
pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
pub async fn read(&mut self) -> StripeReadOutput {
// On the reconstruction-verifying GET path, read every live shard reader
// in lockstep so all readers advance one block per stripe and stay
// mutually aligned. The adaptive data-first path below only reads
@@ -716,7 +723,7 @@ where
};
if shard_size == 0 {
return (vec![None; num_readers], vec![None; num_readers]);
return (smallvec![None; num_readers], smallvec![None; num_readers]);
}
// Advance to the next stripe so the following read() computes the correct
@@ -727,8 +734,8 @@ where
// is only read above to derive `shard_size`, so advancing here is safe.
self.offset += shard_size;
let mut shards: Vec<Option<Vec<u8>>> = vec![None; num_readers];
let mut errs = vec![None; num_readers];
let mut shards: ShardBuffers = smallvec![None; num_readers];
let mut errs: ShardErrors = smallvec![None; num_readers];
let read_costs = self.read_costs.as_slice();
let locality_preference_enabled = self.locality_preference_enabled;
let low_cost_available = self
@@ -759,11 +766,11 @@ where
self.buffers.ensure_slots(num_readers);
let mut retire_readers = Vec::new();
let mut retire_readers = ShardIndexes::new();
if num_readers >= self.data_shards {
let mut reader_iter = ReaderLaunchIter::new(&mut self.readers, read_costs, locality_preference_enabled);
let mut sets = FuturesUnordered::new();
let mut active_readers = vec![false; num_readers];
let mut active_readers: ActiveReaders = smallvec![false; num_readers];
let stripe_read_start = self.metrics_path.map(|_| Instant::now());
let mut scheduled = 0usize;
for _ in 0..self.data_shards {
@@ -1023,7 +1030,7 @@ where
/// stripe would reintroduce the desync. A parity reader that cannot be
/// realigned (no pending deferred handle) is likewise retired instead of
/// being read out of position.
async fn read_lockstep(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
async fn read_lockstep(&mut self) -> StripeReadOutput {
let num_readers = self.readers.len();
let shard_size = if self.offset + self.shard_size > self.shard_file_size {
self.shard_file_size - self.offset
@@ -1031,8 +1038,8 @@ where
self.shard_size
};
let mut shards: Vec<Option<Vec<u8>>> = vec![None; num_readers];
let mut errs: Vec<Option<Error>> = vec![None; num_readers];
let mut shards: ShardBuffers = smallvec![None; num_readers];
let mut errs: ShardErrors = smallvec![None; num_readers];
if shard_size == 0 {
return (shards, errs);
}
@@ -1071,7 +1078,7 @@ where
// Pre-claim per-slot buffers so the `self.readers` borrow below stays
// disjoint from `self.buffers`; `Some(buffer)` also records which slots
// participate, avoiding a per-stripe sidecar allocation.
let mut bufs: Vec<Option<Vec<u8>>> = Vec::with_capacity(num_readers);
let mut bufs: ShardBuffers = SmallVec::with_capacity(num_readers);
for i in 0..num_readers {
bufs.push(if self.engaged[i] && self.readers[i].is_some() {
Some(self.buffers.take(i, shard_size))
@@ -1086,7 +1093,7 @@ where
let locality_preference_enabled = self.locality_preference_enabled;
let stripe_read_start = metrics_path.map(|_| Instant::now());
let mut retire_readers = Vec::new();
let mut retire_readers = ShardIndexes::new();
let mut scheduled = 0usize;
let mut success = 0usize;
let mut completed = 0usize;
@@ -1351,10 +1358,7 @@ fn get_data_block_len(shards: &[Option<Vec<u8>>], data_blocks: usize) -> usize {
/// stripe-read stage timer. Factored out so the depth-1 prefetch loop and the
/// serial loop time reads identically. A free `async fn` (rather than a closure)
/// so the returned future's borrow of `reader` is correctly tied to the call.
async fn read_stripe_timed<R>(
reader: &mut ParallelReader<R>,
stage_metrics_enabled: bool,
) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>)
async fn read_stripe_timed<R>(reader: &mut ParallelReader<R>, stage_metrics_enabled: bool) -> StripeReadOutput
where
R: crate::erasure::coding::ShardSource,
{
@@ -1967,6 +1971,32 @@ mod tests {
type BoxedShardReader = crate::io_support::bitrot::ShardReader;
#[test]
fn shard_scratch_stays_inline_through_the_common_limit_and_spills_safely() {
let inline: ShardBuffers = smallvec![None; INLINE_SHARD_SLOTS];
assert!(!inline.spilled(), "the common shard-count boundary must not allocate");
let spilled: ShardBuffers = smallvec![None; INLINE_SHARD_SLOTS + 1];
assert!(spilled.spilled(), "larger supported shard counts must fall back to the heap");
assert_eq!(spilled.len(), INLINE_SHARD_SLOTS + 1);
}
#[tokio::test]
async fn parallel_reader_preserves_slot_count_above_inline_capacity() {
const DATA_SHARDS: usize = INLINE_SHARD_SLOTS;
const TOTAL_SHARDS: usize = INLINE_SHARD_SLOTS + 1;
let readers = std::iter::repeat_with(|| None).take(TOTAL_SHARDS).collect();
let erasure = Erasure::new(DATA_SHARDS, 1, DATA_SHARDS);
let mut reader: ParallelReader<Cursor<Vec<u8>>> = ParallelReader::new(readers, erasure, 0, DATA_SHARDS);
let (shards, errors) = reader.read().await;
assert!(shards.spilled());
assert!(errors.spilled());
assert_eq!(shards.len(), TOTAL_SHARDS);
assert_eq!(errors.len(), TOTAL_SHARDS);
}
/// Counts the raw bytes pulled from a shard stream, to prove which shards
/// a decode path actually touches (backlog#923 call-count evidence).
struct CountingShardReader {
@@ -2343,6 +2373,19 @@ mod tests {
assert_eq!(err.expect("range beyond total length should fail").kind(), ErrorKind::InvalidInput);
}
#[tokio::test]
async fn test_erasure_decode_zero_length_does_not_read_or_emit() {
let erasure = Erasure::new(2, 1, 64);
let readers: Vec<Option<BitrotReader<Cursor<Vec<u8>>>>> = vec![None, None, None];
let mut output = Vec::new();
let (written, err) = erasure.decode(&mut output, readers, 0, 0, 0).await;
assert_eq!(written, 0);
assert!(err.is_none());
assert!(output.is_empty());
}
#[tokio::test]
async fn test_erasure_decode_with_read_costs_restores_missing_data_shard_range() {
const DATA_SHARDS: usize = 2;
crates/ecstore/src/erasure/coding/encode.rs
+67 -6
View File
@@ -91,6 +91,11 @@ fn use_bytesmut_ingest() -> bool {
})
}
fn small_ingest_capacity(erasure: &Erasure, size_hint: usize) -> usize {
let data_len = size_hint.min(erasure.block_size);
erasure.encoded_capacity_for_data_len(data_len).min(erasure.block_size)
}
/// Keeps the encoder producer scoped to its parent future. Tokio detaches a
/// task when its `JoinHandle` is dropped, so the producer must be aborted when
/// an upload is cancelled before the encode pipeline finishes.
@@ -540,13 +545,14 @@ impl Erasure {
writers: &mut [Option<BitrotWriterWrapper>],
quorum: usize,
require_single_block: bool,
size_hint: usize,
) -> std::io::Result<(R, usize)>
where
R: AsyncRead + Send + Sync + Unpin,
{
use tokio::io::AsyncReadExt;
let mut buf = Vec::with_capacity(self.block_size);
let mut buf = Vec::with_capacity(small_ingest_capacity(&self, size_hint));
let total = if require_single_block {
let read_limit = self
.block_size
@@ -880,7 +886,24 @@ impl Erasure {
where
R: AsyncRead + Send + Sync + Unpin,
{
self.encode_small_direct(reader, writers, quorum, false).await
let size_hint = self.block_size;
self.encode_small_direct(reader, writers, quorum, false, size_hint).await
}
/// Size-aware inline fast path. `size_hint` only controls the bounded initial
/// allocation; reads remain authoritative.
#[hotpath::measure(impl_type = "Erasure")]
pub async fn encode_inline_small_with_size_hint<R>(
self: Arc<Self>,
reader: R,
writers: &mut [Option<BitrotWriterWrapper>],
quorum: usize,
size_hint: usize,
) -> std::io::Result<(R, usize)>
where
R: AsyncRead + Send + Sync + Unpin,
{
self.encode_small_direct(reader, writers, quorum, false, size_hint).await
}
/// Fast path for single-block non-inline objects: avoids the producer/consumer
@@ -895,7 +918,24 @@ impl Erasure {
where
R: AsyncRead + Send + Sync + Unpin,
{
self.encode_small_direct(reader, writers, quorum, true).await
let size_hint = self.block_size;
self.encode_small_direct(reader, writers, quorum, true, size_hint).await
}
/// Size-aware single-block fast path. `size_hint` only controls the bounded
/// initial allocation; reads remain authoritative.
#[hotpath::measure(impl_type = "Erasure")]
pub async fn encode_single_block_non_inline_with_size_hint<R>(
self: Arc<Self>,
reader: R,
writers: &mut [Option<BitrotWriterWrapper>],
quorum: usize,
size_hint: usize,
) -> std::io::Result<(R, usize)>
where
R: AsyncRead + Send + Sync + Unpin,
{
self.encode_small_direct(reader, writers, quorum, true, size_hint).await
}
}
@@ -2293,7 +2333,10 @@ mod tests {
let erasure = Arc::new(Erasure::new(1, 0, 16));
let reader = tokio::io::BufReader::new(Cursor::new(Vec::<u8>::new()));
let (_reader, total) = erasure.encode_inline_small(reader, &mut writers, 1).await.unwrap();
let (_reader, total) = erasure
.encode_inline_small_with_size_hint(reader, &mut writers, 1, 0)
.await
.unwrap();
assert_eq!(total, 0);
// No shutdown was called, so nothing should be committed
@@ -2325,7 +2368,10 @@ mod tests {
let payload = b"hello inline small";
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
let reader = tokio::io::BufReader::new(Cursor::new(payload.to_vec()));
let (_reader, total) = erasure.encode_inline_small(reader, &mut writers, DATA_SHARDS).await.unwrap();
let (_reader, total) = erasure
.encode_inline_small_with_size_hint(reader, &mut writers, DATA_SHARDS, 1)
.await
.unwrap();
assert_eq!(total, payload.len());
// All shards must have received data (shutdown flushed the bitrot header + shard bytes)
@@ -2392,7 +2438,7 @@ mod tests {
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
let reader = tokio::io::BufReader::new(Cursor::new(payload));
let err = erasure
.encode_single_block_non_inline(reader, &mut writers, DATA_SHARDS)
.encode_single_block_non_inline_with_size_hint(reader, &mut writers, DATA_SHARDS, BLOCK_SIZE)
.await
.expect_err("single-block fast path must reject oversized readers");
@@ -2403,6 +2449,21 @@ mod tests {
}
}
#[test]
fn small_ingest_capacity_uses_bounded_size_hint() {
let erasure = Erasure::new(4, 2, 1024 * 1024);
assert_eq!(small_ingest_capacity(&erasure, 0), 0);
assert_eq!(small_ingest_capacity(&erasure, 4 * 1024), 6 * 1024);
assert_eq!(small_ingest_capacity(&erasure, 16 * 1024), 24 * 1024);
assert_eq!(small_ingest_capacity(&erasure, usize::MAX), 1024 * 1024);
let legacy = Erasure::new_with_options(4, 2, 1024 * 1024, true);
assert_eq!(small_ingest_capacity(&legacy, 4 * 1024), 6 * 1024);
let high_parity = Erasure::new(4, 12, 1024 * 1024);
assert_eq!(small_ingest_capacity(&high_parity, usize::MAX), 1024 * 1024);
}
#[tokio::test]
async fn read_full_buf_or_eof_returns_none_on_empty_reader() {
let mut reader = Cursor::new(Vec::<u8>::new());
crates/ecstore/src/erasure/coding/erasure.rs
+9
View File
@@ -968,6 +968,15 @@ impl Erasure {
self.data_shards + self.parity_shards
}
pub(crate) fn encoded_capacity_for_data_len(&self, data_len: usize) -> usize {
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
} else {
calc_shard_size
};
shard_size_fn(data_len, self.data_shards).saturating_mul(self.total_shard_count())
}
/// Whether the erasure dimensions are safe for the shard/offset arithmetic.
///
/// `block_size` and `data_shards` come straight from on-disk metadata; a
crates/ecstore/src/io_support/bitrot.rs
+47 -31
View File
@@ -120,26 +120,41 @@ struct BitrotReaderSource {
impl BitrotReaderSource {
async fn open(self) -> disk::error::Result<Option<BoxedObjectReader>> {
if let Some(data) = self.inline_data {
let mut rd = Cursor::new(data);
let offset = u64::try_from(self.offset).map_err(|_| DiskError::FileCorrupt)?;
rd.set_position(offset);
Ok(Some(ShardReader::InMemory(rd)))
} else if let Some(disk) = self.disk {
open_disk_reader(
&disk,
&self.bucket,
&self.path,
self.offset,
self.length,
self.use_mmap_read,
self.stage_metrics.map(|metrics| metrics.path),
)
open_reader_source(
self.inline_data,
self.disk.as_ref(),
&self.bucket,
&self.path,
self.offset,
self.length,
self.use_mmap_read,
self.stage_metrics.map(|metrics| metrics.path),
)
.await
}
}
#[allow(clippy::too_many_arguments)]
async fn open_reader_source(
inline_data: Option<Bytes>,
disk: Option<&DiskStore>,
bucket: &str,
path: &str,
offset: usize,
length: usize,
use_mmap_read: bool,
metrics_path: Option<&'static str>,
) -> disk::error::Result<Option<BoxedObjectReader>> {
if let Some(data) = inline_data {
let mut reader = Cursor::new(data);
reader.set_position(u64::try_from(offset).map_err(|_| DiskError::FileCorrupt)?);
Ok(Some(ShardReader::InMemory(reader)))
} else if let Some(disk) = disk {
open_disk_reader(disk, bucket, path, offset, length, use_mmap_read, metrics_path)
.await
.map(Some)
} else {
Ok(None)
}
} else {
Ok(None)
}
}
@@ -623,22 +638,22 @@ async fn create_bitrot_reader_from_bytes_with_stage_metrics(
let reader_construction_start = stage_metrics_enabled.then(Instant::now);
let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone());
let source = BitrotReaderSource {
inline_data,
disk: disk.cloned(),
bucket: bucket.to_string(),
path: path.to_string(),
offset,
length,
use_mmap_read,
stage_metrics,
};
if let Some(metrics) = stage_metrics {
record_get_stage_duration_if_enabled(metrics.path, metrics.reader_construction_stage, reader_construction_start);
}
let file_open_start = stage_metrics_enabled.then(Instant::now);
let reader = source.open().await?;
let reader = open_reader_source(
inline_data,
disk,
bucket,
path,
offset,
length,
use_mmap_read,
stage_metrics.map(|metrics| metrics.path),
)
.await?;
if let Some(metrics) = stage_metrics {
record_get_stage_duration_if_enabled(metrics.path, metrics.file_open_stage, file_open_start);
}
@@ -698,11 +713,12 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
) -> (BitrotReader<ShardReader>, DeferredReaderStripeHandle) {
let stripe_stride = shard_size + checksum_algo.size();
let (offset, length) = bitrot_encoded_range(offset, length, shard_size, checksum_algo.clone());
let inline_source = inline_data.is_some();
let source = BitrotReaderSource {
inline_data,
disk,
bucket: bucket.to_string(),
path: path.to_string(),
bucket: if inline_source { String::new() } else { bucket.to_string() },
path: if inline_source { String::new() } else { path.to_string() },
offset,
length,
use_mmap_read,
crates/ecstore/src/layout/format.rs
+26 -6
View File
@@ -234,11 +234,17 @@ mod test {
#[test]
fn test_format_v1() {
// A freshly created format must survive a serialize -> parse roundtrip
// unchanged (identity on every on-disk field).
let format = FormatV3::new(1, 4);
let serialized = serde_json::to_string(&format).expect("FormatV3 must serialize to JSON");
let reparsed = FormatV3::try_from(serialized.as_str()).expect("serialized FormatV3 must parse back");
assert_eq!(reparsed, format);
let str = serde_json::to_string(&format);
println!("{str:?}");
// minio-file-format-compat: this literal pins the on-disk format.json
// shape (erasure version "1", distributionAlgo "CRCMOD"). `this` always
// carries the disk's own UUID in real format.json files; a JSON null
// there was never parseable and never written by MinIO or RustFS.
let data = r#"
{
"version": "1",
@@ -246,7 +252,7 @@ mod test {
"id": "321b3874-987d-4c15-8fa5-757c956b1243",
"xl": {
"version": "1",
"this": null,
"this": "8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
"sets": [
[
"8ab9a908-f869-4f1f-8e42-eb067ffa7eb5",
@@ -259,9 +265,23 @@ mod test {
}
}"#;
let p = FormatV3::try_from(data);
let parsed = FormatV3::try_from(data).expect("pinned v1 format.json literal must keep parsing");
println!("{p:?}");
assert_eq!(parsed.version, FormatMetaVersion::V1);
assert_eq!(parsed.format, FormatBackend::Erasure);
assert_eq!(
parsed.id,
Uuid::parse_str("321b3874-987d-4c15-8fa5-757c956b1243").expect("literal id is a valid UUID")
);
assert_eq!(parsed.erasure.version, FormatErasureVersion::V1);
assert_eq!(
parsed.erasure.this,
Uuid::parse_str("8ab9a908-f869-4f1f-8e42-eb067ffa7eb5").expect("literal this is a valid UUID")
);
assert_eq!(parsed.erasure.sets.len(), 1);
assert_eq!(parsed.erasure.sets[0].len(), 4);
assert_eq!(parsed.erasure.sets[0][0], parsed.erasure.this);
assert_eq!(parsed.erasure.distribution_algo, DistributionAlgoVersion::V1);
}
#[test]
crates/ecstore/src/set_disk/core/io_primitives.rs
+100 -39
View File
@@ -58,6 +58,7 @@ use crate::io_support::bitrot::{
create_deferred_bitrot_reader_with_stripe_handle, object_mmap_read_enabled, object_mmap_read_max_length,
};
use crate::set_disk::shard_source::ShardReadCost;
use futures::FutureExt as _;
use futures::stream::{FuturesUnordered, StreamExt};
use metrics::counter;
use std::{
@@ -221,7 +222,7 @@ impl MetadataFanoutDiagnostics {
self.observations.iter().filter(|observation| observation.ignored).count()
}
pub(in crate::set_disk) fn error_responses(&self) -> usize {
pub(in crate::set_disk) fn non_valid_responses(&self) -> usize {
self.total_responses().saturating_sub(self.valid_responses())
}
@@ -272,7 +273,7 @@ impl MetadataFanoutDiagnostics {
self.total_responses(),
self.valid_responses(),
self.ignored_responses(),
self.error_responses(),
self.non_valid_responses(),
);
for observation in &self.observations {
rustfs_io_metrics::record_get_object_metadata_response(path, observation.outcome);
@@ -2863,8 +2864,6 @@ impl SetDisks {
file_info.validate_for_erasure_write()?;
}
}
let mut futures = Vec::with_capacity(disks.len());
let mut errs = Vec::with_capacity(disks.len());
let src_bucket = Arc::new(src_bucket.to_string());
@@ -2872,48 +2871,65 @@ impl SetDisks {
let dst_bucket = Arc::new(dst_bucket.to_string());
let dst_object = Arc::new(dst_object.to_string());
for (i, (disk, file_info)) in disks.iter().zip(file_infos.iter()).enumerate() {
let mut file_info = file_info.clone();
let disk = disk.clone();
let src_bucket = src_bucket.clone();
let src_object = src_object.clone();
let dst_object = dst_object.clone();
let dst_bucket = dst_bucket.clone();
let disk_count = disks.len();
let fanout_disks = disks.to_vec();
let fanout_file_infos = file_infos.to_vec();
let fanout_src_bucket = src_bucket.clone();
let fanout_src_object = src_object.clone();
let fanout_dst_bucket = dst_bucket.clone();
let fanout_dst_object = dst_object.clone();
// Keep one coordinator task so a cancelled caller cannot drop partially
// completed disk mutations. Per-disk futures stay ordered in `join_all`,
// preserving slot-indexed quorum and convergence accounting without a
// scheduler task for every disk.
let fanout = tokio::spawn(async move {
let futures = fanout_disks
.into_iter()
.zip(fanout_file_infos)
.enumerate()
.map(|(i, (disk, mut file_info))| {
let src_bucket = fanout_src_bucket.clone();
let src_object = fanout_src_object.clone();
let dst_object = fanout_dst_object.clone();
let dst_bucket = fanout_dst_bucket.clone();
futures.push(tokio::spawn(async move {
// Test-only introspection guard: counts this task as in-flight for
// the whole body. Compiles to `()` in production (no behavior).
#[allow(clippy::let_unit_value)]
let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object);
std::panic::AssertUnwindSafe(async move {
// Test-only introspection guard: counts this operation as
// in-flight for the whole body. Compiles to `()` in production.
#[allow(clippy::let_unit_value)]
let _fanout_task_guard = Self::rename_fanout_task_guard(&dst_object);
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
let Some(disk) = disk else {
return Err(DiskError::DiskNotFound);
};
let is_delete_marker = file_info.is_canonical_delete_marker();
if file_info.erasure.index == 0 {
file_info.erasure.index = i + 1;
}
let is_delete_marker = file_info.is_canonical_delete_marker();
if file_info.erasure.index == 0 {
file_info.erasure.index = i + 1;
}
if !is_delete_marker && !file_info.has_valid_erasure_geometry() {
return Err(DiskError::FileCorrupt);
}
if !is_delete_marker && !file_info.has_valid_erasure_geometry() {
return Err(DiskError::FileCorrupt);
}
// Test-only awaitable pause point right before the disk rename.
// A no-op immediately-ready future in production.
Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await;
// Test-only awaitable pause point right before the disk rename.
// A no-op immediately-ready future in production.
Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await;
disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
.await
}));
}
disk.rename_data(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
.await
})
.catch_unwind()
});
join_all(futures).await
});
let mut disk_versions = vec![None; disks.len()];
let mut data_dirs = vec![None; disks.len()];
let mut cleanup_data_dirs = vec![None; disks.len()];
let mut old_current_sizes = vec![None; disks.len()];
let mut disk_versions = vec![None; disk_count];
let mut data_dirs = vec![None; disk_count];
let mut cleanup_data_dirs = vec![None; disk_count];
let mut old_current_sizes = vec![None; disk_count];
let results = join_all(futures).await;
let results = fanout.await.map_err(|_| DiskError::Unexpected)?;
for (idx, result) in results.iter().enumerate() {
match result.as_ref().map_err(|_| DiskError::Unexpected)? {
@@ -5877,6 +5893,51 @@ mod tests {
drop(dirs);
}
#[tokio::test]
async fn rename_fanout_drains_after_caller_cancellation() {
const DISKS: usize = 4;
let bucket = "rename-cancel-bucket";
let object = "rename-cancel-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
let marker = metadata_test_delete_marker(object, Uuid::new_v4(), OffsetDateTime::now_utc());
let file_infos = vec![marker; DISKS];
let tracker = rename_fanout_barrier::observe_tasks(object);
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
let rename =
tokio::spawn(
async move { SetDisks::rename_data(&disks, bucket, object, &file_infos, bucket, object, DISKS - 1).await },
);
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("rename fan-out must reach the armed barrier");
rename.abort();
assert!(
rename
.await
.expect_err("aborted caller should report cancellation")
.is_cancelled(),
"caller task should be cancelled, not panic"
);
assert!(tracker.running() >= 1, "the coordinator must retain in-flight disk mutations");
barrier.release();
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
while tracker.running() != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("cancelled caller's disk mutations must drain");
for (idx, dir) in dirs.iter().enumerate() {
assert!(
dir.path().join(bucket).join(object).join(STORAGE_FORMAT_FILE).exists(),
"disk {idx} must finish the rename after caller cancellation"
);
}
}
/// Demo / regression guard for the barrier on the commit (old-data-dir)
/// cleanup fan-out. Serves the same #1312/#1319 "no background disk write
/// after release" shape, on the reclamation path that runs *after* a write is
@@ -6047,7 +6108,7 @@ mod tests {
assert_eq!(diagnostics.total_responses(), 3);
assert_eq!(diagnostics.valid_responses(), 1);
assert_eq!(diagnostics.ignored_responses(), 1);
assert_eq!(diagnostics.error_responses(), 2);
assert_eq!(diagnostics.non_valid_responses(), 2);
assert_eq!(diagnostics.first_response_latency(), Some(Duration::from_millis(10)));
assert_eq!(diagnostics.first_valid_response_latency(), Some(Duration::from_millis(30)));
assert_eq!(diagnostics.slowest_response_latency(), Some(Duration::from_millis(30)));
crates/ecstore/src/set_disk/mod.rs
+130 -29
View File
@@ -639,9 +639,11 @@ const ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE: &str = "RUSTFS_GET_CODEC_STREAMING_
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENABLE: bool = true;
const ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_MIN_SIZE";
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: usize = MI_B;
// Meet the direct-memory path at its default ceiling. Codec streaming remains
// rollout-gated and starts where the eager small-object path ends.
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE: usize = DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD;
const ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: &str = "RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE";
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: usize = MI_B;
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE: usize = DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE;
const ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE: &str = "RUSTFS_GET_CODEC_STREAMING_ENGINE";
const DEFAULT_RUSTFS_GET_CODEC_STREAMING_ENGINE: &str = GET_CODEC_STREAMING_ENGINE_LEGACY;
@@ -733,10 +735,41 @@ mod transition_matrix_tests;
pub use ops::heal_walk::HealWalkVersion;
pub(in crate::set_disk) enum GetObjectMetadata<T> {
Owned(T),
Shared(Arc<T>),
}
impl<T> std::ops::Deref for GetObjectMetadata<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
match self {
Self::Owned(value) => value,
Self::Shared(value) => value,
}
}
}
impl<T: Clone> GetObjectMetadata<T> {
fn into_owned(self) -> T {
match self {
Self::Owned(value) => value,
Self::Shared(value) => Arc::try_unwrap(value).unwrap_or_else(|value| (*value).clone()),
}
}
}
type GetObjectFileInfo = (
GetObjectMetadata<FileInfo>,
GetObjectMetadata<Vec<FileInfo>>,
GetObjectMetadata<Vec<Option<DiskStore>>>,
);
pub(crate) struct PreparedGetObjectMetadata {
fi: FileInfo,
files: Vec<FileInfo>,
disks: Vec<Option<DiskStore>>,
fi: GetObjectMetadata<FileInfo>,
files: GetObjectMetadata<Vec<FileInfo>>,
disks: GetObjectMetadata<Vec<Option<DiskStore>>>,
object_info: Option<ObjectInfo>,
}
@@ -805,9 +838,9 @@ mod prepared_get_object_metadata_tests {
#[tokio::test]
async fn prepared_metadata_is_consumed_exactly_once() {
let metadata = PreparedGetObjectMetadata {
fi: FileInfo::default(),
files: Vec::new(),
disks: Vec::new(),
fi: GetObjectMetadata::Owned(FileInfo::default()),
files: GetObjectMetadata::Owned(Vec::new()),
disks: GetObjectMetadata::Owned(Vec::new()),
object_info: None,
};
@@ -2334,6 +2367,8 @@ pub struct SetDisks {
pub default_parity_count: usize,
pub set_index: usize,
pub pool_index: usize,
/// Stable namespace shared by every object lock created for this set.
set_lock_namespace: Arc<str>,
pub format: FormatV3,
disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>,
@@ -2463,13 +2498,13 @@ impl Hash for GetObjectMetadataCacheKey {
}
}
#[derive(Clone, Debug)]
#[derive(Debug)]
struct GetObjectMetadataCacheEntry {
#[allow(dead_code)] // Kept for debugging; moka handles TTL internally
created_at: Instant,
fi: FileInfo,
parts_metadata: Vec<FileInfo>,
online_disks: Vec<Option<DiskStore>>,
fi: Arc<FileInfo>,
parts_metadata: Arc<Vec<FileInfo>>,
online_disks: Arc<Vec<Option<DiskStore>>>,
read_quorum: usize,
}
@@ -2735,6 +2770,7 @@ impl SetDisks {
instance_ctx: Arc<InstanceContext>,
) -> Arc<Self> {
let ctx = instance_ctx;
let set_lock_namespace: Arc<str> = format!("set-{pool_index}-{set_index}").into();
Arc::new(SetDisks {
locker_owner,
disks,
@@ -2742,6 +2778,7 @@ impl SetDisks {
default_parity_count,
set_index,
pool_index,
set_lock_namespace,
format,
set_endpoints,
disk_health_cache: Arc::new(RwLock::new(Vec::new())),
@@ -3190,23 +3227,28 @@ async fn try_read_inline_data_shards_direct(
return None;
}
let mut body = Vec::with_capacity(object_size);
let mut remaining = object_size;
for reader in readers.iter_mut().take(data_shards) {
let shards_needed = object_size.div_ceil(read_length);
if shards_needed > data_shards {
return None;
}
let encoded_capacity = read_length.checked_mul(shards_needed)?;
let mut body = Vec::with_capacity(encoded_capacity);
for reader in readers.iter_mut().take(shards_needed) {
let reader = reader.as_mut()?;
let mut shard = vec![0u8; read_length];
let Ok(read) = reader.read(&mut shard).await else {
let Ok(read) = reader.read_appending(&mut body, read_length).await else {
return None;
};
if read != read_length {
return None;
}
let take = remaining.min(shard.len());
body.extend_from_slice(&shard[..take]);
remaining -= take;
if remaining == 0 {
return Some(Bytes::from(body));
if body.len() >= object_size {
let body = Bytes::from(body);
return Some(if body.len() == object_size {
body
} else {
body.slice(..object_size)
});
}
}
@@ -4934,6 +4976,28 @@ mod tests {
);
}
#[tokio::test]
async fn new_ns_lock_reuses_the_set_namespace_allocation() {
let ctx = Arc::new(InstanceContext::new());
ctx.update_erasure_type(SetupType::Erasure).await;
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
assert_eq!(&*set.set_lock_namespace, "set-0-0");
let before = Arc::strong_count(&set.set_lock_namespace);
let lock = set
.new_ns_lock("bucket", "object")
.await
.expect("namespace lock should be created");
assert_eq!(
Arc::strong_count(&set.set_lock_namespace),
before + 1,
"each lock should share the set namespace instead of formatting a new String"
);
drop(lock);
assert_eq!(Arc::strong_count(&set.set_lock_namespace), before);
}
struct SetupTypeGuard {
previous: SetupType,
}
@@ -8974,10 +9038,17 @@ mod tests {
));
}
async fn inline_bitrot_files_for_payload(payload: &[u8]) -> (coding::Erasure, Vec<FileInfo>, usize, HashAlgorithm) {
let erasure = coding::Erasure::new(4, 2, 1024 * 1024);
async fn inline_bitrot_files_for_payload_with_mode(
payload: &[u8],
uses_legacy: bool,
) -> (coding::Erasure, Vec<FileInfo>, usize, HashAlgorithm) {
let erasure = coding::Erasure::new_with_options(4, 2, 1024 * 1024, uses_legacy);
let read_length = erasure.shard_file_offset(0, payload.len(), payload.len());
let checksum_algo = HashAlgorithm::HighwayHash256S;
let checksum_algo = if uses_legacy {
HashAlgorithm::HighwayHash256SLegacy
} else {
HashAlgorithm::HighwayHash256S
};
let shards = erasure.encode_data(payload).expect("payload should encode");
let mut files = Vec::with_capacity(shards.len());
@@ -8999,6 +9070,10 @@ mod tests {
(erasure, files, read_length, checksum_algo)
}
async fn inline_bitrot_files_for_payload(payload: &[u8]) -> (coding::Erasure, Vec<FileInfo>, usize, HashAlgorithm) {
inline_bitrot_files_for_payload_with_mode(payload, false).await
}
fn inline_data_shard_fileinfo(
name: &str,
data_blocks: usize,
@@ -9078,15 +9153,41 @@ mod tests {
assert_eq!(body.as_ref(), payload);
}
#[tokio::test]
async fn inline_data_shards_direct_read_reassembles_legacy_payload_with_padding() {
let payload = b"legacy inline payload whose size is not divisible by the data shard count";
let (erasure, files, read_length, checksum_algo) = inline_bitrot_files_for_payload_with_mode(payload, true).await;
assert_ne!(payload.len() % erasure.data_shards, 0, "test payload must exercise EC padding");
let mut readers = build_inline_bitrot_readers(
&files,
erasure.data_shards,
"bucket",
"object",
read_length,
erasure.shard_size(),
&checksum_algo,
false,
)
.await
.expect("legacy inline bitrot readers should build");
let body = try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, payload.len())
.await
.expect("legacy data shard direct read should succeed");
assert_eq!(body.len(), payload.len());
assert_eq!(body.as_ref(), payload);
}
#[tokio::test]
async fn inline_data_shards_direct_read_rejects_corrupt_shard() {
let payload = b"small inline object payload that will be corrupted";
let (erasure, mut files, read_length, checksum_algo) = inline_bitrot_files_for_payload(payload).await;
let first = files[0].data.as_mut().expect("first shard should exist");
let mut corrupted = first.to_vec();
let second = files[1].data.as_mut().expect("second shard should exist");
let mut corrupted = second.to_vec();
let last = corrupted.last_mut().expect("encoded shard should not be empty");
*last ^= 0xff;
*first = Bytes::from(corrupted);
*second = Bytes::from(corrupted);
let mut readers = build_inline_bitrot_readers(
&files,
@@ -9103,7 +9204,7 @@ mod tests {
let body = try_read_inline_data_shards_direct(&mut readers, 4, read_length, payload.len()).await;
assert!(body.is_none());
assert!(body.is_none(), "a later corrupt shard must discard the already-appended body prefix");
}
#[test]
crates/ecstore/src/set_disk/ops/locking.rs
+2 -9
View File
@@ -39,16 +39,9 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
// Calculate quorum based on lockers count (majority)
let lockers_count = self.lockers.len();
let write_quorum = if lockers_count > 1 { (lockers_count / 2) + 1 } else { 1 };
NamespaceLock::with_clients_and_quorum(
format!("set-{}-{}", self.pool_index, self.set_index),
self.lockers.clone(),
write_quorum,
)
NamespaceLock::with_clients_and_quorum_shared(self.set_lock_namespace.clone(), self.lockers.clone(), write_quorum)
} else {
NamespaceLock::Local(LocalLock::new(
format!("set-{}-{}", self.pool_index, self.set_index),
self.local_lock_manager.clone(),
))
NamespaceLock::with_local_manager_shared(self.set_lock_namespace.clone(), self.local_lock_manager.clone())
};
let resource = ObjectKey {
crates/ecstore/src/set_disk/ops/multipart.rs
+315 -29
View File
@@ -162,6 +162,22 @@ fn map_upload_id_metadata_error(bucket: &str, object: &str, upload_id: &str, err
err.into()
}
/// Abort a multipart commit when the guard's refresh heartbeat has observed a
/// refresh-quorum loss (backlog#899 Phase 2): a stale holder must not race a
/// concurrent committer past its fenced commit point.
fn fence_commit_on_lock_loss(guard: Option<&ObjectLockDiagGuard>, mode: &'static str, lock_path: &str) -> Result<()> {
if guard.is_some_and(|guard| guard.is_lock_lost()) {
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode,
bucket: RUSTFS_META_MULTIPART_BUCKET.to_string(),
object: lock_path.to_string(),
required: 1,
achieved: 0,
});
}
Ok(())
}
fn multipart_bucket_incarnation_id(metadata: &HashMap<String, String>) -> Result<Option<Uuid>> {
let Some(value) = rustfs_utils::http::metadata_compat::get_consistent_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) else {
if rustfs_utils::http::metadata_compat::contains_key_str(metadata, SUFFIX_BUCKET_INCARNATION_ID) {
@@ -975,12 +991,17 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let write_path = classify_multipart_part_write_path(multipart_part_size, fi.erasure.block_size);
rustfs_io_metrics::record_put_object_path(write_path.multipart_metric_label());
let small_size_hint = if matches!(write_path, SmallWritePath::SingleBlockNonInline) {
usize::try_from(multipart_part_size).map_err(Error::other)?
} else {
0
};
let encode_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
let (reader, w_size) = match write_path {
SmallWritePath::SingleBlockNonInline => {
Arc::clone(&erasure)
.encode_single_block_non_inline(stream, &mut writers, write_quorum)
.encode_single_block_non_inline_with_size_hint(stream, &mut writers, write_quorum, small_size_hint)
.await?
}
SmallWritePath::PipelineBatchedLarge => {
@@ -1087,29 +1108,38 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
}
let part_path = format!("{}/{}/{}", upload_id_path, fi.data_dir.unwrap_or_default(), part_suffix);
let part_lock_path = format!("{upload_id_path}/{part_suffix}");
#[cfg(test)]
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockAcquire).await;
// Serialize only the commit (rename_part), not the whole upload. Each
// concurrent stream writes to its own unique temp dir (see `tmp_part`
// above), so the encode/stream phase never conflicts and must stay
// lock-free — holding a lock across it would serialize slow re-transmits
// of the same part and defeat the S3 "last finisher wins" semantics
// (it also caused UploadPart lock-acquire timeouts). The mixed-generation
// hazard is confined to rename_part, where two temp parts are moved
// cross-disk onto the SAME final part_path: interleaving there can leave
// shards from two generations, each individually bitrot-valid, that only
// surface as silent corruption at read time (backlog#853). A write lock
// scoped to the uploadId namespace makes each commit atomic across disks,
// so the last committer wins consistently. A guarded completion takes
// the object lock before this upload lock to preserve global ordering.
let _upload_commit_guard = if opts.no_lock {
None
// Serialize only same-part commits (rename_part), not the whole upload.
// Each concurrent stream writes to its own unique temp dir (see
// `tmp_part` above), so the encode/stream phase never conflicts and must
// stay lock-free — holding a lock across it would serialize slow
// re-transmits of the same part and defeat the S3 "last finisher wins"
// semantics. The mixed-generation hazard is confined to rename_part,
// where two temp parts are moved cross-disk onto the SAME final
// part_path: interleaving there can leave shards from two generations,
// each individually bitrot-valid, that only surface as silent corruption
// at read time (backlog#853). A write lock scoped to this part number
// makes each same-part commit atomic across disks, so the last committer
// wins consistently, while different part numbers commit onto disjoint
// part paths and stay concurrent (issue#5961 — an uploadId-wide write
// lock serialized them into 503 lock-acquire timeouts). The shared
// uploadId read lock keeps completion/abort (which take the uploadId
// write lock) from racing any in-flight part commit; a guarded
// completion takes the object lock before the upload lock to preserve
// global ordering.
let (_upload_commit_guard, _part_commit_guard) = if opts.no_lock {
(None, None)
} else {
Some(
self.acquire_write_lock_diag("put_object_part_commit", RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
.await?,
)
let upload_guard = self
.acquire_read_lock_diag("put_object_part_commit", RUSTFS_META_MULTIPART_BUCKET, &upload_id_path)
.await?;
let part_guard = self
.acquire_write_lock_diag("put_object_part_commit", RUSTFS_META_MULTIPART_BUCKET, &part_lock_path)
.await?;
(Some(upload_guard), Some(part_guard))
};
let (commit_fi, _) = self.check_upload_id_exists(bucket, object, upload_id, false).await?;
ensure_data_movement_upload_access(&commit_fi, bucket, object, upload_id, opts)?;
@@ -1124,15 +1154,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
.await?;
#[cfg(test)]
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost).await;
if _upload_commit_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "put_object_part_commit",
bucket: RUSTFS_META_MULTIPART_BUCKET.to_string(),
object: upload_id_path.clone(),
required: 1,
achieved: 0,
});
}
fence_commit_on_lock_loss(_upload_commit_guard.as_ref(), "put_object_part_commit", &upload_id_path)?;
fence_commit_on_lock_loss(_part_commit_guard.as_ref(), "put_object_part_commit", &part_lock_path)?;
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
let _ = self
@@ -1156,6 +1179,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
#[cfg(test)]
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterRename).await;
drop(_part_commit_guard);
drop(_upload_commit_guard);
let ret: PartInfo = PartInfo {
@@ -3988,6 +4012,268 @@ mod tests {
.expect("abort should delete the upload after UploadPart releases the lock");
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn put_object_part_different_part_numbers_commit_concurrently() {
use tokio::io::AsyncReadExt as _;
const PART1_SIZE: usize = 5 * 1024 * 1024; // non-final parts must be >= 5MiB to complete
const PART2_SIZE: usize = 4096;
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let locker: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager));
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, vec![locker]).await;
let bucket = "multipart-concurrent-part-numbers-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let upload = set_disks
.new_multipart_upload(bucket, object, &ObjectOptions::default())
.await
.expect("multipart upload should be created");
let upload_id = upload.upload_id;
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
// issue#5961: the barrier releases only once BOTH commits are paused
// inside their commit sections, so reaching wait_until_paused proves the
// two part numbers held their commit locks concurrently. Under an
// uploadId-wide exclusive commit lock the second put errors at the 5s
// lock-acquire timeout instead of arriving, and wait_until_paused fails
// deterministically. No wall-clock bound on the success path.
let barrier = MultipartCommitBarrier::install_for_arrivals(bucket, object, MultipartCommitPause::PutPartAfterRename, 2);
let put1_store = set_disks.clone();
let put1_upload_id = upload_id.clone();
let put1 = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![0x51; PART1_SIZE]);
put1_store
.put_object_part(bucket, object, &put1_upload_id, 1, &mut reader, &ObjectOptions::default())
.await
});
let put2_store = set_disks.clone();
let put2_upload_id = upload_id.clone();
let put2 = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![0x52; PART2_SIZE]);
put2_store
.put_object_part(bucket, object, &put2_upload_id, 2, &mut reader, &ObjectOptions::default())
.await
});
barrier.wait_until_paused().await;
barrier.release();
let part1 = put1
.await
.expect("part 1 task should not panic")
.expect("part 1 should commit after the barrier is released");
let part2 = put2
.await
.expect("part 2 task should not panic")
.expect("part 2 should commit after the barrier is released");
assert_eq!(part1.part_num, 1);
assert_eq!(part2.part_num, 2);
set_disks
.clone()
.complete_multipart_upload(
bucket,
object,
&upload_id,
vec![
CompletePart {
part_num: part1.part_num,
etag: part1.etag.clone(),
..Default::default()
},
CompletePart {
part_num: part2.part_num,
etag: part2.etag.clone(),
..Default::default()
},
],
&ObjectOptions::default(),
)
.await
.expect("completion should succeed with both concurrently committed parts");
let mut reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("completed object should open");
let mut body = Vec::new();
reader
.stream
.read_to_end(&mut body)
.await
.expect("completed object should stream fully");
assert_eq!(body.len(), PART1_SIZE + PART2_SIZE);
assert!(body[..PART1_SIZE].iter().all(|b| *b == 0x51), "part 1 bytes must round-trip");
assert!(body[PART1_SIZE..].iter().all(|b| *b == 0x52), "part 2 bytes must round-trip");
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn put_object_part_same_part_retries_serialize_on_part_lock() {
use tokio::io::AsyncReadExt as _;
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
let bucket = "multipart-same-part-retry-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let upload = set_disks
.new_multipart_upload(bucket, object, &ObjectOptions::default())
.await
.expect("multipart upload should be created");
let upload_id = upload.upload_id;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
let part_lock_path = format!("{upload_id_path}/part.1");
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartAfterRename);
let first_store = set_disks.clone();
let first_upload_id = upload_id.clone();
let first = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![0x53; 4096]);
first_store
.put_object_part(bucket, object, &first_upload_id, 1, &mut reader, &ObjectOptions::default())
.await
});
barrier.wait_until_paused().await;
// The paused commit must hold its part lock EXCLUSIVELY: even a shared
// probe on the part key has to time out. This pins the write-ness of the
// part lock — a shared part lock would let two same-part rename_part
// calls interleave into mixed-generation shards (backlog#853).
let probe = set_disks
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &part_lock_path)
.await
.expect("part namespace lock should be created")
.get_read_lock(Duration::from_secs(1))
.await;
assert!(
probe.is_err(),
"the in-flight part commit must hold an exclusive write lock on its part key"
);
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, part_lock_path));
let retry_store = set_disks.clone();
let retry_upload_id = upload_id.clone();
let retry = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![0x54; 4096]);
retry_store
.put_object_part(bucket, object, &retry_upload_id, 1, &mut reader, &ObjectOptions::default())
.await
});
signaling.wait_for_attempts(1).await;
tokio::task::yield_now().await;
assert!(
!retry.is_finished(),
"a retry of the same part number must wait for the in-flight commit (backlog#853)"
);
barrier.release();
first
.await
.expect("first attempt task should not panic")
.expect("first attempt should commit after the barrier is released");
let retry_part = retry
.await
.expect("retry task should not panic")
.expect("the retry should commit after the first attempt releases the part lock");
set_disks
.clone()
.complete_multipart_upload(
bucket,
object,
&upload_id,
vec![CompletePart {
part_num: retry_part.part_num,
etag: retry_part.etag.clone(),
..Default::default()
}],
&ObjectOptions::default(),
)
.await
.expect("the last committed retry must win the final part generation");
let mut reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("completed object should open");
let mut body = Vec::new();
reader
.stream
.read_to_end(&mut body)
.await
.expect("completed object should stream fully");
assert_eq!(body, vec![0x54; 4096], "the retry's generation must be the one served");
}
#[tokio::test(start_paused = true)]
#[serial]
async fn put_object_part_fences_part_lock_loss_before_rename() {
let target = Arc::new(std::sync::RwLock::new(None));
let refresh_calls = Arc::new(AtomicUsize::new(0));
let lockers: Vec<Arc<dyn LockClient>> = (0..4)
.map(|_| {
Arc::new(SelectiveLockLossClient::new(Arc::clone(&target), Arc::clone(&refresh_calls))) as Arc<dyn LockClient>
})
.collect();
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
let bucket = "multipart-put-part-part-lock-loss-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let upload = set_disks
.new_multipart_upload(bucket, object, &ObjectOptions::default())
.await
.expect("multipart upload should be created");
let upload_id = upload.upload_id;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
let part_lock_path = format!("{upload_id_path}/part.1");
*target.write().expect("lock-loss target should be writable") =
Some(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, part_lock_path.clone()));
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
let put_store = set_disks.clone();
let put_upload_id = upload_id.clone();
let put = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![0x47; 4096]);
put_store
.put_object_part(bucket, object, &put_upload_id, 1, &mut reader, &ObjectOptions::default())
.await
});
barrier.wait_until_paused().await;
tokio::time::advance(Duration::from_secs(11)).await;
tokio::task::yield_now().await;
assert!(
refresh_calls.load(Ordering::Acquire) > 0,
"part lock heartbeat should reach the test client"
);
barrier.release();
let err = put
.await
.expect("UploadPart task should not panic")
.expect_err("UploadPart must fail after losing the part lock");
match err {
StorageError::NamespaceLockQuorumUnavailable {
bucket: lock_bucket,
object: lock_object,
..
} => {
assert_eq!(lock_bucket, RUSTFS_META_MULTIPART_BUCKET);
assert_eq!(lock_object, part_lock_path);
}
other => panic!("unexpected lock-loss error: {other:?}"),
}
let listed = set_disks
.list_object_parts(bucket, object, &upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
.await
.expect("part lock loss before rename must leave the upload readable");
assert!(listed.parts.is_empty(), "part lock loss before rename must not publish the part");
}
#[tokio::test(start_paused = true)]
#[serial]
async fn put_object_part_fences_upload_lock_loss_before_rename() {
crates/ecstore/src/set_disk/ops/object.rs
+254 -30
View File
@@ -56,6 +56,22 @@ use http::HeaderValue;
use rustfs_utils::path::decode_dir_object;
use std::future::Future;
#[inline]
fn duration_millis_f64(duration: std::time::Duration) -> f64 {
duration.as_secs_f64() * 1000.0
}
#[cfg(test)]
mod duration_metrics_tests {
use super::duration_millis_f64;
use std::time::Duration;
#[test]
fn duration_millis_preserves_sub_millisecond_precision() {
assert_eq!(duration_millis_f64(Duration::from_micros(125)), 0.125);
}
}
fn is_restore_control_metadata(key: &str) -> bool {
key.eq_ignore_ascii_case(X_AMZ_RESTORE.as_str())
|| key.eq_ignore_ascii_case(rustfs_utils::http::headers::AMZ_RESTORE_EXPIRY_DAYS)
@@ -734,8 +750,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
0,
object_info.size,
&mut output,
fi,
files,
fi.into_owned(),
files.into_owned(),
&disks,
self.set_index,
self.pool_index,
@@ -851,8 +867,8 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
offset,
length,
&mut writer,
fi,
files,
fi.into_owned(),
files.into_owned(),
&disks,
set_index,
pool_index,
@@ -1107,8 +1123,12 @@ impl SetDisks {
writers.push(w);
errors.push(e);
}
let writer_setup_ms = writer_setup_stage_start.elapsed().as_millis() as u64;
rustfs_io_metrics::record_put_object_stage_duration("set_disk_writer_setup", writer_setup_ms as f64);
let writer_setup_elapsed = writer_setup_stage_start.elapsed();
let writer_setup_ms = writer_setup_elapsed.as_millis() as u64;
rustfs_io_metrics::record_put_object_stage_duration(
"set_disk_writer_setup",
duration_millis_f64(writer_setup_elapsed),
);
let nil_count = errors.iter().filter(|&e| e.is_none()).count();
if nil_count < write_quorum {
@@ -1138,11 +1158,16 @@ impl SetDisks {
let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size);
rustfs_io_metrics::record_put_object_path(write_path.metric_label());
let small_size_hint = if matches!(write_path, SmallWritePath::Inline | SmallWritePath::SingleBlockNonInline) {
usize::try_from(put_object_size).map_err(Error::other)?
} else {
0
};
let encode_stage_start = Instant::now();
let (reader, w_size) = match write_path {
SmallWritePath::Inline => match Arc::clone(&erasure)
.encode_inline_small(stream, &mut writers, write_quorum)
.encode_inline_small_with_size_hint(stream, &mut writers, write_quorum, small_size_hint)
.await
{
Ok((r, w)) => (r, w),
@@ -1152,7 +1177,7 @@ impl SetDisks {
}
},
SmallWritePath::SingleBlockNonInline => match Arc::clone(&erasure)
.encode_single_block_non_inline(stream, &mut writers, write_quorum)
.encode_single_block_non_inline_with_size_hint(stream, &mut writers, write_quorum, small_size_hint)
.await
{
Ok((r, w)) => (r, w),
@@ -1178,8 +1203,9 @@ impl SetDisks {
}
},
};
let encode_ms = encode_stage_start.elapsed().as_millis() as u64;
rustfs_io_metrics::record_put_object_stage_duration("set_disk_encode", encode_ms as f64);
let encode_elapsed = encode_stage_start.elapsed();
let encode_ms = encode_elapsed.as_millis() as u64;
rustfs_io_metrics::record_put_object_stage_duration("set_disk_encode", duration_millis_f64(encode_elapsed));
let _ = mem::replace(&mut data.stream, reader);
// if let Err(err) = close_bitrot_writers(&mut writers).await {
@@ -1497,8 +1523,18 @@ impl SetDisks {
let _ = rustfs_common::heal_channel::send_heal_request(request).await;
});
}
let rename_stage_ms = rename_stage_start.elapsed().as_millis() as u64;
rustfs_io_metrics::record_put_object_stage_duration("set_disk_rename", rename_stage_ms as f64);
let rename_stage_elapsed = rename_stage_start.elapsed();
let rename_stage_ms = rename_stage_elapsed.as_millis() as u64;
self.invalidate_get_object_metadata_cache(bucket, object).await;
// `rename_data` has completed the authoritative quorum commit. The
// exact old-data-dir reclamation below is best-effort space cleanup;
// it must not serialize the next operation on this object.
drop(object_lock_guard);
rustfs_io_metrics::record_put_object_stage_duration("set_disk_rename", duration_millis_f64(rename_stage_elapsed));
if (rename_stage_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS {
warn!(
event = EVENT_SET_DISK_COMMIT_TAIL_SLOW,
@@ -1527,9 +1563,13 @@ impl SetDisks {
let cleanup = self
.commit_rename_data_dir(&cleanup_disks, bucket, object, &old_dir.to_string(), &committed_dir, write_quorum)
.await;
let cleanup_ms = cleanup_stage_start.elapsed().as_millis() as u64;
let cleanup_elapsed = cleanup_stage_start.elapsed();
let cleanup_ms = cleanup_elapsed.as_millis() as u64;
cleanup_stage_ms = Some(cleanup_ms);
rustfs_io_metrics::record_put_object_stage_duration("set_disk_old_data_cleanup", cleanup_ms as f64);
rustfs_io_metrics::record_put_object_stage_duration(
"set_disk_old_data_cleanup",
duration_millis_f64(cleanup_elapsed),
);
self.report_old_data_dir_cleanup(bucket, object, &old_dir.to_string(), &cleanup)
.await;
if (cleanup_ms as u128) >= SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS {
@@ -1550,8 +1590,6 @@ impl SetDisks {
}
}
drop(object_lock_guard); // drop object lock guard to release the lock
for (i, op_disk) in online_disks.iter().enumerate() {
if let Some(disk) = op_disk
&& disk.is_online().await
@@ -1650,10 +1688,6 @@ impl SetDisks {
);
}
if result.is_ok() {
self.invalidate_get_object_metadata_cache(bucket, object).await;
}
if issue3031_diag_enabled() {
warn!(
target: "rustfs_ecstore::set_disk",
@@ -3169,9 +3203,10 @@ impl SetDisks {
// quorum, failing write quorum on update_object_meta (backlog#872).
let mut read_opts = opts.clone();
read_opts.include_part_checksums = true;
let (mut fi, _, disks) = self
let (fi, _, disks) = self
.get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
.await?;
let mut fi = fi.into_owned();
fi.metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_owned());
if let Some(eval_metadata) = &opts.eval_metadata {
@@ -3194,7 +3229,7 @@ impl SetDisks {
});
}
self.update_object_meta(bucket, object, fi.clone(), disks.as_slice()).await?;
self.update_object_meta(bucket, object, fi.clone(), &disks).await?;
Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended))
}
@@ -4619,9 +4654,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let mut transition_read_opts = opts.clone();
transition_read_opts.include_part_checksums = true;
let (mut fi, meta_arr, online_disks) = self
let (fi, meta_arr, online_disks) = self
.get_object_fileinfo(bucket, object, &transition_read_opts, true, false)
.await?;
let mut fi = fi.into_owned();
/*if err != nil {
return Err(to_object_err(err, vec![bucket, object]));
}*/
@@ -4736,7 +4772,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
cloned_fi.size,
&mut writer,
cloned_fi,
meta_arr,
meta_arr.into_owned(),
&online_disks,
set_index,
pool_index,
@@ -4861,7 +4897,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
};
self.invalidate_get_object_metadata_cache(bucket, object).await;
let current = self.get_object_fileinfo(bucket, object, &commit_opts, true, false).await;
let (mut current_fi, _, _) = match current {
let (current_fi, _, _) = match current {
Ok(current) => current,
Err(err) => {
drop(transition_lock_guard);
@@ -4872,6 +4908,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
return Err(err);
}
};
let mut current_fi = current_fi.into_owned();
let source_matches = current_fi.version_id == fi.version_id
&& current_fi.data_dir == fi.data_dir
&& current_fi.mod_time == fi.mod_time
@@ -5596,7 +5633,7 @@ mod get_object_downstream_close_accounting_tests {
let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
let (decode_failures, emit_failures) = metrics::with_local_recorder(&recorder, || {
let (decode_failures, emit_failures, legacy_fanout, internal_fanout) = metrics::with_local_recorder(&recorder, || {
runtime.block_on(async {
let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "get-downstream-close-accounting";
@@ -5664,6 +5701,14 @@ mod get_object_downstream_close_accounting_tests {
("reason", GetObjectFailureReason::DownstreamClosed.as_str()),
],
),
recorder.histogram_values(
"rustfs_io_get_object_metadata_fanout_total_responses",
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)],
),
recorder.histogram_values(
"rustfs_io_get_object_metadata_fanout_total_responses",
&[("path", GET_OBJECT_PATH_INTERNAL_META)],
),
)
})
});
@@ -5671,6 +5716,11 @@ mod get_object_downstream_close_accounting_tests {
assert!(decode_failures > 0, "the producer must expose the downstream close at decode");
assert_eq!(emit_failures, 0, "downstream closure must not be counted as an emit failure");
assert_eq!(legacy_fanout, vec![4.0], "ordinary object fanout must retain the legacy_duplex path");
assert!(
internal_fanout.is_empty(),
"ordinary object fanout must not be attributed to internal_meta"
);
}
#[test]
@@ -5684,7 +5734,7 @@ mod get_object_downstream_close_accounting_tests {
let previous_gate = rustfs_io_metrics::get_stage_metrics_enabled();
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
let (internal_missing, legacy_unknown) = metrics::with_local_recorder(&recorder, || {
let (internal_missing, legacy_unknown, internal_fanout, legacy_fanout) = metrics::with_local_recorder(&recorder, || {
runtime.block_on(async {
let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
let options = ObjectOptions {
@@ -5720,6 +5770,14 @@ mod get_object_downstream_close_accounting_tests {
("reason", GetObjectFailureReason::Unknown.as_str()),
],
),
recorder.histogram_values(
"rustfs_io_get_object_metadata_fanout_error_responses",
&[("path", GET_OBJECT_PATH_INTERNAL_META)],
),
recorder.histogram_values(
"rustfs_io_get_object_metadata_fanout_error_responses",
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)],
),
)
})
});
@@ -5730,6 +5788,8 @@ mod get_object_downstream_close_accounting_tests {
legacy_unknown, 0,
"internal metadata miss must not be attributed to legacy_duplex/unknown"
);
assert_eq!(internal_fanout, vec![4.0], "internal metadata fanout must retain its path label");
assert!(legacy_fanout.is_empty(), "internal metadata fanout must not leak into legacy_duplex");
}
}
@@ -6396,9 +6456,9 @@ mod transition_commit_failure_tests {
cache_key.clone(),
Arc::new(GetObjectMetadataCacheEntry {
created_at: Instant::now(),
fi: fi.clone(),
parts_metadata,
online_disks,
fi: Arc::new((*fi).clone()),
parts_metadata: Arc::new(parts_metadata.into_owned()),
online_disks: Arc::new(online_disks.into_owned()),
read_quorum: 2,
}),
)
@@ -9018,8 +9078,10 @@ mod put_object_tmp_cleanup_tests {
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
use super::*;
use crate::disk::DiskAPI as _;
use crate::set_disk::core::io_primitives::rename_fanout_barrier;
use std::time::Duration;
use tempfile::TempDir;
use tokio::io::AsyncReadExt;
/// Large enough that the erasure shards are written as real tmp files
/// (never inlined into xl.meta), so both tests exercise actual cleanup.
@@ -9104,6 +9166,168 @@ mod put_object_tmp_cleanup_tests {
drop(temp_dirs);
}
#[tokio::test]
async fn committed_put_releases_namespace_lock_before_old_data_cleanup() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "put-commit-lock-window";
let object = "commit-lock-window-object";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut initial_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]);
set_disks
.put_object(bucket, object, &mut initial_reader, &ObjectOptions::default())
.await
.expect("initial object should be committed");
let mut initial = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("initial object should populate the metadata cache");
let mut initial_body = Vec::new();
initial
.stream
.read_to_end(&mut initial_body)
.await
.expect("initial body should drain");
assert_eq!(initial_body, vec![b'0'; TEST_OBJECT_SIZE]);
let cleanup_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_CLEANUP);
let first_store = Arc::clone(&set_disks);
let first = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
first_store
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
});
tokio::time::timeout(Duration::from_secs(30), cleanup_barrier.wait_until_paused())
.await
.expect("first overwrite should reach old-data cleanup");
let mut committed = tokio::time::timeout(
Duration::from_secs(30),
set_disks.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()),
)
.await
.expect("GET should not wait for old-data cleanup")
.expect("committed overwrite should be readable during old-data cleanup");
let mut committed_body = Vec::new();
committed
.stream
.read_to_end(&mut committed_body)
.await
.expect("committed overwrite body should drain");
assert_eq!(committed_body, vec![b'1'; TEST_OBJECT_SIZE]);
let second_commit_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
let second_store = Arc::clone(&set_disks);
let second = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]);
second_store
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
});
tokio::time::timeout(Duration::from_secs(30), second_commit_barrier.wait_until_paused())
.await
.expect("second overwrite should acquire the namespace lock during cleanup");
cleanup_barrier.release();
first
.await
.expect("first overwrite task should join")
.expect("first overwrite should remain successful after cleanup");
drop(cleanup_barrier);
second_commit_barrier.release();
second
.await
.expect("second overwrite task should join")
.expect("second overwrite should commit after acquiring the released namespace lock");
let mut reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("the latest overwrite should be readable");
let mut body = Vec::new();
reader.stream.read_to_end(&mut body).await.expect("latest body should drain");
assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
}
#[tokio::test]
async fn cancelled_post_commit_cleanup_does_not_retain_namespace_lock() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "put-commit-lock-cancelled-cleanup";
let object = "commit-lock-cancelled-cleanup-object";
for disk in &disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut initial_reader = PutObjReader::from_vec(vec![b'0'; TEST_OBJECT_SIZE]);
set_disks
.put_object(bucket, object, &mut initial_reader, &ObjectOptions::default())
.await
.expect("initial object should be committed");
let cleanup_tasks = rename_fanout_barrier::observe_tasks(object);
let cleanup_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_CLEANUP);
let first_store = Arc::clone(&set_disks);
let first = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
first_store
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
});
tokio::time::timeout(Duration::from_secs(30), cleanup_barrier.wait_until_paused())
.await
.expect("first overwrite should reach old-data cleanup");
let second_commit_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
let second_store = Arc::clone(&set_disks);
let second = tokio::spawn(async move {
let mut reader = PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]);
second_store
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
});
tokio::time::timeout(Duration::from_secs(30), second_commit_barrier.wait_until_paused())
.await
.expect("second overwrite should acquire the namespace lock before cancellation");
first.abort();
assert!(
first
.await
.expect_err("the first request should be cancelled during cleanup")
.is_cancelled()
);
assert!(
cleanup_tasks.running() >= 1,
"cancelled cleanup must remain observable until its disk task drains"
);
cleanup_barrier.release();
tokio::time::timeout(Duration::from_secs(30), async {
while cleanup_tasks.running() != 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("cancelled cleanup disk tasks should drain");
drop(cleanup_barrier);
second_commit_barrier.release();
second
.await
.expect("second overwrite task should join")
.expect("second overwrite should survive the earlier request cancellation");
let mut reader = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("the latest overwrite should be readable");
let mut body = Vec::new();
reader.stream.read_to_end(&mut body).await.expect("latest body should drain");
assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
}
#[tokio::test]
async fn put_object_no_lock_aborts_after_outer_namespace_lock_loss() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
crates/ecstore/src/set_disk/read.rs
+127 -69
View File
@@ -30,12 +30,12 @@ use crate::diagnostics::get::{
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY,
GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP,
GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, GET_STAGE_READER_SETUP_DROP_PENDING,
GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT,
GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, GetObjectFailureReason, classify_disk_error,
get_stage_timer_if_enabled, mark_get_object_downstream_closed, record_get_object_pipeline_failure,
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE,
GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, mark_get_object_downstream_closed,
record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
};
use crate::erasure::coding::BitrotReader;
use crate::io_support::bitrot::{
@@ -116,9 +116,9 @@ impl SetDisks {
.then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE)
}
async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option<GetObjectMetadataCacheEntry> {
async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option<Arc<GetObjectMetadataCacheEntry>> {
match self.lookup_cached_get_object_fileinfo(bucket, object).await {
MetadataCacheLookup::Hit(entry) => Some((*entry).clone()),
MetadataCacheLookup::Hit(entry) => Some(entry),
MetadataCacheLookup::Miss | MetadataCacheLookup::RejectedInsufficientQuorum => None,
}
}
@@ -180,9 +180,9 @@ impl SetDisks {
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
let entry = Arc::new(GetObjectMetadataCacheEntry {
created_at: Instant::now(),
fi: fi.clone(),
parts_metadata: parts_metadata.to_vec(),
online_disks: online_disks.to_vec(),
fi: Arc::new(fi.clone()),
parts_metadata: Arc::new(parts_metadata.to_vec()),
online_disks: Arc::new(online_disks.to_vec()),
read_quorum,
});
self.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {})
@@ -257,7 +257,7 @@ impl SetDisks {
opts: &ObjectOptions,
read_data: bool,
caller_allows_early_stop: bool,
) -> Result<(FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>)> {
) -> Result<GetObjectFileInfo> {
self.get_object_fileinfo_gated(bucket, object, opts, read_data, caller_allows_early_stop)
.await
}
@@ -274,7 +274,7 @@ impl SetDisks {
opts: &ObjectOptions,
read_data: bool,
allow_early_stop: bool,
) -> Result<(FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>)> {
) -> Result<GetObjectFileInfo> {
let vid = opts.version_id.clone().unwrap_or_default();
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
@@ -300,7 +300,11 @@ impl SetDisks {
GET_STAGE_METADATA_CACHE_LOOKUP,
metadata_cache_lookup_start,
);
return Ok((cached.fi.clone(), cached.parts_metadata.clone(), cached.online_disks.clone()));
return Ok((
GetObjectMetadata::Shared(Arc::clone(&cached.fi)),
GetObjectMetadata::Shared(Arc::clone(&cached.parts_metadata)),
GetObjectMetadata::Shared(Arc::clone(&cached.online_disks)),
));
}
MetadataCacheLookup::Miss => {
rustfs_io_metrics::record_get_object_metadata_cache_decision(
@@ -349,7 +353,12 @@ impl SetDisks {
self.default_parity_count,
)
.await?;
metadata_fanout_diagnostics.record(GET_OBJECT_PATH_LEGACY_DUPLEX);
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
GET_OBJECT_PATH_INTERNAL_META
} else {
GET_OBJECT_PATH_LEGACY_DUPLEX
};
metadata_fanout_diagnostics.record(metadata_metrics_path);
let metadata_fanout_complete = metadata_fanout_diagnostics.total_responses() >= disks.len();
// warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata);
// warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs);
@@ -396,7 +405,7 @@ impl SetDisks {
rustfs_utils::http::remove_str(&mut metadata.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS);
}
}
metadata_fanout_diagnostics.record_quorum_candidate_latency(GET_OBJECT_PATH_LEGACY_DUPLEX, fileinfo_selection_quorum);
metadata_fanout_diagnostics.record_quorum_candidate_latency(metadata_metrics_path, fileinfo_selection_quorum);
if errs.iter().any(|err| err.is_some()) {
let version_id = resolved_read_repair_version_id(&fi, opts.version_id.as_deref());
submit_read_repair_heal(
@@ -427,7 +436,11 @@ impl SetDisks {
// let online_disks: Vec<Option<DiskStore>> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect();
Ok((fi, parts_metadata, op_online_disks))
Ok((
GetObjectMetadata::Owned(fi),
GetObjectMetadata::Owned(parts_metadata),
GetObjectMetadata::Owned(op_online_disks),
))
}
#[hotpath::measure(impl_type = "SetDisks")]
@@ -2696,6 +2709,39 @@ mod metadata_cache_tests {
assert_eq!(cached.read_quorum, 0);
}
#[tokio::test]
async fn get_object_fileinfo_cache_hit_shares_cached_metadata() {
let set = new_metadata_cache_test_set().await;
let fi = valid_test_fileinfo("object");
let parts_metadata = vec![fi.clone()];
let online_disks = Vec::new();
let generation = set.get_object_metadata_cache_generation("bucket", "object");
set.cache_get_object_fileinfo(("bucket", "object"), generation, &fi, &parts_metadata, &online_disks, 0)
.await;
let cached = set
.cached_get_object_fileinfo("bucket", "object")
.await
.expect("fresh cache entry should be returned");
let (returned_fi, returned_parts_metadata, returned_online_disks) = set
.get_object_fileinfo("bucket", "object", &ObjectOptions::default(), true, false)
.await
.expect("cache-backed metadata lookup should succeed");
assert!(
matches!(returned_fi, GetObjectMetadata::Shared(ref value) if Arc::ptr_eq(value, &cached.fi)),
"cache hits must share FileInfo ownership"
);
assert!(
matches!(returned_parts_metadata, GetObjectMetadata::Shared(ref value) if Arc::ptr_eq(value, &cached.parts_metadata)),
"cache hits must share the metadata vector"
);
assert!(
matches!(returned_online_disks, GetObjectMetadata::Shared(ref value) if Arc::ptr_eq(value, &cached.online_disks)),
"cache hits must share the online-disk vector"
);
}
#[tokio::test]
async fn get_object_metadata_cache_rejects_deleted_and_invalid_fileinfo() {
let set = new_metadata_cache_test_set().await;
@@ -2735,9 +2781,9 @@ mod metadata_cache_tests {
),
Arc::new(GetObjectMetadataCacheEntry {
created_at: Instant::now(),
fi: fi.clone(),
parts_metadata: vec![fi],
online_disks: vec![None],
fi: Arc::new(fi.clone()),
parts_metadata: Arc::new(vec![fi]),
online_disks: Arc::new(vec![None]),
read_quorum: 1,
}),
)
@@ -2751,9 +2797,6 @@ mod metadata_cache_tests {
#[tokio::test]
async fn get_object_metadata_cache_rejects_stale_entries() {
// moka handles TTL expiry automatically via time_to_live(250ms).
// This test verifies that entries inserted with the cache API are retrievable
// while fresh, and that the cache API works correctly.
let set = new_metadata_cache_test_set().await;
let fi = valid_test_fileinfo("object");
@@ -2765,6 +2808,14 @@ mod metadata_cache_tests {
set.cached_get_object_fileinfo("bucket", "object").await.is_some(),
"freshly inserted entry should be returned"
);
tokio::time::timeout(GET_OBJECT_METADATA_CACHE_TTL + Duration::from_secs(1), async {
while set.cached_get_object_fileinfo("bucket", "object").await.is_some() {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("metadata cache entry should expire after its TTL");
}
#[tokio::test]
@@ -2826,9 +2877,13 @@ mod metadata_cache_tests {
barrier.wait_until_paused().await;
set.invalidate_get_object_metadata_cache(bucket, object).await;
barrier.release();
read.await
let (fi, parts_metadata, online_disks) = read
.await
.expect("metadata read task should not panic")
.expect("metadata fanout should still return its selected FileInfo");
assert!(matches!(fi, GetObjectMetadata::Owned(_)));
assert!(matches!(parts_metadata, GetObjectMetadata::Owned(_)));
assert!(matches!(online_disks, GetObjectMetadata::Owned(_)));
assert!(
set.get_object_metadata_cache
@@ -2875,9 +2930,9 @@ mod metadata_cache_tests {
let key = GetObjectMetadataCacheKey::new("bucket", "object", generation);
let entry = Arc::new(GetObjectMetadataCacheEntry {
created_at: Instant::now(),
fi: fi.clone(),
parts_metadata: vec![fi],
online_disks: Vec::new(),
fi: Arc::new(fi.clone()),
parts_metadata: Arc::new(vec![fi]),
online_disks: Arc::new(Vec::new()),
read_quorum: 0,
});
@@ -2985,9 +3040,9 @@ mod metadata_cache_tests {
let entry = |fi: FileInfo| {
Arc::new(GetObjectMetadataCacheEntry {
created_at: Instant::now(),
parts_metadata: vec![fi.clone()],
fi,
online_disks: Vec::new(),
parts_metadata: Arc::new(vec![fi.clone()]),
fi: Arc::new(fi),
online_disks: Arc::new(Vec::new()),
read_quorum: 0,
})
};
@@ -3437,7 +3492,7 @@ mod tests {
);
assert_eq!(diagnostics.total_responses(), 9);
assert_eq!(diagnostics.valid_responses(), 1);
assert_eq!(diagnostics.error_responses(), 8);
assert_eq!(diagnostics.non_valid_responses(), 8);
}
#[test]
@@ -3452,7 +3507,7 @@ mod tests {
);
assert_eq!(diagnostics.ignored_responses(), 2);
assert_eq!(diagnostics.error_responses(), 3);
assert_eq!(diagnostics.non_valid_responses(), 3);
assert_eq!(diagnostics.observations[0].outcome, GET_METADATA_RESPONSE_DISK_NOT_FOUND);
assert_eq!(diagnostics.observations[1].outcome, GET_METADATA_RESPONSE_IGNORED);
assert_eq!(diagnostics.observations[2].outcome, GET_METADATA_RESPONSE_NOT_FOUND);
@@ -3520,7 +3575,7 @@ mod tests {
assert_eq!(diagnostics.total_responses(), 3);
assert_eq!(diagnostics.valid_responses(), 3);
assert_eq!(diagnostics.error_responses(), 0);
assert_eq!(diagnostics.non_valid_responses(), 0);
assert!(
diagnostics
.observations
@@ -5496,33 +5551,36 @@ mod tests {
}
#[test]
fn rustfs_codec_streaming_uses_conservative_default_min_size() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS)),
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")),
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE, None::<&str>),
],
|| {
let below_threshold_fi = codec_streaming_test_fileinfo(512 * 1024, 1);
let below_threshold_object_info = codec_streaming_test_object_info(&below_threshold_fi);
assert_eq!(
codec_streaming_reader_gate_for_test(&None, &below_threshold_object_info, &below_threshold_fi, true).decision,
GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::BelowMinSize)
);
fn codec_streaming_default_min_size_meets_direct_memory_ceiling() {
for engine in [None, Some(GET_CODEC_STREAMING_ENGINE_RUSTFS)] {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, engine),
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("benchmark")),
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE, None::<&str>),
],
|| {
let below_threshold_fi = codec_streaming_test_fileinfo(128 * 1024 - 1, 1);
let below_threshold_object_info = codec_streaming_test_object_info(&below_threshold_fi);
assert_eq!(
codec_streaming_reader_gate_for_test(&None, &below_threshold_object_info, &below_threshold_fi, true)
.decision,
GetCodecStreamingDecision::Fallback(GetCodecStreamingFallbackReason::BelowMinSize)
);
let threshold_fi = codec_streaming_test_fileinfo(1_048_576, 1);
let threshold_object_info = codec_streaming_test_object_info(&threshold_fi);
assert_eq!(
codec_streaming_reader_gate_for_test(&None, &threshold_object_info, &threshold_fi, true).decision,
GetCodecStreamingDecision::Use
);
},
);
let threshold_fi = codec_streaming_test_fileinfo(128 * 1024, 1);
let threshold_object_info = codec_streaming_test_object_info(&threshold_fi);
assert_eq!(
codec_streaming_reader_gate_for_test(&None, &threshold_object_info, &threshold_fi, true).decision,
GetCodecStreamingDecision::Use
);
},
);
}
}
#[test]
@@ -5820,10 +5878,10 @@ mod tests {
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
],
|| {
let fi = codec_streaming_test_fileinfo(1024, 1);
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
let object_info = codec_streaming_test_object_info(&fi);
assert_eq!(
@@ -5844,10 +5902,10 @@ mod tests {
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("on")),
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, None::<&str>),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
],
|| {
let fi = codec_streaming_test_fileinfo(1024, 1);
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
let object_info = codec_streaming_test_object_info(&fi);
assert_eq!(
@@ -5865,10 +5923,10 @@ mod tests {
[
(ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("false")),
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("on")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
],
|| {
let fi = codec_streaming_test_fileinfo(1024, 1);
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
let object_info = codec_streaming_test_object_info(&fi);
assert_eq!(
@@ -5948,10 +6006,10 @@ mod tests {
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, Some("0")),
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
],
|| {
let fi = codec_streaming_test_fileinfo(1024, 1);
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
let object_info = codec_streaming_test_object_info(&fi);
assert_eq!(
@@ -5968,10 +6026,10 @@ mod tests {
(ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT, Some("100")),
(ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
(ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, None::<&str>),
],
|| {
let fi = codec_streaming_test_fileinfo(1024, 1);
let fi = codec_streaming_test_fileinfo(128 * 1024, 1);
let object_info = codec_streaming_test_object_info(&fi);
assert_eq!(
crates/ecstore/src/set_disk/replication.rs
+6 -4
View File
@@ -78,9 +78,10 @@ impl SetDisks {
include_part_checksums: true,
..Default::default()
};
let (mut fi, _, disks) = self
let (fi, _, disks) = self
.get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
.await?;
let mut fi = fi.into_owned();
if let Some(expected_operation_id) = expected_operation_id {
require_restore_operation_id(&fi.metadata, expected_operation_id)?;
}
@@ -102,7 +103,7 @@ impl SetDisks {
bucket,
object,
fi.clone(),
disks.as_slice(),
&disks,
&UpdateMetadataOpts {
replace_user_metadata: true,
..Default::default()
@@ -145,9 +146,10 @@ impl SetDisks {
include_part_checksums: true,
..Default::default()
};
let (mut fi, _, disks) = self
let (fi, _, disks) = self
.get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
.await?;
let mut fi = fi.into_owned();
if let Some(expected_operation_id) = expected_operation_id {
match restore_operation_id_from_metadata(&fi.metadata)? {
Some(actual_operation_id) if actual_operation_id == expected_operation_id => {}
@@ -172,7 +174,7 @@ impl SetDisks {
bucket,
object,
fi,
disks.as_slice(),
&disks,
&UpdateMetadataOpts {
replace_user_metadata: true,
..Default::default()
crates/ecstore/src/set_disk/shard_source.rs
+9 -8
View File
@@ -117,16 +117,17 @@ impl StripeReadState {
Self::from_parts_with_read_costs(shards, errors, &[], read_quorum)
}
pub(crate) fn from_parts_with_read_costs(
shards: Vec<Option<Vec<u8>>>,
errors: Vec<Option<Error>>,
read_costs: &[ShardReadCost],
read_quorum: usize,
) -> Self {
let slot_count = shards.len().max(errors.len());
let mut slots = Vec::with_capacity(slot_count);
pub(crate) fn from_parts_with_read_costs<S, E>(shards: S, errors: E, read_costs: &[ShardReadCost], read_quorum: usize) -> Self
where
S: IntoIterator<Item = Option<Vec<u8>>>,
S::IntoIter: ExactSizeIterator,
E: IntoIterator<Item = Option<Error>>,
E::IntoIter: ExactSizeIterator,
{
let mut shards = shards.into_iter();
let mut errors = errors.into_iter();
let slot_count = shards.len().max(errors.len());
let mut slots = Vec::with_capacity(slot_count);
for index in 0..slot_count {
let read_cost = read_costs.get(index).copied().unwrap_or(ShardReadCost::Unknown);
slots.push(ShardSlot::with_read_cost(
crates/ecstore/src/store/list_objects.rs
+28 -288
View File
@@ -309,9 +309,17 @@ const ENV_API_LIST_OBJECTS_INDEX_PROVIDER: &str = "RUSTFS_LIST_OBJECTS_INDEX_PRO
const ENV_API_LIST_OBJECTS_INDEX_PROVIDER_PATH: &str = "RUSTFS_LIST_OBJECTS_INDEX_PROVIDER_PATH";
const ENV_API_LIST_OBJECTS_INDEX_PROVIDER_GENERATION: &str = "RUSTFS_LIST_OBJECTS_INDEX_PROVIDER_GENERATION";
const ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_PATH: &str = "RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_PATH";
// The chaos machinery below is compiled only for tests and the opt-in
// `list-chaos` feature (backlog#1832): a production binary without the
// feature carries no chaos symbols, so the two env vars cannot silently
// rewrite a bucket's namespace-journal state.
#[cfg(any(test, feature = "list-chaos"))]
const ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED: &str = "RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED";
#[cfg(any(test, feature = "list-chaos"))]
const ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET: &str = "RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET";
#[cfg(any(test, feature = "list-chaos"))]
const ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_SEQUENCE: &str = "RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_SEQUENCE";
#[cfg(any(test, feature = "list-chaos"))]
const ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS: &str = "RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS";
const ENV_API_LIST_OBJECTS_METADATA_FAST_ENABLED: &str = "RUSTFS_LIST_OBJECTS_METADATA_FAST_ENABLED";
const ENV_API_LIST_OBJECTS_METADATA_FAST_STALENESS_MS: &str = "RUSTFS_LIST_OBJECTS_METADATA_FAST_STALENESS_MS";
@@ -552,7 +560,9 @@ static LIST_OBJECTS_MUTATION_SEQUENCE: AtomicU64 = AtomicU64::new(0);
static SCANNER_NAMESPACE_MUTATION_GENERATION: AtomicU64 = AtomicU64::new(0);
static LIST_OBJECTS_BUCKET_MUTATION_SEQUENCE: OnceCell<RwLock<HashMap<String, u64>>> = OnceCell::const_new();
static LIST_OBJECTS_NAMESPACE_JOURNAL_DEGRADED_BUCKETS: OnceCell<RwLock<HashSet<String>>> = OnceCell::const_new();
#[cfg(any(test, feature = "list-chaos"))]
static LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_CONFIG: OnceCell<Option<NamespaceMutationJournalChaosConfig>> = OnceCell::const_new();
#[cfg(any(test, feature = "list-chaos"))]
static LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_APPLIED: OnceCell<RwLock<HashSet<String>>> = OnceCell::const_new();
async fn persistent_key_only_index_cache() -> &'static RwLock<Option<PersistentKeyOnlyIndexCache>> {
@@ -579,6 +589,7 @@ async fn list_objects_namespace_journal_degraded_buckets() -> &'static RwLock<Ha
.await
}
#[cfg(any(test, feature = "list-chaos"))]
async fn list_objects_namespace_journal_chaos_config() -> Option<&'static NamespaceMutationJournalChaosConfig> {
LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_CONFIG
.get_or_init(|| async { namespace_mutation_journal_chaos_config_from_env() })
@@ -586,6 +597,7 @@ async fn list_objects_namespace_journal_chaos_config() -> Option<&'static Namesp
.as_ref()
}
#[cfg(any(test, feature = "list-chaos"))]
async fn list_objects_namespace_journal_chaos_applied() -> &'static RwLock<HashSet<String>> {
LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_APPLIED
.get_or_init(|| async { RwLock::new(HashSet::new()) })
@@ -681,6 +693,7 @@ enum NamespaceMutationJournalStatus {
}
impl NamespaceMutationJournalStatus {
#[cfg(any(test, feature = "list-chaos"))]
fn from_env_value(value: &str) -> Option<Self> {
if value.eq_ignore_ascii_case(LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_HEALTHY) {
Some(Self::Healthy)
@@ -691,6 +704,7 @@ impl NamespaceMutationJournalStatus {
}
}
#[cfg(any(test, feature = "list-chaos"))]
fn env_value(self) -> &'static str {
match self {
Self::Healthy => LIST_OBJECTS_NAMESPACE_JOURNAL_STATUS_HEALTHY,
@@ -712,6 +726,7 @@ struct NamespaceMutationJournalSnapshot {
degraded: bool,
}
#[cfg(any(test, feature = "list-chaos"))]
#[derive(Debug, Clone, PartialEq, Eq)]
struct NamespaceMutationJournalChaosConfig {
bucket: String,
@@ -795,30 +810,35 @@ fn list_objects_namespace_journal_root_from_env() -> Option<PathBuf> {
.filter(|path| !path.as_os_str().is_empty())
}
#[cfg(any(test, feature = "list-chaos"))]
fn namespace_mutation_journal_chaos_enabled_from_env() -> bool {
std::env::var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_ENABLED)
.ok()
.is_some_and(|value| value == "1" || value.eq_ignore_ascii_case("on") || value.eq_ignore_ascii_case("true"))
}
#[cfg(any(test, feature = "list-chaos"))]
fn namespace_mutation_journal_chaos_bucket_from_env() -> Option<String> {
std::env::var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_BUCKET)
.ok()
.filter(|bucket| !bucket.is_empty())
}
#[cfg(any(test, feature = "list-chaos"))]
fn namespace_mutation_journal_chaos_sequence_from_env() -> Option<u64> {
std::env::var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_SEQUENCE)
.ok()
.and_then(|value| value.parse::<u64>().ok())
}
#[cfg(any(test, feature = "list-chaos"))]
fn namespace_mutation_journal_chaos_status_from_env() -> Option<NamespaceMutationJournalStatus> {
std::env::var(ENV_API_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_STATUS)
.ok()
.and_then(|value| NamespaceMutationJournalStatus::from_env_value(&value))
}
#[cfg(any(test, feature = "list-chaos"))]
fn namespace_mutation_journal_chaos_config_from_env() -> Option<NamespaceMutationJournalChaosConfig> {
if !namespace_mutation_journal_chaos_enabled_from_env() {
return None;
@@ -846,6 +866,7 @@ fn namespace_mutation_journal_chaos_config_from_env() -> Option<NamespaceMutatio
})
}
#[cfg(any(test, feature = "list-chaos"))]
fn namespace_mutation_journal_chaos_applied_key(bucket: &str, status: NamespaceMutationJournalStatus) -> String {
let mut key = String::with_capacity(bucket.len() + 1 + status.env_value().len());
key.push_str(bucket);
@@ -854,6 +875,13 @@ fn namespace_mutation_journal_chaos_applied_key(bucket: &str, status: NamespaceM
key
}
/// Production no-op twin of the chaos injector: without `list-chaos` the
/// injection point compiles to nothing (backlog#1832).
#[cfg(not(any(test, feature = "list-chaos")))]
#[inline]
async fn maybe_apply_system_namespace_mutation_journal_chaos(_store: &ECStore, _bucket: &str, _default_sequence: u64) {}
#[cfg(any(test, feature = "list-chaos"))]
async fn maybe_apply_system_namespace_mutation_journal_chaos(store: &ECStore, bucket: &str, default_sequence: u64) {
let Some(config) = list_objects_namespace_journal_chaos_config().await else {
return;
@@ -9529,294 +9557,6 @@ mod test {
.expect("a partial outage with a healthy set must not fail the walk");
}
// use std::sync::Arc;
// use crate::cache_value::metacache_set::list_path_raw;
// use crate::cache_value::metacache_set::ListPathRawOptions;
// use crate::disk::endpoint::Endpoint;
// use crate::disk::error::is_err_eof;
// use crate::disk::format::FormatV3;
// use crate::disk::new_disk;
// use crate::disk::DiskAPI;
// use crate::disk::DiskOption;
// use crate::disk::MetaCacheEntries;
// use crate::disk::MetaCacheEntry;
// use crate::disk::WalkDirOptions;
// use crate::layout::endpoints::EndpointServerPools;
// use crate::error::Error;
// use crate::metacache::writer::MetacacheReader;
// use crate::set_disk::SetDisks;
// use crate::store::list_objects::ListPathOptions;
// use crate::store::list_objects::WalkOptions;
// use crate::store::list_objects::WalkVersionsSortOrder;
// use futures::future::join_all;
// use rustfs_lock::namespace_lock::NsLockMap;
// use tokio::sync::broadcast;
// use tokio::sync::mpsc;
// use tokio::sync::RwLock;
// use uuid::Uuid;
// #[tokio::test]
// async fn test_walk_dir() {
// let mut ep = Endpoint::try_from("/Users/weisd/project/weisd/s3-rustfs/target/volume/test").unwrap();
// ep.pool_idx = 0;
// ep.set_idx = 0;
// ep.disk_idx = 0;
// ep.is_local = true;
// let disk = new_disk(&ep, &DiskOption::default()).await.expect("init disk fail");
// // let disk = match LocalDisk::new(&ep, false).await {
// // Ok(res) => res,
// // Err(err) => {
// // println!("LocalDisk::new err {:?}", err);
// // return;
// // }
// // };
// let (rd, mut wr) = tokio::io::duplex(64);
// let job = tokio::spawn(async move {
// let opts = WalkDirOptions {
// bucket: "dada".to_owned(),
// base_dir: "".to_owned(),
// recursive: true,
// ..Default::default()
// };
// println!("walk opts {:?}", opts);
// if let Err(err) = disk.walk_dir(opts, &mut wr).await {
// println!("walk_dir err {:?}", err);
// }
// });
// let job2 = tokio::spawn(async move {
// let mut mrd = MetacacheReader::new(rd);
// loop {
// match mrd.peek().await {
// Ok(res) => {
// if let Some(info) = res {
// println!("info {:?}", info.name)
// } else {
// break;
// }
// }
// Err(err) => {
// if is_err_eof(&err) {
// break;
// }
// println!("get err {:?}", err);
// break;
// }
// }
// }
// });
// join_all(vec![job, job2]).await;
// }
// #[tokio::test]
// async fn test_list_path_raw() {
// let mut ep = Endpoint::try_from("/Users/weisd/project/weisd/s3-rustfs/target/volume/test").unwrap();
// ep.pool_idx = 0;
// ep.set_idx = 0;
// ep.disk_idx = 0;
// ep.is_local = true;
// let disk = new_disk(&ep, &DiskOption::default()).await.expect("init disk fail");
// // let disk = match LocalDisk::new(&ep, false).await {
// // Ok(res) => res,
// // Err(err) => {
// // println!("LocalDisk::new err {:?}", err);
// // return;
// // }
// // };
// let (_, rx) = broadcast::channel(1);
// let bucket = "dada".to_owned();
// let forward_to = None;
// let disks = vec![Some(disk)];
// let fallback_disks = Vec::new();
// list_path_raw(
// rx,
// ListPathRawOptions {
// disks,
// fallback_disks,
// bucket,
// path: "".to_owned(),
// recursice: true,
// forward_to,
// min_disks: 1,
// report_not_found: false,
// agreed: Some(Box::new(move |entry: MetaCacheEntry| {
// Box::pin(async move { println!("get entry: {}", entry.name) })
// })),
// partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<Error>]| {
// Box::pin(async move { println!("get entries: {:?}", entries) })
// })),
// finished: None,
// ..Default::default()
// },
// )
// .await
// .unwrap();
// }
// #[tokio::test]
// async fn test_set_list_path() {
// let mut ep = Endpoint::try_from("/Users/weisd/project/weisd/s3-rustfs/target/volume/test").unwrap();
// ep.pool_idx = 0;
// ep.set_idx = 0;
// ep.disk_idx = 0;
// ep.is_local = true;
// let disk = new_disk(&ep, &DiskOption::default()).await.expect("init disk fail");
// let _ = disk.set_disk_id(Some(Uuid::new_v4())).await;
// let set = SetDisks {
// lockers: Vec::new(),
// locker_owner: String::new(),
// ns_mutex: Arc::new(RwLock::new(NsLockMap::new(false))),
// disks: RwLock::new(vec![Some(disk)]),
// set_endpoints: Vec::new(),
// set_drive_count: 1,
// default_parity_count: 0,
// set_index: 0,
// pool_index: 0,
// format: FormatV3::new(1, 1),
// };
// let (_tx, rx) = broadcast::channel(1);
// let bucket = "dada".to_owned();
// let opts = ListPathOptions {
// bucket,
// recursive: true,
// ..Default::default()
// };
// let (sender, mut recv) = mpsc::channel(10);
// set.list_path(rx, opts, sender).await.unwrap();
// while let Some(entry) = recv.recv().await {
// println!("get entry {:?}", entry.name)
// }
// }
// #[tokio::test]
//walk() {
// let server_address = "localhost:9000";
// let (endpoint_pools, _setup_type) = EndpointServerPools::from_volumes(
// server_address,
// vec!["/Users/weisd/project/weisd/s3-rustfs/target/volume/test".to_string()],
// )
// .unwrap();
// let store = ECStore::new(server_address.to_string(), endpoint_pools.clone())
// .await
// .unwrap();
// let (_tx, rx) = broadcast::channel(1);
// let bucket = "dada".to_owned();
// let opts = ListPathOptions {
// bucket,
// recursive: true,
// ..Default::default()
// };
// let (sender, mut recv) = mpsc::channel(10);
// store.list_merged(rx, opts, sender).await.unwrap();
// while let Some(entry) = recv.recv().await {
// println!("get entry {:?}", entry.name)
// }
// }
// #[tokio::test]
// async fn test_list_path() {
// let server_address = "localhost:9000";
// let (endpoint_pools, _setup_type) = EndpointServerPools::from_volumes(
// server_address,
// vec!["/Users/weisd/project/weisd/s3-rustfs/target/volume/test".to_string()],
// )
// .unwrap();
// let store = ECStore::new(server_address.to_string(), endpoint_pools.clone())
// .await
// .unwrap();
// let bucket = "dada".to_owned();
// let opts = ListPathOptions {
// bucket,
// recursive: true,
// limit: 100,
// ..Default::default()
// };
// let ret = store.list_path(&opts).await.unwrap();
// println!("ret {:?}", ret);
// }
// #[tokio::test]
// async fn test_list_objects_v2() {
// let server_address = "localhost:9000";
// let (endpoint_pools, _setup_type) = EndpointServerPools::from_volumes(
// server_address,
// vec!["/Users/weisd/project/weisd/s3-rustfs/target/volume/test".to_string()],
// )
// .unwrap();
// let store = ECStore::new(server_address.to_string(), endpoint_pools.clone())
// .await
// .unwrap();
// let ret = store.list_objects_v2("data", "", "", "", 100, false, "").await.unwrap();
// println!("ret {:?}", ret);
// }
// #[tokio::test]
// async fn test_walk() {
// let server_address = "localhost:9000";
// let (endpoint_pools, _setup_type) = EndpointServerPools::from_volumes(
// server_address,
// vec!["/Users/weisd/project/weisd/s3-rustfs/target/volume/test".to_string()],
// )
// .unwrap();
// let store = ECStore::new(server_address.to_string(), endpoint_pools.clone())
// .await
// .unwrap();
// ECStore::init(store.clone()).await.unwrap();
// let (_tx, rx) = broadcast::channel(1);
// let bucket = ".rustfs.sys";
// let prefix = "config/iam/sts/";
// let (sender, mut recv) = mpsc::channel(10);
// let opts = WalkOptions::default();
// store.walk(rx, bucket, prefix, sender, opts).await.unwrap();
// while let Some(entry) = recv.recv().await {
// println!("get entry {:?}", entry)
// }
// }
#[tokio::test]
async fn merge_entry_channels_produces_sorted_unique_output_from_two_channels() {
let (tx_a, rx_a) = mpsc::channel(4);
crates/ecstore/src/store/object.rs
+3 -3
View File
@@ -1781,7 +1781,7 @@ impl ECStore {
) -> Result<GetObjectReader> {
check_get_obj_args(bucket, object)?;
let object = encode_dir_object(object);
let object = rustfs_utils::path::encode_dir_object_ref(object);
let mut opts = opts.clone();
let read_lock_guard = self
.acquire_object_read_lock_if_needed("get_object", bucket, &object, &mut opts)
@@ -1789,14 +1789,14 @@ impl ECStore {
let reader = if self.single_pool() {
self.pools[0]
.get_object_reader(bucket, object.as_str(), range, h, &opts)
.get_object_reader(bucket, object.as_ref(), range, h, &opts)
.await?
} else {
let (_, idx) = self
.get_latest_accessible_object_info_with_idx(bucket, &object, &opts)
.await?;
self.pools[idx]
.get_object_reader(bucket, object.as_str(), range, h, &opts)
.get_object_reader(bucket, object.as_ref(), range, h, &opts)
.await?
};
crates/filemeta/src/fileinfo.rs
+3 -2
View File
@@ -288,9 +288,10 @@ pub struct FileInfo {
/// Values of these keys must never reach logs at any level.
fn is_sensitive_metadata_key(key: &str) -> bool {
// `is_encryption_metadata_key` covers the x-minio-internal- SSE prefix but not
// its x-rustfs-internal- twin, which the dual-key invariant writes alongside it.
// its reserved x-rustfs-internal- twin, which has no writer today but must
// stay redacted in case one appears.
is_encryption_metadata_key(key)
|| starts_with_ignore_ascii_case(key, "x-rustfs-internal-server-side-encryption-")
|| starts_with_ignore_ascii_case(key, rustfs_utils::http::RUSTFS_INTERNAL_ENCRYPTION_PREFIX)
|| rustfs_utils::http::REPLICATION_SSE_TRANSPORT_PREFIXES
.iter()
.any(|prefix| starts_with_ignore_ascii_case(key, prefix))
crates/heal/src/heal/channel.rs
+56 -5
View File
@@ -785,9 +785,15 @@ mod tests {
let heal_manager = create_test_heal_manager();
let processor = HealChannelProcessor::new(heal_manager);
// Verify processor is created successfully
let _sender = processor.get_response_sender();
// If we can get the sender, processor was created correctly
let sender = processor.get_response_sender();
sender
.send(HealChannelResponse {
request_id: "request-id".to_string(),
success: true,
data: None,
error: None,
})
.expect("a freshly constructed processor must accept responses on its channel");
}
#[test]
@@ -1778,9 +1784,22 @@ mod tests {
}
#[tokio::test]
async fn test_process_cancel_request_treats_unknown_path_as_stopped() {
async fn test_process_cancel_request_cancels_cluster_task_for_legacy_root_path() {
let heal_manager = create_test_heal_manager();
let processor = HealChannelProcessor::new(heal_manager);
let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High);
let cluster_task_id = cluster_request.id.clone();
let bucket_request = HealRequest::bucket("bucket".to_string());
let bucket_task_id = bucket_request.id.clone();
heal_manager
.submit_heal_request(cluster_request)
.await
.expect("cluster request should be accepted");
heal_manager
.submit_heal_request(bucket_request)
.await
.expect("bucket request should be accepted");
let processor = HealChannelProcessor::new(heal_manager.clone());
let (tx, rx) = oneshot::channel();
processor
@@ -1796,6 +1815,38 @@ mod tests {
assert_eq!(response.request_id, ".");
assert_eq!(response.data.as_deref(), Some("stopped".as_bytes()));
assert!(response.error.is_none());
assert!(matches!(
heal_manager.get_task_status(&cluster_task_id).await,
Err(crate::Error::TaskNotFound { .. })
));
assert_eq!(
heal_manager
.get_task_status(&bucket_task_id)
.await
.expect("bucket request should not match the root path"),
HealTaskStatus::Pending
);
}
#[tokio::test]
async fn test_process_cancel_request_treats_unknown_path_as_stopped() {
let heal_manager = create_test_heal_manager();
let processor = HealChannelProcessor::new(heal_manager);
let (tx, rx) = oneshot::channel();
processor
.process_cancel_request("missing".to_string(), String::new(), tx)
.await
.expect("cancel should process");
let response = rx
.await
.expect("oneshot should resolve")
.expect("cancel response should be returned");
assert!(response.success);
assert_eq!(response.request_id, "missing");
assert_eq!(response.data.as_deref(), Some("stopped".as_bytes()));
assert!(response.error.is_none());
}
#[tokio::test]
crates/heal/src/heal/manager.rs
+13 -1
View File
@@ -52,6 +52,7 @@ const EVENT_HEAL_MAINLINE_THROTTLE: &str = "heal_mainline_throttle";
const EVENT_HEAL_SCHEDULER_STATE: &str = "heal_scheduler_state";
const EVENT_HEAL_QUEUE_STATE: &str = "heal_queue_state";
const EVENT_HEAL_UNCLEAN_SHUTDOWN: &str = "heal_unclean_shutdown";
const LEGACY_ROOT_HEAL_PATH: &str = ".";
const MAX_RECOVERABLE_HEAL_RETRIES: u32 = 3;
const MAX_RECOVERABLE_HEAL_RETRY_DELAY: Duration = Duration::from_secs(30);
@@ -601,7 +602,7 @@ impl RetryingHeal {
fn heal_type_matches_path(heal_type: &HealType, heal_path: &str) -> bool {
let heal_path = heal_path.trim_matches('/');
if heal_path.is_empty() {
if heal_path.is_empty() || heal_path == LEGACY_ROOT_HEAL_PATH {
return matches!(heal_type, HealType::Cluster);
}
@@ -5110,6 +5111,17 @@ mod tests {
assert!(manager.retrying_heals.lock().await.get(&bucket_request_id).is_some());
}
#[test]
fn test_heal_type_matches_path_accepts_legacy_root() {
assert!(heal_type_matches_path(&HealType::Cluster, LEGACY_ROOT_HEAL_PATH));
assert!(!heal_type_matches_path(
&HealType::Bucket {
bucket: "bucket".to_string(),
},
LEGACY_ROOT_HEAL_PATH,
));
}
#[tokio::test]
async fn test_retrying_duplicate_token_can_query_and_cancel_original_retry() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
crates/iam/src/error.rs
+60 -49
View File
@@ -14,19 +14,23 @@
use crate::IamStorageError;
use rustfs_policy::policy::Error as PolicyError;
use std::sync::Arc;
pub type Result<T> = core::result::Result<T, Error>;
#[derive(thiserror::Error, Debug)]
pub enum Error {
#[error(transparent)]
PolicyError(#[from] PolicyError),
// Arc payloads keep Clone variant-preserving for the non-cloneable inner
// errors (backlog#1831 PR2). Display is unchanged; the source() chain is
// not forwarded (Arc<E> does not implement std::error::Error).
#[error("{0}")]
PolicyError(Arc<PolicyError>),
#[error("{0}")]
StringError(String),
#[error("crypto: {0}")]
CryptoError(#[from] rustfs_crypto::Error),
CryptoError(Arc<rustfs_crypto::Error>),
#[error("user '{0}' does not exist")]
NoSuchUser(String),
@@ -58,15 +62,6 @@ pub enum Error {
#[error("not initialized")]
IamSysNotInitialized,
#[error("invalid service type: {0}")]
InvalidServiceType(String),
#[error("malformed credential")]
ErrCredMalformed,
#[error("CredNotInitialized")]
CredNotInitialized,
#[error("invalid access key length")]
InvalidAccessKeyLength,
@@ -79,27 +74,12 @@ pub enum Error {
#[error("group name contains reserved characters =,")]
GroupNameContainsReservedChars,
#[error("jwt err {0}")]
JWTError(jsonwebtoken::errors::Error),
#[error("no access key")]
NoAccessKey,
#[error("invalid token")]
InvalidToken,
#[error("invalid access_key")]
InvalidAccessKey,
#[error("access key is already in use")]
AccessKeyAlreadyExists,
#[error("action not allowed")]
IAMActionNotAllowed,
#[error("invalid expiration")]
InvalidExpiration,
#[error("no secret key with access key")]
NoSecretKeyWithAccessKey,
@@ -128,9 +108,8 @@ impl PartialEq for Error {
(Error::NoSuchServiceAccount(a), Error::NoSuchServiceAccount(b)) => a == b,
(Error::NoSuchTempAccount(a), Error::NoSuchTempAccount(b)) => a == b,
(Error::NoSuchGroup(a), Error::NoSuchGroup(b)) => a == b,
(Error::InvalidServiceType(a), Error::InvalidServiceType(b)) => a == b,
(Error::Io(a), Error::Io(b)) => a.kind() == b.kind() && a.to_string() == b.to_string(),
// For complex types like PolicyError, CryptoError, JWTError, compare string representations
// For complex types like PolicyError and CryptoError, compare string representations
(a, b) => std::mem::discriminant(a) == std::mem::discriminant(b) && a.to_string() == b.to_string(),
}
}
@@ -139,9 +118,9 @@ impl PartialEq for Error {
impl Clone for Error {
fn clone(&self) -> Self {
match self {
Error::PolicyError(e) => Error::StringError(e.to_string()), // Convert to string since PolicyError may not be cloneable
Error::PolicyError(e) => Error::PolicyError(Arc::clone(e)),
Error::StringError(s) => Error::StringError(s.clone()),
Error::CryptoError(e) => Error::StringError(format!("crypto: {e}")), // Convert to string
Error::CryptoError(e) => Error::CryptoError(Arc::clone(e)),
Error::NoSuchUser(s) => Error::NoSuchUser(s.clone()),
Error::NoSuchAccount(s) => Error::NoSuchAccount(s.clone()),
Error::NoSuchServiceAccount(s) => Error::NoSuchServiceAccount(s.clone()),
@@ -152,20 +131,12 @@ impl Clone for Error {
Error::GroupNotEmpty => Error::GroupNotEmpty,
Error::InvalidArgument => Error::InvalidArgument,
Error::IamSysNotInitialized => Error::IamSysNotInitialized,
Error::InvalidServiceType(s) => Error::InvalidServiceType(s.clone()),
Error::ErrCredMalformed => Error::ErrCredMalformed,
Error::CredNotInitialized => Error::CredNotInitialized,
Error::InvalidAccessKeyLength => Error::InvalidAccessKeyLength,
Error::InvalidSecretKeyLength => Error::InvalidSecretKeyLength,
Error::ContainsReservedChars => Error::ContainsReservedChars,
Error::GroupNameContainsReservedChars => Error::GroupNameContainsReservedChars,
Error::JWTError(e) => Error::StringError(format!("jwt err {e}")), // Convert to string
Error::NoAccessKey => Error::NoAccessKey,
Error::InvalidToken => Error::InvalidToken,
Error::InvalidAccessKey => Error::InvalidAccessKey,
Error::AccessKeyAlreadyExists => Error::AccessKeyAlreadyExists,
Error::IAMActionNotAllowed => Error::IAMActionNotAllowed,
Error::InvalidExpiration => Error::InvalidExpiration,
Error::NoSecretKeyWithAccessKey => Error::NoSecretKeyWithAccessKey,
Error::NoAccessKeyWithSecretKey => Error::NoAccessKeyWithSecretKey,
Error::PolicyTooLarge => Error::PolicyTooLarge,
@@ -176,6 +147,18 @@ impl Clone for Error {
}
}
impl From<PolicyError> for Error {
fn from(e: PolicyError) -> Self {
Error::PolicyError(Arc::new(e))
}
}
impl From<rustfs_crypto::Error> for Error {
fn from(e: rustfs_crypto::Error) -> Self {
Error::CryptoError(Arc::new(e))
}
}
impl Error {
pub fn other<E>(error: E) -> Self
where
@@ -208,16 +191,10 @@ impl From<rustfs_policy::error::Error> for Error {
match e {
rustfs_policy::error::Error::PolicyTooLarge => Error::PolicyTooLarge,
rustfs_policy::error::Error::InvalidArgument => Error::InvalidArgument,
rustfs_policy::error::Error::InvalidServiceType(s) => Error::InvalidServiceType(s),
rustfs_policy::error::Error::IAMActionNotAllowed => Error::IAMActionNotAllowed,
rustfs_policy::error::Error::InvalidExpiration => Error::InvalidExpiration,
rustfs_policy::error::Error::NoAccessKey => Error::NoAccessKey,
rustfs_policy::error::Error::InvalidToken => Error::InvalidToken,
rustfs_policy::error::Error::InvalidAccessKey => Error::InvalidAccessKey,
rustfs_policy::error::Error::NoSecretKeyWithAccessKey => Error::NoSecretKeyWithAccessKey,
rustfs_policy::error::Error::NoAccessKeyWithSecretKey => Error::NoAccessKeyWithSecretKey,
rustfs_policy::error::Error::Io(e) => Error::Io(e),
rustfs_policy::error::Error::JWTError(e) => Error::JWTError(e),
rustfs_policy::error::Error::NoSuchUser(s) => Error::NoSuchUser(s),
rustfs_policy::error::Error::NoSuchAccount(s) => Error::NoSuchAccount(s),
rustfs_policy::error::Error::NoSuchServiceAccount(s) => Error::NoSuchServiceAccount(s),
@@ -230,13 +207,22 @@ impl From<rustfs_policy::error::Error> for Error {
rustfs_policy::error::Error::InvalidSecretKeyLength => Error::InvalidSecretKeyLength,
rustfs_policy::error::Error::ContainsReservedChars => Error::ContainsReservedChars,
rustfs_policy::error::Error::GroupNameContainsReservedChars => Error::GroupNameContainsReservedChars,
rustfs_policy::error::Error::CredNotInitialized => Error::CredNotInitialized,
rustfs_policy::error::Error::IamSysNotInitialized => Error::IamSysNotInitialized,
rustfs_policy::error::Error::PolicyError(e) => Error::PolicyError(e),
rustfs_policy::error::Error::PolicyError(e) => Error::PolicyError(Arc::new(e)),
rustfs_policy::error::Error::StringError(s) => Error::StringError(s),
rustfs_policy::error::Error::CryptoError(e) => Error::CryptoError(e),
rustfs_policy::error::Error::ErrCredMalformed => Error::ErrCredMalformed,
rustfs_policy::error::Error::CryptoError(e) => Error::CryptoError(Arc::new(e)),
rustfs_policy::error::Error::IamSysAlreadyInitialized => Error::IamSysAlreadyInitialized,
// These policy variants had dead same-name twins on iam::Error (zero
// construction and zero match sites, removed in backlog#1831); the
// message is preserved through StringError instead.
err @ (rustfs_policy::error::Error::InvalidServiceType(_)
| rustfs_policy::error::Error::InvalidExpiration
| rustfs_policy::error::Error::NoAccessKey
| rustfs_policy::error::Error::InvalidToken
| rustfs_policy::error::Error::InvalidAccessKey
| rustfs_policy::error::Error::JWTError(_)
| rustfs_policy::error::Error::CredNotInitialized
| rustfs_policy::error::Error::ErrCredMalformed) => Error::StringError(err.to_string()),
}
}
}
@@ -415,6 +401,31 @@ mod tests {
assert!(converted_io.to_string().contains("access denied"));
}
#[test]
fn clone_preserves_variant_identity_and_message() {
// backlog#1831 PR2: cloning must never demote a variant to a different
// one (the old Clone stringified PolicyError/CryptoError into
// StringError). Pin discriminant and rendered message across clone.
let errors = vec![
Error::PolicyError(Arc::new(PolicyError::NonAction)),
Error::CryptoError(Arc::new(rustfs_crypto::Error::ErrInvalidKeyLength)),
Error::Io(std::io::Error::other("io payload")),
Error::StringError("plain".to_string()),
Error::NoSuchUser("u".to_string()),
Error::ConfigNotFound,
];
for error in errors {
let cloned = error.clone();
assert_eq!(
std::mem::discriminant(&error),
std::mem::discriminant(&cloned),
"clone must keep the variant of {error:?}"
);
assert_eq!(error.to_string(), cloned.to_string(), "clone must keep the rendered message");
}
}
#[test]
fn test_error_display_format() {
let test_cases = vec![
crates/io-core/CHANGELOG.md
+7
View File
@@ -5,6 +5,13 @@ All notable changes to the rustfs-io-core and rustfs-io-metrics crates will be d
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
### Removed
#### rustfs-io-metrics
- **Unified configuration** (added in 0.0.5): the zero-consumer `IoConfig`, `CacheSettings`, `IoSchedulerSettings`, `BackpressureSettings`, `TimeoutSettings`, `DeadlockDetectionSettings` types and their `DEFAULT_*` constants were removed (rustfs/rustfs#6008); rustfs-io-core's `IoSchedulerConfig`/`BackpressureConfig` remain the canonical configuration types.
## [0.0.5] - 2025-01-XX
### Added
crates/io-metrics/README.md
-27
View File
@@ -27,7 +27,6 @@
- **Metrics Collection**: Unified metrics recording and reporting
- **Bandwidth Monitoring**: Real-time bandwidth observation and analysis
- **Performance Metrics**: I/O performance metrics collection
- **Unified Configuration**: Centralized configuration management
- **Exporter Boundary**: Emit via `metrics`, export via `rustfs-obs`, no Prometheus HTTP endpoint
## Features
@@ -203,30 +202,6 @@ path and include:
deltas with `operation` and `backend` columns, so the TCP baseline can attribute
bytes and request/error counts to `tcp-http` transport operations.
### Unified Configuration
Centralized configuration management:
```rust
use rustfs_io_metrics::{
IoConfig, CacheSettings, IoSchedulerSettings,
BackpressureSettings, TimeoutSettings,
};
let config = IoConfig::new()
.with_cache(CacheSettings::new()
.with_max_capacity(10_000)
.with_ttl(std::time::Duration::from_secs(300)))
.with_scheduler(IoSchedulerSettings::new()
.with_max_concurrent_reads(64))
.with_backpressure(BackpressureSettings::new())
.with_timeout(TimeoutSettings::new());
// Access configuration
println!("Cache capacity: {}", config.cache.max_capacity);
println!("Max concurrent reads: {}", config.scheduler.max_concurrent_reads);
```
## Module Structure
```
@@ -235,7 +210,6 @@ rustfs-io-metrics/
│ ├── lib.rs # Module entry
│ ├── cache_config.rs # Cache configuration
│ ├── adaptive_ttl.rs # Adaptive TTL
│ ├── config.rs # Unified configuration
│ ├── io_metrics.rs # I/O metrics
│ ├── backpressure_metrics.rs # Backpressure metrics
│ ├── deadlock_metrics.rs # Deadlock metrics
@@ -278,7 +252,6 @@ Useful source references:
- [Crate API overview](./src/lib.rs)
- [Metrics example](./examples/metrics_example.rs)
- [Configuration module](./src/config.rs)
- [Adaptive TTL module](./src/adaptive_ttl.rs)
## Related Modules
crates/io-metrics/README_zh.md
-42
View File
@@ -27,7 +27,6 @@
- **指标收集**：统一的指标记录和上报
- **带宽监控**：实时带宽观测和分析
- **性能指标**：I/O 性能指标收集
- **统一配置**：集中式配置管理
- **导出边界**：通过 `metrics` 主动上报，由 `rustfs-obs` 负责 OTEL 导出，不提供 Prometheus HTTP 端点
## ✨ 核心功能
@@ -172,30 +171,6 @@ println!("读取速率: {} bytes/s", snapshot.read_bytes_per_sec);
println!("写入速率: {} bytes/s", snapshot.write_bytes_per_sec);
```
### 统一配置 (IoConfig)
集中式配置管理：
```rust
use rustfs_io_metrics::{
IoConfig, CacheSettings, IoSchedulerSettings,
BackpressureSettings, TimeoutSettings,
};
let config = IoConfig::new()
.with_cache(CacheSettings::new()
.with_max_capacity(10_000)
.with_ttl(std::time::Duration::from_secs(300)))
.with_scheduler(IoSchedulerSettings::new()
.with_max_concurrent_reads(64))
.with_backpressure(BackpressureSettings::new())
.with_timeout(TimeoutSettings::new());
// 访问配置
println!("缓存容量: {}", config.cache.max_capacity);
println!("最大并发读: {}", config.scheduler.max_concurrent_reads);
```
## 📊 指标类型
### I/O 调度指标
@@ -233,21 +208,6 @@ println!("最大并发读: {}", config.scheduler.max_concurrent_reads);
| `operation_duration_secs` | 操作时长 | Histogram |
| `operation_progress` | 操作进度 | Gauge |
## 🔧 配置
### 代码配置
```rust
use rustfs_io_metrics::{CacheSettings, IoConfig};
let settings = CacheSettings::new()
.with_max_capacity(5000)
.with_ttl(std::time::Duration::from_secs(600))
.with_max_memory(200 * 1024 * 1024);
let config = IoConfig::new().with_cache(settings);
```
## 📁 模块结构
```
@@ -256,7 +216,6 @@ rustfs-io-metrics/
│ ├── lib.rs # 模块入口
│ ├── cache_config.rs # 缓存配置
│ ├── adaptive_ttl.rs # 自适应 TTL
│ ├── config.rs # 统一配置
│ ├── io_metrics.rs # I/O 指标
│ ├── backpressure_metrics.rs # 背压指标
│ ├── deadlock_metrics.rs # 死锁指标
@@ -297,7 +256,6 @@ cargo doc --package rustfs-io-metrics --no-deps --open
- [Crate API 概览](./src/lib.rs)
- [指标示例](./examples/metrics_example.rs)
- [配置模块](./src/config.rs)
- [自适应 TTL 模块](./src/adaptive_ttl.rs)
## 🔗 相关模块
crates/io-metrics/examples/metrics_example.rs
+2 -27
View File
@@ -14,9 +14,7 @@
//! Example demonstrating metrics and configuration usage.
use rustfs_io_metrics::{
AccessTracker, AdaptiveTTL, CacheConfig, CacheSettings, IoConfig, IoSchedulerSettings, record_cache_size,
};
use rustfs_io_metrics::{AccessTracker, AdaptiveTTL, CacheConfig, record_cache_size};
use std::time::Duration;
fn main() {
@@ -31,10 +29,7 @@ fn main() {
// 3. Access tracking example
access_tracker_example();
// 4. Unified configuration example
unified_config_example();
// 5. Metrics recording example
// 4. Metrics recording example
metrics_recording_example();
}
@@ -109,26 +104,6 @@ fn access_tracker_example() {
println!();
}
fn unified_config_example() {
println!("--- Unified Configuration ---");
let config = IoConfig::new()
.with_cache(
CacheSettings::new()
.with_max_capacity(5000)
.with_ttl(Duration::from_secs(600)),
)
.with_scheduler(IoSchedulerSettings::new().with_max_concurrent_reads(64));
println!(" Cache capacity: {}", config.cache.max_capacity);
println!(" Cache TTL: {:?}", config.cache.default_ttl);
println!(" Max concurrent reads: {}", config.scheduler.max_concurrent_reads);
println!(" Backpressure high watermark: {}", config.backpressure.high_watermark);
println!(" Default timeout: {:?}", config.timeout.default_timeout);
println!();
}
fn metrics_recording_example() {
println!("--- Metrics Recording ---");
crates/io-metrics/src/adaptive_ttl.rs
+38 -24
View File
@@ -315,6 +315,44 @@ impl Default for AccessTracker {
mod tests {
use super::*;
/// Replaces the per-helper smoke tests that called the record_* helpers
/// and asserted nothing: the calls (same literals) now run against a local
/// DebuggingRecorder and every metric name the helpers own must actually
/// be emitted (rustfs/backlog#1836 PR3).
#[test]
fn record_helpers_emit_their_metrics() {
let recorder = metrics_util::debugging::DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
metrics::with_local_recorder(&recorder, || {
record_ttl_adjustment("test-key", 100, 150);
record_ttl_adjustment("test-key", 100, 50);
record_ttl_expiration();
record_early_eviction("cold");
record_early_eviction("low_priority");
record_access_pattern_change("sequential", "random");
record_access_pattern_change("random", "sequential");
});
let emitted: std::collections::HashSet<String> = snapshotter
.snapshot()
.into_vec()
.into_iter()
.map(|(composite, _, _, _)| composite.key().name().to_string())
.collect();
for expected in [
"rustfs_cache_ttl_adjustments",
"rustfs_cache_ttl_base",
"rustfs_cache_ttl_adjusted",
"rustfs_cache_ttl_extensions",
"rustfs_cache_ttl_reductions",
"rustfs_cache_ttl_expirations",
"rustfs_cache_evictions_early",
"rustfs_cache_access_pattern_changes",
] {
assert!(emitted.contains(expected), "{expected} must be emitted by its record helper");
}
}
#[test]
fn test_adaptive_ttl_stats() {
let mut stats = AdaptiveTTLStats::new();
@@ -335,30 +373,6 @@ mod tests {
assert!((stats.reduction_rate() - 0.3333333333333333).abs() < 0.01);
}
#[test]
fn test_record_ttl_adjustment() {
// This test verifies the function compiles and runs
record_ttl_adjustment("test-key", 100, 150);
record_ttl_adjustment("test-key", 100, 50);
}
#[test]
fn test_record_ttl_expiration() {
record_ttl_expiration();
}
#[test]
fn test_record_early_eviction() {
record_early_eviction("cold");
record_early_eviction("low_priority");
}
#[test]
fn test_record_access_pattern_change() {
record_access_pattern_change("sequential", "random");
record_access_pattern_change("random", "sequential");
}
#[test]
fn test_access_record() {
let mut record = AccessRecord::new();
crates/io-metrics/src/backpressure_metrics.rs
+31 -23
View File
@@ -53,30 +53,38 @@ pub fn record_backpressure_deactivation() {
mod tests {
use super::*;
/// Replaces the per-helper smoke tests that called the record_* helpers
/// and asserted nothing: the calls (same literals) now run against a local
/// DebuggingRecorder and every metric name the helpers own must actually
/// be emitted (rustfs/backlog#1836 PR3).
#[test]
fn test_record_backpressure_state_change() {
record_backpressure_state_change("normal", "warning");
record_backpressure_state_change("warning", "critical");
}
fn record_helpers_emit_their_metrics() {
let recorder = metrics_util::debugging::DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
metrics::with_local_recorder(&recorder, || {
record_backpressure_state_change("normal", "warning");
record_backpressure_state_change("warning", "critical");
record_backpressure_rejection();
record_concurrent_operations(10);
record_concurrent_operations(32);
record_backpressure_activation();
record_backpressure_deactivation();
});
#[test]
fn test_record_backpressure_rejection() {
record_backpressure_rejection();
}
#[test]
fn test_record_concurrent_operations() {
record_concurrent_operations(10);
record_concurrent_operations(32);
}
#[test]
fn test_record_backpressure_activation() {
record_backpressure_activation();
}
#[test]
fn test_record_backpressure_deactivation() {
record_backpressure_deactivation();
let emitted: std::collections::HashSet<String> = snapshotter
.snapshot()
.into_vec()
.into_iter()
.map(|(composite, _, _, _)| composite.key().name().to_string())
.collect();
for expected in [
"rustfs_backpressure_state_changes",
"rustfs_backpressure_rejections",
"rustfs_backpressure_concurrent",
"rustfs_backpressure_activations",
"rustfs_backpressure_deactivations",
] {
assert!(emitted.contains(expected), "{expected} must be emitted by its record helper");
}
}
}
crates/io-metrics/src/config.rs
-391
View File
@@ -1,391 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Unified configuration interface for I/O operations.
//!
//! This module provides a centralized configuration interface
//! for all I/O-related settings.
use std::time::Duration;
// ============================================================================
// Configuration Constants
// ============================================================================
/// Default cache max capacity.
pub const DEFAULT_CACHE_MAX_CAPACITY: u64 = 10_000;
/// Default cache TTL in seconds.
pub const DEFAULT_CACHE_TTL_SECS: u64 = 300;
/// Default cache max memory in bytes (100 MB).
pub const DEFAULT_CACHE_MAX_MEMORY: u64 = 100 * 1024 * 1024;
/// Default I/O scheduler max concurrent reads.
pub const DEFAULT_MAX_CONCURRENT_READS: usize = 32;
/// Default high priority size threshold (64 KB).
pub const DEFAULT_HIGH_PRIORITY_SIZE_THRESHOLD: usize = 64 * 1024;
/// Default low priority size threshold (4 MB).
pub const DEFAULT_LOW_PRIORITY_SIZE_THRESHOLD: usize = 4 * 1024 * 1024;
/// Default backpressure high watermark.
pub const DEFAULT_BACKPRESSURE_HIGH_WATERMARK: f64 = 0.8;
/// Default backpressure low watermark.
pub const DEFAULT_BACKPRESSURE_LOW_WATERMARK: f64 = 0.5;
/// Default lock acquire timeout in seconds.
pub const DEFAULT_LOCK_ACQUIRE_TIMEOUT_SECS: u64 = 5;
/// Default deadlock detection interval in seconds.
pub const DEFAULT_DEADLOCK_DETECTION_INTERVAL_SECS: u64 = 1;
/// Default base buffer size (128 KB).
pub const DEFAULT_BASE_BUFFER_SIZE: usize = 128 * 1024;
/// Default max buffer size (1 MB).
pub const DEFAULT_MAX_BUFFER_SIZE: usize = 1024 * 1024;
/// Default min buffer size (4 KB).
pub const DEFAULT_MIN_BUFFER_SIZE: usize = 4 * 1024;
// ============================================================================
// Cache Configuration
// ============================================================================
/// Cache configuration settings.
#[derive(Debug, Clone)]
pub struct CacheSettings {
/// Maximum cache capacity.
pub max_capacity: u64,
/// Default TTL.
pub default_ttl: Duration,
/// Maximum memory usage.
pub max_memory: u64,
/// Whether adaptive TTL is enabled.
pub adaptive_ttl_enabled: bool,
}
impl Default for CacheSettings {
fn default() -> Self {
Self {
max_capacity: DEFAULT_CACHE_MAX_CAPACITY,
default_ttl: Duration::from_secs(DEFAULT_CACHE_TTL_SECS),
max_memory: DEFAULT_CACHE_MAX_MEMORY,
adaptive_ttl_enabled: true,
}
}
}
impl CacheSettings {
/// Create new cache settings.
pub fn new() -> Self {
Self::default()
}
/// Builder: set max capacity.
pub fn with_max_capacity(mut self, capacity: u64) -> Self {
self.max_capacity = capacity;
self
}
/// Builder: set TTL.
pub fn with_ttl(mut self, ttl: Duration) -> Self {
self.default_ttl = ttl;
self
}
/// Builder: set max memory.
pub fn with_max_memory(mut self, memory: u64) -> Self {
self.max_memory = memory;
self
}
}
// ============================================================================
// I/O Scheduler Configuration
// ============================================================================
/// I/O scheduler configuration settings.
#[derive(Debug, Clone)]
pub struct IoSchedulerSettings {
/// Maximum concurrent reads.
pub max_concurrent_reads: usize,
/// High priority size threshold.
pub high_priority_threshold: usize,
/// Low priority size threshold.
pub low_priority_threshold: usize,
/// Base buffer size.
pub base_buffer_size: usize,
/// Max buffer size.
pub max_buffer_size: usize,
/// Min buffer size.
pub min_buffer_size: usize,
/// Whether priority scheduling is enabled.
pub priority_enabled: bool,
}
impl Default for IoSchedulerSettings {
fn default() -> Self {
Self {
max_concurrent_reads: DEFAULT_MAX_CONCURRENT_READS,
high_priority_threshold: DEFAULT_HIGH_PRIORITY_SIZE_THRESHOLD,
low_priority_threshold: DEFAULT_LOW_PRIORITY_SIZE_THRESHOLD,
base_buffer_size: DEFAULT_BASE_BUFFER_SIZE,
max_buffer_size: DEFAULT_MAX_BUFFER_SIZE,
min_buffer_size: DEFAULT_MIN_BUFFER_SIZE,
priority_enabled: true,
}
}
}
impl IoSchedulerSettings {
/// Create new settings.
pub fn new() -> Self {
Self::default()
}
/// Builder: set max concurrent reads.
pub fn with_max_concurrent_reads(mut self, max: usize) -> Self {
self.max_concurrent_reads = max;
self
}
/// Builder: set buffer sizes.
pub fn with_buffer_sizes(mut self, base: usize, min: usize, max: usize) -> Self {
self.base_buffer_size = base;
self.min_buffer_size = min;
self.max_buffer_size = max;
self
}
}
// ============================================================================
// Backpressure Configuration
// ============================================================================
/// Backpressure configuration settings.
#[derive(Debug, Clone)]
pub struct BackpressureSettings {
/// Whether backpressure is enabled.
pub enabled: bool,
/// High watermark (percentage).
pub high_watermark: f64,
/// Low watermark (percentage).
pub low_watermark: f64,
/// Cooldown duration.
pub cooldown: Duration,
}
impl Default for BackpressureSettings {
fn default() -> Self {
Self {
enabled: true,
high_watermark: DEFAULT_BACKPRESSURE_HIGH_WATERMARK,
low_watermark: DEFAULT_BACKPRESSURE_LOW_WATERMARK,
cooldown: Duration::from_millis(100),
}
}
}
impl BackpressureSettings {
/// Create new settings.
pub fn new() -> Self {
Self::default()
}
/// Get high watermark threshold for a given max value.
pub fn high_threshold(&self, max: usize) -> usize {
(max as f64 * self.high_watermark) as usize
}
/// Get low watermark threshold for a given max value.
pub fn low_threshold(&self, max: usize) -> usize {
(max as f64 * self.low_watermark) as usize
}
}
// ============================================================================
// Timeout Configuration
// ============================================================================
/// Timeout configuration settings.
#[derive(Debug, Clone)]
pub struct TimeoutSettings {
/// Default operation timeout.
pub default_timeout: Duration,
/// Maximum retries.
pub max_retries: usize,
/// Retry backoff factor.
pub retry_backoff_factor: f64,
/// Lock acquire timeout.
pub lock_acquire_timeout: Duration,
}
impl Default for TimeoutSettings {
fn default() -> Self {
Self {
default_timeout: Duration::from_secs(30),
max_retries: 3,
retry_backoff_factor: 2.0,
lock_acquire_timeout: Duration::from_secs(DEFAULT_LOCK_ACQUIRE_TIMEOUT_SECS),
}
}
}
impl TimeoutSettings {
/// Create new settings.
pub fn new() -> Self {
Self::default()
}
/// Calculate timeout with backoff for a given retry count.
pub fn timeout_with_backoff(&self, retry_count: usize) -> Duration {
let multiplier = self.retry_backoff_factor.powi(retry_count as i32);
Duration::from_secs_f64(self.default_timeout.as_secs_f64() * multiplier)
}
}
// ============================================================================
// Deadlock Detection Configuration
// ============================================================================
/// Deadlock detection configuration settings.
#[derive(Debug, Clone)]
pub struct DeadlockDetectionSettings {
/// Whether detection is enabled.
pub enabled: bool,
/// Detection interval.
pub detection_interval: Duration,
/// Maximum lock hold time before warning.
pub max_hold_time: Duration,
}
impl Default for DeadlockDetectionSettings {
fn default() -> Self {
Self {
enabled: true,
detection_interval: Duration::from_secs(DEFAULT_DEADLOCK_DETECTION_INTERVAL_SECS),
max_hold_time: Duration::from_secs(30),
}
}
}
impl DeadlockDetectionSettings {
/// Create new settings.
pub fn new() -> Self {
Self::default()
}
}
// ============================================================================
// Unified Configuration
// ============================================================================
/// Unified configuration for all I/O operations.
#[derive(Debug, Clone, Default)]
pub struct IoConfig {
/// Cache settings.
pub cache: CacheSettings,
/// I/O scheduler settings.
pub scheduler: IoSchedulerSettings,
/// Backpressure settings.
pub backpressure: BackpressureSettings,
/// Timeout settings.
pub timeout: TimeoutSettings,
/// Deadlock detection settings.
pub deadlock_detection: DeadlockDetectionSettings,
}
impl IoConfig {
/// Create new unified configuration.
pub fn new() -> Self {
Self::default()
}
/// Builder: set cache settings.
pub fn with_cache(mut self, cache: CacheSettings) -> Self {
self.cache = cache;
self
}
/// Builder: set scheduler settings.
pub fn with_scheduler(mut self, scheduler: IoSchedulerSettings) -> Self {
self.scheduler = scheduler;
self
}
/// Builder: set backpressure settings.
pub fn with_backpressure(mut self, backpressure: BackpressureSettings) -> Self {
self.backpressure = backpressure;
self
}
/// Builder: set timeout settings.
pub fn with_timeout(mut self, timeout: TimeoutSettings) -> Self {
self.timeout = timeout;
self
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_cache_settings() {
let settings = CacheSettings::new()
.with_max_capacity(5000)
.with_ttl(Duration::from_secs(600));
assert_eq!(settings.max_capacity, 5000);
assert_eq!(settings.default_ttl, Duration::from_secs(600));
}
#[test]
fn test_io_scheduler_settings() {
let settings =
IoSchedulerSettings::new()
.with_max_concurrent_reads(64)
.with_buffer_sizes(256 * 1024, 8 * 1024, 2 * 1024 * 1024);
assert_eq!(settings.max_concurrent_reads, 64);
assert_eq!(settings.base_buffer_size, 256 * 1024);
}
#[test]
fn test_backpressure_settings() {
let settings = BackpressureSettings::new();
assert_eq!(settings.high_threshold(100), 80);
assert_eq!(settings.low_threshold(100), 50);
}
#[test]
fn test_timeout_settings() {
let settings = TimeoutSettings::new();
// First retry: 30s * 2 = 60s
let timeout1 = settings.timeout_with_backoff(1);
assert!(timeout1.as_secs() >= 60);
// Second retry: 30s * 4 = 120s
let timeout2 = settings.timeout_with_backoff(2);
assert!(timeout2.as_secs() >= 120);
}
#[test]
fn test_unified_config() {
let config = IoConfig::new()
.with_cache(CacheSettings::new().with_max_capacity(5000))
.with_scheduler(IoSchedulerSettings::new().with_max_concurrent_reads(64));
assert_eq!(config.cache.max_capacity, 5000);
assert_eq!(config.scheduler.max_concurrent_reads, 64);
}
}
crates/io-metrics/src/deadlock_metrics.rs
+41 -32
View File
@@ -72,39 +72,48 @@ pub fn record_wait_edge_removed() {
mod tests {
use super::*;
/// Replaces the per-helper smoke tests that called the record_* helpers
/// and asserted nothing: the calls (same literals) now run against a local
/// DebuggingRecorder and every metric name the helpers own must actually
/// be emitted (rustfs/backlog#1836 PR3).
#[test]
fn test_record_deadlock_detected() {
record_deadlock_detected(3);
record_deadlock_detected(5);
}
fn record_helpers_emit_their_metrics() {
let recorder = metrics_util::debugging::DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
metrics::with_local_recorder(&recorder, || {
record_deadlock_detected(3);
record_deadlock_detected(5);
record_long_held_lock(1, Duration::from_secs(30));
record_long_held_lock(2, Duration::from_secs(60));
record_lock_acquisition("mutex");
record_lock_acquisition("rwlock");
record_lock_release("mutex", Duration::from_millis(10));
record_lock_release("rwlock", Duration::from_millis(5));
record_lock_contention("mutex");
record_lock_contention("rwlock");
record_wait_edge_added();
record_wait_edge_removed();
});
#[test]
fn test_record_long_held_lock() {
record_long_held_lock(1, Duration::from_secs(30));
record_long_held_lock(2, Duration::from_secs(60));
}
#[test]
fn test_record_lock_acquisition() {
record_lock_acquisition("mutex");
record_lock_acquisition("rwlock");
}
#[test]
fn test_record_lock_release() {
record_lock_release("mutex", Duration::from_millis(10));
record_lock_release("rwlock", Duration::from_millis(5));
}
#[test]
fn test_record_lock_contention() {
record_lock_contention("mutex");
record_lock_contention("rwlock");
}
#[test]
fn test_record_wait_edge() {
record_wait_edge_added();
record_wait_edge_removed();
let emitted: std::collections::HashSet<String> = snapshotter
.snapshot()
.into_vec()
.into_iter()
.map(|(composite, _, _, _)| composite.key().name().to_string())
.collect();
for expected in [
"rustfs_deadlock_detected_total",
"rustfs_deadlock_cycle_length",
"rustfs_deadlock_long_held",
"rustfs_deadlock_hold_time_secs",
"rustfs_lock_acquisitions",
"rustfs_lock_releases",
"rustfs_lock_hold_time_secs",
"rustfs_lock_contentions",
"rustfs_deadlock_wait_edges_added",
"rustfs_deadlock_wait_edges_removed",
] {
assert!(emitted.contains(expected), "{expected} must be emitted by its record helper");
}
}
}
crates/io-metrics/src/io_metrics.rs
+50 -38
View File
@@ -169,46 +169,58 @@ impl IoSchedulerStats {
mod tests {
use super::*;
/// Replaces the per-helper smoke tests that called the record_* helpers
/// and asserted nothing: the calls (same literals) now run against a local
/// DebuggingRecorder and every metric name the helpers own must actually
/// be emitted (rustfs/backlog#1836 PR3).
#[test]
fn test_record_io_scheduler_decision() {
record_io_scheduler_decision(128 * 1024, "low", "sequential");
record_io_scheduler_decision(64 * 1024, "high", "random");
}
fn record_helpers_emit_their_metrics() {
let recorder = metrics_util::debugging::DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
metrics::with_local_recorder(&recorder, || {
record_io_scheduler_decision(128 * 1024, "low", "sequential");
record_io_scheduler_decision(64 * 1024, "high", "random");
record_io_priority_decision("high", 1024);
record_io_priority_decision("normal", 1024 * 1024);
record_io_priority_decision("low", 10 * 1024 * 1024);
record_load_level_change("low", "medium");
record_load_level_change("medium", "high");
record_bandwidth_observation(100 * 1024 * 1024);
record_bandwidth_observation(500 * 1024 * 1024);
record_buffer_size_adjustment(128 * 1024, 64 * 1024, "concurrency");
record_buffer_size_adjustment(128 * 1024, 256 * 1024, "sequential");
record_queue_operation("enqueue", "high", 10);
record_queue_operation("dequeue", "high", 9);
record_starvation_event("low");
});
#[test]
fn test_record_io_priority_decision() {
record_io_priority_decision("high", 1024);
record_io_priority_decision("normal", 1024 * 1024);
record_io_priority_decision("low", 10 * 1024 * 1024);
}
#[test]
fn test_record_load_level_change() {
record_load_level_change("low", "medium");
record_load_level_change("medium", "high");
}
#[test]
fn test_record_bandwidth_observation() {
record_bandwidth_observation(100 * 1024 * 1024);
record_bandwidth_observation(500 * 1024 * 1024);
}
#[test]
fn test_record_buffer_size_adjustment() {
record_buffer_size_adjustment(128 * 1024, 64 * 1024, "concurrency");
record_buffer_size_adjustment(128 * 1024, 256 * 1024, "sequential");
}
#[test]
fn test_record_queue_operation() {
record_queue_operation("enqueue", "high", 10);
record_queue_operation("dequeue", "high", 9);
}
#[test]
fn test_record_starvation_event() {
record_starvation_event("low");
let emitted: std::collections::HashSet<String> = snapshotter
.snapshot()
.into_vec()
.into_iter()
.map(|(composite, _, _, _)| composite.key().name().to_string())
.collect();
for expected in [
"rustfs_io_scheduler_decisions",
"rustfs_io_scheduler_buffer_size",
"rustfs_io_scheduler_load",
"rustfs_io_scheduler_strategy",
"rustfs_io_scheduler_buffer_size_histogram",
"rustfs_io_priority_decisions",
"rustfs_io_priority_by_level",
"rustfs_io_priority_request_size",
"rustfs_io_load_changes",
"rustfs_io_bandwidth_bps",
"rustfs_io_bandwidth_histogram",
"rustfs_io_buffer_adjustments",
"rustfs_io_buffer_original",
"rustfs_io_buffer_adjusted",
"rustfs_io_queue_operations",
"rustfs_io_queue_size",
"rustfs_io_starvation_events",
] {
assert!(emitted.contains(expected), "{expected} must be emitted by its record helper");
}
}
#[test]
crates/io-metrics/src/lib.rs
+6 -10
View File
@@ -173,7 +173,6 @@ pub mod backpressure_metrics;
pub mod cache_config;
pub mod capacity_metrics;
pub mod collector;
pub mod config;
pub mod deadlock_metrics;
pub mod internode_metrics;
pub mod io_metrics;
@@ -260,13 +259,6 @@ pub use timeout_metrics::{
record_operation_progress, record_stalled_operation, record_timeout_event,
};
// Config exports
pub use config::{
BackpressureSettings, CacheSettings, DEFAULT_BASE_BUFFER_SIZE, DEFAULT_CACHE_MAX_CAPACITY, DEFAULT_CACHE_MAX_MEMORY,
DEFAULT_CACHE_TTL_SECS, DEFAULT_MAX_BUFFER_SIZE, DEFAULT_MAX_CONCURRENT_READS, DEFAULT_MIN_BUFFER_SIZE,
DeadlockDetectionSettings, IoConfig, IoSchedulerSettings, TimeoutSettings,
};
// Re-exports for convenience
pub use collector::MetricsCollector;
pub use performance::PerformanceMetrics;
@@ -795,8 +787,12 @@ pub fn record_get_object_metadata_cache_decision(path: &'static str, decision: &
}
/// Record aggregate metadata fanout shape for one GetObject metadata read.
///
/// The legacy `metadata_fanout_error_responses` series records every non-valid
/// response, including not-found and ignored outcomes. Use
/// `metadata_response_total` outcome labels for failure attribution.
#[inline(always)]
pub fn record_get_object_metadata_fanout_shape(path: &'static str, total: usize, valid: usize, ignored: usize, errors: usize) {
pub fn record_get_object_metadata_fanout_shape(path: &'static str, total: usize, valid: usize, ignored: usize, non_valid: usize) {
if !get_stage_metrics_enabled() {
return;
}
@@ -807,7 +803,7 @@ pub fn record_get_object_metadata_fanout_shape(path: &'static str, total: usize,
histogram!("rustfs_io_get_object_metadata_fanout_ignored_responses", "path" => path)
.record(metadata_fanout_count_to_f64(ignored));
histogram!("rustfs_io_get_object_metadata_fanout_error_responses", "path" => path)
.record(metadata_fanout_count_to_f64(errors));
.record(metadata_fanout_count_to_f64(non_valid));
}
/// Record a guarded metadata early-stop hit for GetObject.
crates/io-metrics/src/lock_metrics.rs
+40 -34
View File
@@ -163,6 +163,46 @@ impl LockMetricsSummary {
#[cfg(test)]
mod tests {
use super::*;
/// Replaces the per-helper smoke tests that called the record_* helpers
/// and asserted nothing: the calls (same literals) now run against a local
/// DebuggingRecorder and every metric name the helpers own must actually
/// be emitted (rustfs/backlog#1836 PR3).
#[test]
fn record_helpers_emit_their_metrics() {
let recorder = metrics_util::debugging::DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
metrics::with_local_recorder(&recorder, || {
record_lock_optimization_enabled(true);
record_lock_optimization_enabled(false);
record_spin_attempt(true);
record_spin_attempt(false);
record_spin_count_change(100);
record_spin_count_change(200);
record_lock_hold_time(Duration::from_millis(10));
record_lock_hold_time(Duration::from_millis(100));
record_early_release();
record_contention_event();
});
let emitted: std::collections::HashSet<String> = snapshotter
.snapshot()
.into_vec()
.into_iter()
.map(|(composite, _, _, _)| composite.key().name().to_string())
.collect();
for expected in [
"rustfs_lock_optimization_enabled",
"rustfs_lock_spin_successes",
"rustfs_lock_spin_failures",
"rustfs_lock_spin_count",
"rustfs_lock_hold_time_secs",
"rustfs_lock_early_releases",
"rustfs_lock_contentions",
] {
assert!(emitted.contains(expected), "{expected} must be emitted by its record helper");
}
}
use metrics::{Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Metadata, SharedString, Unit};
use std::sync::{Arc, Mutex};
@@ -255,40 +295,6 @@ mod tests {
fn record(&self, _value: f64) {}
}
#[test]
fn test_record_lock_optimization_enabled() {
record_lock_optimization_enabled(true);
record_lock_optimization_enabled(false);
}
#[test]
fn test_record_spin_attempt() {
record_spin_attempt(true);
record_spin_attempt(false);
}
#[test]
fn test_record_spin_count_change() {
record_spin_count_change(100);
record_spin_count_change(200);
}
#[test]
fn test_record_lock_hold_time() {
record_lock_hold_time(Duration::from_millis(10));
record_lock_hold_time(Duration::from_millis(100));
}
#[test]
fn test_record_early_release() {
record_early_release();
}
#[test]
fn test_record_contention_event() {
record_contention_event();
}
#[test]
fn test_record_object_lock_diag_enabled() {
let recorder = SeenMetricsRecorder::default();
crates/io-metrics/src/timeout_metrics.rs
+38 -31
View File
@@ -114,39 +114,46 @@ impl TimeoutMetricsSummary {
mod tests {
use super::*;
/// Replaces the per-helper smoke tests that called the record_* helpers
/// and asserted nothing: the calls (same literals) now run against a local
/// DebuggingRecorder and every metric name the helpers own must actually
/// be emitted (rustfs/backlog#1836 PR3).
#[test]
fn test_record_timeout_event() {
record_timeout_event("get_object");
record_timeout_event("put_object");
}
fn record_helpers_emit_their_metrics() {
let recorder = metrics_util::debugging::DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
metrics::with_local_recorder(&recorder, || {
record_timeout_event("get_object");
record_timeout_event("put_object");
record_operation_duration("get_object", Duration::from_millis(100));
record_operation_duration("put_object", Duration::from_millis(500));
record_dynamic_timeout(1024 * 1024, Duration::from_secs(10));
record_dynamic_timeout(100 * 1024 * 1024, Duration::from_secs(30));
record_operation_progress("get_object", 50.0);
record_operation_progress("get_object", 100.0);
record_stalled_operation("get_object");
record_operation_completion("get_object", true);
record_operation_completion("get_object", false);
});
#[test]
fn test_record_operation_duration() {
record_operation_duration("get_object", Duration::from_millis(100));
record_operation_duration("put_object", Duration::from_millis(500));
}
#[test]
fn test_record_dynamic_timeout() {
record_dynamic_timeout(1024 * 1024, Duration::from_secs(10));
record_dynamic_timeout(100 * 1024 * 1024, Duration::from_secs(30));
}
#[test]
fn test_record_operation_progress() {
record_operation_progress("get_object", 50.0);
record_operation_progress("get_object", 100.0);
}
#[test]
fn test_record_stalled_operation() {
record_stalled_operation("get_object");
}
#[test]
fn test_record_operation_completion() {
record_operation_completion("get_object", true);
record_operation_completion("get_object", false);
let emitted: std::collections::HashSet<String> = snapshotter
.snapshot()
.into_vec()
.into_iter()
.map(|(composite, _, _, _)| composite.key().name().to_string())
.collect();
for expected in [
"rustfs_io_timeout_events_total",
"rustfs_io_operation_duration_seconds",
"rustfs_timeout_dynamic_size",
"rustfs_timeout_dynamic_secs",
"rustfs_timeout_dynamic_size_histogram",
"rustfs_operation_progress",
"rustfs_operation_stalled",
"rustfs_operation_completions",
] {
assert!(emitted.contains(expected), "{expected} must be emitted by its record helper");
}
}
#[test]
crates/kms/Cargo.toml
+5 -1
View File
@@ -35,6 +35,10 @@ tokio = { workspace = true, features = ["fs", "io-util", "macros", "rt-multi-thr
uuid = { workspace = true, features = ["serde", "v4", "fast-rng", "macro-diagnostics"] }
jiff = { workspace = true, features = ["serde"] }
serde = { workspace = true, features = ["derive"] }
# Observes fields a persisted-format deserialization ignored, per the
# repository rule that formats too compatibility-bound for
# deny_unknown_fields must at least warn (AGENTS.md).
serde_ignored = { workspace = true }
serde_json = { workspace = true, features = ["raw_value"] }
tracing = { workspace = true }
thiserror = { workspace = true }
@@ -62,7 +66,7 @@ moka = { workspace = true, features = ["future"] }
# Additional dependencies
md-5 = { workspace = true }
arc-swap = { workspace = true }
rustfs-utils = { workspace = true }
rustfs-utils = { workspace = true, features = ["http"] }
rustfs-security-governance = { workspace = true }
# `EventName` for KMS audit records. A leaf crate with no rustfs dependencies,
# so the audit sink can live outside this crate without a second, drifting
crates/kms/src/backends/aws.rs
+1
View File
@@ -657,6 +657,7 @@ impl KmsBackend for AwsKmsBackend {
created_by: None,
rotation_due: false,
rotation_due_reason: None,
wrap_budget_reserved: None,
});
}
crates/kms/src/backends/scripted_vault.rs
+11
View File
@@ -57,6 +57,17 @@ impl ScriptedResponse {
}
}
/// The 404 Vault answers a LIST of an empty path with: something routed the
/// request and found nothing under it, so the `errors` array comes back
/// empty. [`ScriptedResponse::error`] cannot stand in — it always fills
/// `errors`, which is what marks a 404 as an unrouted path instead.
pub(crate) fn empty_list_404() -> Self {
Self::Http {
status: 404,
body: serde_json::json!({ "errors": [] }).to_string(),
}
}
/// Close the connection after consuming a request without sending an HTTP response.
pub(crate) fn close() -> Self {
Self::Close
crates/kms/src/backends/static_kms.rs
+1
View File
@@ -271,6 +271,7 @@ impl StaticKmsBackend {
created_by: None,
rotation_due: false,
rotation_due_reason: None,
wrap_budget_reserved: None,
})
}
crates/kms/src/backends/vault.rs
+943 -33
View File
File diff suppressed because it is too large Load Diff
crates/kms/src/backends/vault_transit.rs
+344 -12
View File
@@ -27,6 +27,7 @@ use crate::backends::{
use crate::config::{KmsConfig, VaultTransitConfig};
use crate::encryption::{DataKeyEnvelope, generate_key_material};
use crate::error::{KmsError, Result};
use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummary};
use crate::policy::{self, AttemptError, OpClass, RetryPolicy};
use crate::types::*;
use async_trait::async_trait;
@@ -100,6 +101,23 @@ fn is_cas_conflict(error: &ClientError) -> bool {
)
}
/// Whether a transit LIST failed with the 404 Vault uses for "mounted, but no
/// keys yet".
///
/// Vault answers a LIST on a mounted transit engine that holds no keys with a
/// 404 whose `errors` array is empty — the mount routed and answered the
/// request, so the engine is reachable. A 404 for a path with no mount behind
/// it instead carries a "no handler for route" message, so the empty `errors`
/// array is what separates "engine reachable but empty" from "engine missing".
///
/// An empty non-transit engine (e.g. KV v1) at the configured path answers
/// with byte-identical 404s, so this probe cannot detect that misconfiguration
/// — no LIST-based probe can. The data path still fails hard on the first real
/// transit operation against such a mount.
fn is_empty_transit_list(error: &ClientError) -> bool {
matches!(error, ClientError::APIError { code: 404, errors } if errors.is_empty())
}
#[derive(Debug, Clone)]
struct TransitKeyMetadata {
key_usage: KeyUsage,
@@ -114,7 +132,12 @@ struct TransitKeyMetadata {
}
/// Serializable version of TransitKeyMetadata for KV v2 persistence.
#[derive(Debug, Clone, Serialize, Deserialize)]
///
/// `Deserialize` is hand-written so fields the current build does not know
/// are counted and warned about instead of vanishing silently — this record
/// is compatibility-bound in both directions (older and newer builds read
/// each other's writes), so `deny_unknown_fields` is not an option.
#[derive(Debug, Clone, Serialize)]
struct TransitKeyMetadataPersisted {
key_usage: KeyUsage,
description: Option<String>,
@@ -127,6 +150,168 @@ struct TransitKeyMetadataPersisted {
current_version: u32,
}
impl UnknownFieldSummary {
fn record_for_transit_key_metadata(&self) {
let Some((field, field_name_truncated, field_count)) = self.record("vault-transit-key-metadata") else {
return;
};
static RECORDS_WITH_UNKNOWN_FIELDS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let observed_records = RECORDS_WITH_UNKNOWN_FIELDS
.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
.saturating_add(1);
if observed_records.is_power_of_two() {
tracing::warn!(
field = ?field,
field_name_truncated,
field_count,
observed_records,
"Vault Transit key metadata record contains unknown fields"
);
}
}
}
impl<'de> Deserialize<'de> for TransitKeyMetadataPersisted {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
use serde::de::{self, IgnoredAny, MapAccess, Visitor};
use std::fmt;
enum Field {
KeyUsage,
Description,
Tags,
KeyState,
CreatedAt,
DeletionDate,
Origin,
CreatedBy,
CurrentVersion,
Unknown(BoundedUnknownFieldName),
}
impl<'de> Deserialize<'de> for Field {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
struct FieldVisitor;
impl Visitor<'_> for FieldVisitor {
type Value = Field;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("a Vault Transit key metadata field name")
}
fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E>
where
E: de::Error,
{
Ok(match value {
"key_usage" => Field::KeyUsage,
"description" => Field::Description,
"tags" => Field::Tags,
"key_state" => Field::KeyState,
"created_at" => Field::CreatedAt,
"deletion_date" => Field::DeletionDate,
"origin" => Field::Origin,
"created_by" => Field::CreatedBy,
"current_version" => Field::CurrentVersion,
_ => Field::Unknown(BoundedUnknownFieldName::new(value)),
})
}
}
deserializer.deserialize_identifier(FieldVisitor)
}
}
struct TransitKeyMetadataPersistedVisitor;
impl<'de> Visitor<'de> for TransitKeyMetadataPersistedVisitor {
type Value = TransitKeyMetadataPersisted;
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("a Vault Transit key metadata record")
}
fn visit_map<A>(self, mut map: A) -> std::result::Result<Self::Value, A::Error>
where
A: MapAccess<'de>,
{
macro_rules! read_field {
($slot:ident, $name:literal) => {{
if $slot.is_some() {
return Err(de::Error::duplicate_field($name));
}
$slot = Some(map.next_value()?);
}};
}
let mut key_usage = None;
let mut description = None;
let mut tags = None;
let mut key_state = None;
let mut created_at = None;
let mut deletion_date = None;
let mut origin = None;
let mut created_by = None;
let mut current_version = None;
let mut unknown_fields = UnknownFieldSummary::default();
while let Some(field) = map.next_key()? {
match field {
Field::KeyUsage => read_field!(key_usage, "key_usage"),
Field::Description => read_field!(description, "description"),
Field::Tags => read_field!(tags, "tags"),
Field::KeyState => read_field!(key_state, "key_state"),
Field::CreatedAt => read_field!(created_at, "created_at"),
Field::DeletionDate => read_field!(deletion_date, "deletion_date"),
Field::Origin => read_field!(origin, "origin"),
Field::CreatedBy => read_field!(created_by, "created_by"),
Field::CurrentVersion => read_field!(current_version, "current_version"),
Field::Unknown(field) => {
let _: IgnoredAny = map.next_value()?;
unknown_fields.observe(field);
}
}
}
let metadata = TransitKeyMetadataPersisted {
key_usage: key_usage.ok_or_else(|| de::Error::missing_field("key_usage"))?,
description: description.unwrap_or(None),
tags: tags.ok_or_else(|| de::Error::missing_field("tags"))?,
key_state: key_state.ok_or_else(|| de::Error::missing_field("key_state"))?,
created_at: created_at.ok_or_else(|| de::Error::missing_field("created_at"))?,
deletion_date: deletion_date.unwrap_or(None),
origin: origin.ok_or_else(|| de::Error::missing_field("origin"))?,
created_by: created_by.unwrap_or(None),
current_version: current_version.ok_or_else(|| de::Error::missing_field("current_version"))?,
};
unknown_fields.record_for_transit_key_metadata();
Ok(metadata)
}
}
const FIELDS: &[&str] = &[
"key_usage",
"description",
"tags",
"key_state",
"created_at",
"deletion_date",
"origin",
"created_by",
"current_version",
];
deserializer.deserialize_struct("TransitKeyMetadataPersisted", FIELDS, TransitKeyMetadataPersistedVisitor)
}
}
impl TransitKeyMetadata {
fn from_create_request(request: &CreateKeyRequest) -> Self {
Self {
@@ -706,6 +891,7 @@ impl VaultTransitKmsClient {
created_by: metadata.created_by,
rotation_due: false,
rotation_due_reason: None,
wrap_budget_reserved: None,
})
}
@@ -1074,12 +1260,17 @@ impl VaultTransitKmsClient {
let mut all_keys = self
.run("vault_transit_list_keys", OpClass::ReadIdempotent, move || async move {
let vault = self.vault().map_err(AttemptError::fatal)?;
key::list(&vault.client, &self.config.mount_path).await.map_err(|e| {
AttemptError::from_vaultrs(e, |e| KmsError::backend_error(format!("Failed to list Vault Transit keys: {e}")))
})
match key::list(&vault.client, &self.config.mount_path).await {
Ok(response) => Ok(response.keys),
// An empty transit engine answers LIST with a bare 404;
// that is an empty listing, not a backend failure.
Err(error) if is_empty_transit_list(&error) => Ok(Vec::new()),
Err(e) => Err(AttemptError::from_vaultrs(e, |e| {
KmsError::backend_error(format!("Failed to list Vault Transit keys: {e}"))
})),
}
})
.await?
.keys;
.await?;
// Vault's own LIST ordering is not part of its contract, so the sort is
// what makes the marker a stable cursor across calls.
all_keys.sort_unstable();
@@ -1252,12 +1443,17 @@ impl VaultTransitKmsClient {
pub(crate) async fn health_check(&self) -> Result<()> {
self.run("vault_transit_health_check", OpClass::ReadIdempotent, move || async move {
let vault = self.vault().map_err(AttemptError::fatal)?;
key::list(&vault.client, &self.config.mount_path)
.await
.map(|_| ())
.map_err(|e| {
AttemptError::from_vaultrs(e, |e| KmsError::backend_error(format!("Vault Transit health check failed: {e}")))
})
match key::list(&vault.client, &self.config.mount_path).await {
Ok(_) => Ok(()),
// A brand-new transit mount holds no keys until something
// creates one, and this check gates startup before the service
// creates its own probe key — treating "empty" as unhealthy
// would keep a first-ever deployment from ever starting.
Err(error) if is_empty_transit_list(&error) => Ok(()),
Err(e) => Err(AttemptError::from_vaultrs(e, |e| {
KmsError::backend_error(format!("Vault Transit health check failed: {e}"))
})),
}
})
.await
}
@@ -1916,6 +2112,107 @@ mod tests {
);
}
/// Regression test for the first-boot chicken-and-egg on a fresh transit
/// mount (rustfs/backlog#1774).
///
/// Vault answers a LIST on a mounted-but-empty transit engine with a 404
/// carrying an empty `errors` array. The health check gates startup before
/// the service creates its probe key, so this 404 must count as healthy —
/// failing it means a first-ever deployment on a fresh mount can never
/// start until an operator creates some transit key out-of-band.
#[tokio::test]
async fn health_check_passes_on_an_empty_transit_engine() {
let (vault, client) = scripted_client(vec![ScriptedResponse::Http {
status: 404,
body: serde_json::json!({ "errors": [] }).to_string(),
}])
.await;
client
.health_check()
.await
.expect("an empty transit engine is reachable and must pass the health check");
let requests = vault.requests();
assert_eq!(
requests,
vec!["LIST /v1/transit/keys".to_string()],
"the empty-list 404 must be accepted on the first attempt, not retried"
);
}
/// A 404 whose body says "no handler for route" means no transit engine is
/// mounted at the configured path at all; that must keep failing the
/// health check instead of riding the empty-engine allowance.
#[tokio::test]
async fn health_check_fails_when_the_transit_mount_is_missing() {
let (_vault, client) = scripted_client(vec![ScriptedResponse::error(
404,
"no handler for route \"transit/keys\". route entry not found.",
)])
.await;
let error = client
.health_check()
.await
.expect_err("a missing transit mount must fail the health check");
assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}");
}
/// The empty-engine allowance is scoped to 404 alone: any other status
/// whose body happens to carry an empty `errors` array (an intermediary
/// answering for Vault, for instance) must keep failing the health check.
#[tokio::test]
async fn health_check_fails_on_a_non_404_error_with_an_empty_errors_body() {
let (_vault, client) = scripted_client(vec![ScriptedResponse::Http {
status: 403,
body: serde_json::json!({ "errors": [] }).to_string(),
}])
.await;
let error = client
.health_check()
.await
.expect_err("only a 404 may ride the empty-engine allowance");
assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}");
}
/// The listing's own copy of the discriminator must not widen into "every
/// LIST failure is an empty listing" — a missing mount still fails loudly.
#[tokio::test]
async fn list_fails_when_the_transit_mount_is_missing() {
let (_vault, client) = scripted_client(vec![ScriptedResponse::error(
404,
"no handler for route \"transit/keys\". route entry not found.",
)])
.await;
let error = client
.list_keys(&ListKeysRequest::default(), None)
.await
.expect_err("a missing transit mount must fail the listing, not empty it");
assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}");
}
/// The same empty-engine 404 on the listing path is an empty result set,
/// not a backend failure.
#[tokio::test]
async fn list_keys_returns_an_empty_page_on_an_empty_transit_engine() {
let (_vault, client) = scripted_client(vec![ScriptedResponse::Http {
status: 404,
body: serde_json::json!({ "errors": [] }).to_string(),
}])
.await;
let response = client
.list_keys(&ListKeysRequest::default(), None)
.await
.expect("an empty transit engine must list as empty, not fail");
assert!(response.keys.is_empty(), "got {:?}", response.keys);
assert!(!response.truncated, "an empty listing has nothing left to page through");
assert_eq!(response.next_marker, None);
}
fn test_vault_transit_config() -> VaultTransitConfig {
VaultTransitConfig {
address: "http://127.0.0.1:8200".to_string(),
@@ -2133,6 +2430,41 @@ mod tests {
assert!(metadata.deletion_date.is_none());
}
#[test]
fn transit_key_metadata_unknown_fields_remain_readable_and_are_observed() {
// A record written by a newer build carries fields this build does not
// know. It must stay readable — and the drop must be visible, not
// silent (rustfs/backlog#1641). Only the field name may be logged.
let persisted: TransitKeyMetadataPersisted = TransitKeyMetadata::synthesized().into();
let mut value = serde_json::to_value(&persisted).expect("serialize metadata record");
let object = value.as_object_mut().expect("metadata record serializes to an object");
object.insert("field_from_the_future".to_string(), serde_json::json!("field value must not be logged"));
let logs = crate::test_support::CapturedLogs::default();
let subscriber = tracing_subscriber::fmt()
.with_ansi(false)
.with_max_level(tracing::Level::WARN)
.with_writer(logs.clone())
.finish();
let dispatch = tracing::Dispatch::new(subscriber);
let recorder = metrics_util::debugging::DebuggingRecorder::new();
let parsed: TransitKeyMetadataPersisted = metrics::with_local_recorder(&recorder, || {
tracing::dispatcher::with_default(&dispatch, || {
serde_json::from_value(value).expect("unknown fields must remain readable")
})
});
assert_eq!(parsed.key_state, KeyState::Enabled);
assert_eq!(crate::test_support::unknown_field_metric(&recorder, "vault-transit-key-metadata"), 1);
let output = logs.output();
assert!(
output.contains("Vault Transit key metadata record contains unknown fields"),
"got: {output}"
);
assert!(output.contains("field_from_the_future"));
assert!(!output.contains("field value must not be logged"));
}
/// KV2 write acknowledgement (`SecretVersionMetadata`) for `kv2::set`.
fn kv2_write_ack() -> serde_json::Value {
serde_json::json!({
crates/kms/src/config.rs
+135
View File
@@ -868,6 +868,14 @@ impl KmsConfig {
// `mount_path` is deprecated and unused by this backend, so an empty value
// is deliberately not an error.
// `kv_mount` is: it is the mount every read, write and listing is
// routed through, and an empty one produces a path Vault has no
// handler for. Rejecting it here names the setting; letting it
// through spends a round-trip to report an unroutable path.
if config.kv_mount.is_empty() {
return Err(KmsError::configuration_error("Vault KV2 mount cannot be empty"));
}
// Validate TLS configuration if using HTTPS
if config.address.starts_with("https://")
&& let Some(ref tls) = config.tls
@@ -1129,6 +1137,53 @@ pub fn allow_immediate_deletion_from_env() -> bool {
get_env_bool(ENV_KMS_ALLOW_IMMEDIATE_DELETION, false)
}
impl crate::persisted_observability::UnknownFieldSummary {
fn record_for_kms_config(&self) {
let Some((field, field_name_truncated, field_count)) = self.record("kms-config") else {
return;
};
static RECORDS_WITH_UNKNOWN_FIELDS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
let observed_records = RECORDS_WITH_UNKNOWN_FIELDS
.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
.saturating_add(1);
if observed_records.is_power_of_two() {
tracing::warn!(
field = ?field,
field_name_truncated,
field_count,
observed_records,
"persisted KMS configuration contains unknown fields"
);
}
}
}
/// Deserialize a persisted KMS configuration, observing ignored fields.
///
/// The persisted configuration deliberately tolerates unknown fields — a
/// rolling upgrade writes fields the previous build does not know, and
/// rejecting them would turn every upgrade into a hard stop (see the
/// regression test pinning that tolerance). Tolerated must not mean
/// invisible: this loader wraps the deserializer with `serde_ignored`, so
/// every field the configuration silently dropped is counted and sampled
/// into a warning, per the repository rule that formats too
/// compatibility-bound for `deny_unknown_fields` must at least log unknown
/// fields. Only field paths are recorded, never values — a mistyped field
/// name can sit next to a secret.
pub fn kms_config_from_persisted_json(data: &[u8]) -> serde_json::Result<KmsConfig> {
use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummary};
let mut deserializer = serde_json::Deserializer::from_slice(data);
let mut unknown_fields = UnknownFieldSummary::default();
let config: KmsConfig = serde_ignored::deserialize(&mut deserializer, |path| {
unknown_fields.observe(BoundedUnknownFieldName::new(&path.to_string()));
})?;
deserializer.end()?;
unknown_fields.record_for_kms_config();
Ok(config)
}
fn vault_tls_config(skip_tls_verify: bool) -> Option<TlsConfig> {
skip_tls_verify.then_some(TlsConfig {
ca_cert_path: None,
@@ -1920,6 +1975,34 @@ mod tests {
.expect("well-formed token file auth must validate");
}
/// Every KV2 read, write and listing is routed through `kv_mount`, so an
/// empty one names a path no Vault engine answers. The Transit backend
/// already rejects its own empty mounts; this closes the same gap on the
/// setting whose absence otherwise surfaces as an unroutable-path failure at
/// the first Vault call.
#[test]
fn test_validate_rejects_an_empty_kv2_mount() {
let kv2_config = |kv_mount: &str| KmsConfig {
backend: KmsBackend::VaultKv2,
backend_config: BackendConfig::VaultKv2(Box::new(VaultConfig {
address: "https://vault.example.com:8200".to_string(),
auth_method: VaultAuthMethod::Token {
token: "a-real-token".to_string(),
},
kv_mount: kv_mount.to_string(),
..Default::default()
})),
..Default::default()
};
let error = kv2_config("")
.validate()
.expect_err("an empty KV2 mount must be rejected as a configuration error");
assert!(error.to_string().contains("mount"), "got {error}");
kv2_config("secret").validate().expect("a named KV2 mount must validate");
}
#[test]
fn test_approle_config_deserializes_legacy_shape_with_defaults() {
// Persisted configurations from before the AppRole implementation only
@@ -1979,6 +2062,58 @@ mod tests {
});
}
#[test]
fn persisted_config_unknown_fields_remain_readable_and_are_observed() {
// Unknown fields in a persisted config are deliberately tolerated (a
// rolling upgrade writes fields the previous build does not know), but
// tolerated must not mean invisible (rustfs/backlog#1641): the
// observing loader counts and warns, naming only the field path —
// never the value, which can sit next to a secret. Coverage includes a
// field nested inside the backend variant, which the externally tagged
// enum exposes to the observer.
let mut value = serde_json::to_value(KmsConfig::default()).expect("serialize config");
value.as_object_mut().expect("config serializes to an object").insert(
"top_level_field_from_the_future".to_string(),
serde_json::json!("top-level value must not be logged"),
);
value
.pointer_mut("/backend_config/Local")
.expect("default config has a Local backend section")
.as_object_mut()
.expect("Local backend section is an object")
.insert(
"nested_field_from_the_future".to_string(),
serde_json::json!("nested value must not be logged"),
);
let data = serde_json::to_vec(&value).expect("encode config");
let logs = crate::test_support::CapturedLogs::default();
let subscriber = tracing_subscriber::fmt()
.with_ansi(false)
.with_max_level(tracing::Level::WARN)
.with_writer(logs.clone())
.finish();
let dispatch = tracing::Dispatch::new(subscriber);
let recorder = metrics_util::debugging::DebuggingRecorder::new();
let config = metrics::with_local_recorder(&recorder, || {
tracing::dispatcher::with_default(&dispatch, || {
kms_config_from_persisted_json(&data).expect("unknown fields must remain readable")
})
});
assert!(matches!(config.backend_config, BackendConfig::Local(_)));
assert_eq!(crate::test_support::unknown_field_metric(&recorder, "kms-config"), 2);
let output = logs.output();
assert!(output.contains("persisted KMS configuration contains unknown fields"), "got: {output}");
assert!(!output.contains("must not be logged"));
// A clean config observes nothing and logs nothing.
let clean = serde_json::to_vec(&KmsConfig::default()).expect("encode clean config");
let recorder = metrics_util::debugging::DebuggingRecorder::new();
metrics::with_local_recorder(&recorder, || kms_config_from_persisted_json(&clean).expect("clean config must parse"));
assert_eq!(crate::test_support::unknown_field_metric(&recorder, "kms-config"), 0);
}
#[test]
fn test_validate_rejects_incomplete_approle() {
let mut config = KmsConfig::vault_approle(
crates/kms/src/deletion_worker.rs
+86
View File
@@ -62,6 +62,12 @@ const METRIC_TOMBSTONE_KEYS: &str = "rustfs_kms_deletion_tombstone_keys";
/// Gauge: seconds since the least recently rotated usable key was rotated
/// (its creation time when it was never rotated); `0` when there are none.
const METRIC_OLDEST_ROTATION_AGE_SECONDS: &str = "rustfs_kms_oldest_key_rotation_age_seconds";
/// Gauge: the largest persisted wrap-operation reservation across usable keys,
/// as of the end of the last sweep that saw the whole key set. Published only
/// when the backend counts wraps (the Vault KV2 backend today); an aggregate
/// that by design overestimates actual wraps. The value to alert on against
/// the AES-256-GCM bound of 2^32 wraps per key material.
const METRIC_MAX_KEY_WRAP_OPERATIONS: &str = "rustfs_kms_max_key_wrap_operations";
/// Counter: keys the sweep acted on, by `outcome` (`removed`, `blocked`,
/// `skipped`, `failed`, `unreadable`).
const METRIC_SWEEP_KEYS_TOTAL: &str = "rustfs_kms_deletion_sweep_keys_total";
@@ -82,6 +88,10 @@ fn describe_metrics() {
METRIC_OLDEST_ROTATION_AGE_SECONDS,
"Seconds since the least recently rotated usable KMS key was last rotated, counting from creation for keys that were never rotated"
);
metrics::describe_gauge!(
METRIC_MAX_KEY_WRAP_OPERATIONS,
"Largest reserved wrap-operation count across usable KMS keys; overestimates actual wraps and is only reported by backends that count them"
);
metrics::describe_counter!(METRIC_SWEEP_KEYS_TOTAL, "Total keys acted on by the KMS deletion sweep, by outcome");
});
}
@@ -99,6 +109,11 @@ struct KeyCensus {
/// way out are excluded: they will never be rotated again, and would
/// otherwise pin the gauge high until the sweep finishes removing them.
oldest_rotation_age_seconds: f64,
/// Largest reserved wrap count across usable keys, `None` when no key
/// reported one — either the backend does not count wraps, or no usable
/// key was seen. Excluding departing keys mirrors the rotation age: their
/// material will never wrap again, so its consumed nonce budget is moot.
max_wrap_operations: Option<u64>,
}
impl KeyCensus {
@@ -107,6 +122,9 @@ impl KeyCensus {
KeyStatus::PendingDeletion => self.pending_deletion += 1,
KeyStatus::Deleted => self.tombstones += 1,
KeyStatus::Active | KeyStatus::Disabled => {
if let Some(reserved) = key.wrap_budget_reserved {
self.max_wrap_operations = Some(self.max_wrap_operations.unwrap_or(0).max(reserved));
}
// A missing rotation time means either "never rotated" or "the
// build that rotated it did not record when". Both fall back to
// creation, and the two are not worth separate series: for the
@@ -154,6 +172,11 @@ fn record_sweep(report: &SweepReport, census: Option<KeyCensus>) {
metrics::gauge!(METRIC_PENDING_DELETION_KEYS).set(census.pending_deletion as f64);
metrics::gauge!(METRIC_TOMBSTONE_KEYS).set(census.tombstones as f64);
metrics::gauge!(METRIC_OLDEST_ROTATION_AGE_SECONDS).set(census.oldest_rotation_age_seconds);
// Only emitted when a usable key reported a count: backends that do not
// count wraps must not publish a `0` that reads as "no wraps consumed".
if let Some(max_wrap_operations) = census.max_wrap_operations {
metrics::gauge!(METRIC_MAX_KEY_WRAP_OPERATIONS).set(max_wrap_operations as f64);
}
}
/// Reports configuration that still references a KMS key.
@@ -772,6 +795,7 @@ mod tests {
created_by: None,
rotation_due: false,
rotation_due_reason: None,
wrap_budget_reserved: None,
}
}
@@ -803,6 +827,63 @@ mod tests {
);
}
/// The wrap census is the max over usable keys that report a counter.
/// Keys without one (backends that do not count wraps) leave it `None`
/// rather than dragging in a zero, and departing keys are excluded — their
/// material never wraps again, so its consumed nonce budget is moot.
#[test]
fn census_takes_the_max_wrap_reservation_of_usable_keys_only() {
let now = Zoned::now();
let mut census = KeyCensus::default();
census.observe(&key_info("uncounted", KeyStatus::Active, now.clone(), None), &now);
assert_eq!(census.max_wrap_operations, None, "a key without a counter must not report zero");
let mut low = key_info("low", KeyStatus::Active, now.clone(), None);
low.wrap_budget_reserved = Some(1_000_000);
let mut high = key_info("high", KeyStatus::Disabled, now.clone(), None);
high.wrap_budget_reserved = Some(3_000_000);
let mut departing = key_info("departing", KeyStatus::PendingDeletion, now.clone(), None);
departing.wrap_budget_reserved = Some(9_000_000);
census.observe(&low, &now);
census.observe(&high, &now);
census.observe(&departing, &now);
assert_eq!(census.max_wrap_operations, Some(3_000_000));
}
/// The wrap gauge is a single aggregate: one value, no labels at all — a
/// per-key label would carry key identifiers into the metric stream and
/// grow the series count with the key set.
#[test]
fn wrap_budget_gauge_is_aggregate_and_carries_no_key_label() {
let (snapshot, ()) = record_metrics(|| {
Box::pin(async {
let now = Zoned::now();
let mut census = KeyCensus::default();
let mut wrapped = key_info("wrapped-key-id", KeyStatus::Active, now.clone(), None);
wrapped.wrap_budget_reserved = Some(2_000_000);
census.observe(&wrapped, &now);
record_sweep(&SweepReport::default(), Some(census));
})
});
assert_eq!(gauge_value(&snapshot, METRIC_MAX_KEY_WRAP_OPERATIONS), Some(2_000_000.0));
for (composite, ..) in &snapshot {
if composite.key().name() == METRIC_MAX_KEY_WRAP_OPERATIONS {
assert_eq!(composite.key().labels().count(), 0, "the wrap gauge must stay label-less");
}
for label in composite.key().labels() {
assert!(
!label.value().contains("wrapped-key-id"),
"metric {} leaked a key identifier through label {}",
composite.key().name(),
label.key()
);
}
}
}
#[test]
fn sweep_publishes_lifecycle_gauges_without_key_labels() {
let (snapshot, key_ids) = record_metrics(|| {
@@ -835,6 +916,11 @@ mod tests {
);
assert_eq!(counter_value(&snapshot, METRIC_SWEEP_KEYS_TOTAL, "skipped"), 1);
assert_eq!(counter_value(&snapshot, METRIC_SWEEP_KEYS_TOTAL, "removed"), 0);
assert_eq!(
gauge_value(&snapshot, METRIC_MAX_KEY_WRAP_OPERATIONS),
None,
"a backend that does not count wraps must not publish a wrap gauge that reads as zero consumption"
);
for (composite, ..) in &snapshot {
for label in composite.key().labels() {
crates/kms/src/manager.rs
+1
View File
@@ -1707,6 +1707,7 @@ mod tests {
created_by: None,
rotation_due: false,
rotation_due_reason: None,
wrap_budget_reserved: None,
}
}
crates/kms/src/service.rs
+12 -19
View File
@@ -27,6 +27,10 @@ use base64::Engine;
use jiff::Zoned;
use md5::{Digest as Md5Digest, Md5};
use rand::random;
use rustfs_utils::http::object_encryption_keys::{
INTERNAL_ENCRYPTION_ALGORITHM_HEADER, INTERNAL_ENCRYPTION_CONTEXT_HEADER, INTERNAL_ENCRYPTION_IV_HEADER,
INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, INTERNAL_ENCRYPTION_TAG_HEADER,
};
use std::collections::HashMap;
use std::io::Cursor;
use tokio::io::{AsyncRead, AsyncReadExt};
@@ -81,17 +85,6 @@ fn request_encryption_context(context: &ObjectEncryptionContext) -> HashMap<Stri
enc_context
}
const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id";
/// Carries the AEAD algorithm the object was sealed with.
///
/// The S3 `x-amz-server-side-encryption` header records the *SSE mode*
/// (`AES256` / `aws:kms`), not the cipher, so it cannot round-trip
/// `ChaCha20Poly1305`. Without this header a ChaCha-sealed object comes back
/// from the projection claiming `aws:kms` and is then opened with the wrong
/// cipher.
const INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "x-rustfs-encryption-algorithm";
/// Result of object encryption
#[derive(Debug, Clone)]
pub struct EncryptionResult {
@@ -807,19 +800,19 @@ impl ObjectEncryptionService {
// Internal headers for decryption
headers.insert(
"x-rustfs-encryption-iv".to_string(),
INTERNAL_ENCRYPTION_IV_HEADER.to_string(),
base64::engine::general_purpose::STANDARD.encode(&metadata.iv),
);
if let Some(ref tag) = metadata.tag {
headers.insert(
"x-rustfs-encryption-tag".to_string(),
INTERNAL_ENCRYPTION_TAG_HEADER.to_string(),
base64::engine::general_purpose::STANDARD.encode(tag),
);
}
headers.insert(
"x-rustfs-encryption-key".to_string(),
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
base64::engine::general_purpose::STANDARD.encode(&metadata.encrypted_data_key),
);
@@ -831,7 +824,7 @@ impl ObjectEncryptionService {
None => context_aad(&metadata.encryption_context).unwrap_or_default(),
};
headers.insert(
"x-rustfs-encryption-context".to_string(),
INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(),
String::from_utf8_lossy(&context_bytes).into_owned(),
);
@@ -876,13 +869,13 @@ impl ObjectEncryptionService {
};
let iv = headers
.get("x-rustfs-encryption-iv")
.get(INTERNAL_ENCRYPTION_IV_HEADER)
.ok_or_else(|| KmsError::validation_error("Missing IV header"))?;
let iv = base64::engine::general_purpose::STANDARD
.decode(iv)
.map_err(|e| KmsError::validation_error(format!("Invalid IV: {e}")))?;
let tag = if let Some(tag_str) = headers.get("x-rustfs-encryption-tag") {
let tag = if let Some(tag_str) = headers.get(INTERNAL_ENCRYPTION_TAG_HEADER) {
Some(
base64::engine::general_purpose::STANDARD
.decode(tag_str)
@@ -892,7 +885,7 @@ impl ObjectEncryptionService {
None
};
let encrypted_data_key = if let Some(key_str) = headers.get("x-rustfs-encryption-key") {
let encrypted_data_key = if let Some(key_str) = headers.get(INTERNAL_ENCRYPTION_KEY_HEADER) {
base64::engine::general_purpose::STANDARD
.decode(key_str)
.map_err(|e| KmsError::validation_error(format!("Invalid encrypted key: {e}")))?
@@ -904,7 +897,7 @@ impl ObjectEncryptionService {
// callers that inspect the context, but the bytes are carried through
// untouched: re-serializing the parsed map is exactly how the original
// ordering — and with it the ability to open the object — was lost.
let (encryption_context, context_aad) = match headers.get("x-rustfs-encryption-context") {
let (encryption_context, context_aad) = match headers.get(INTERNAL_ENCRYPTION_CONTEXT_HEADER) {
Some(context_str) => (
serde_json::from_str(context_str)
.map_err(|e| KmsError::validation_error(format!("Invalid encryption context: {e}")))?,
crates/kms/src/types.rs
+9
View File
@@ -259,6 +259,14 @@ pub struct KeyInfo {
/// verdict to explain.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub rotation_due_reason: Option<RotationDueReason>,
/// Wrap operations reserved against the key's current material, reported
/// only by backends that count wraps (the Vault KV2 backend today). An
/// approximate value that by design overestimates the wraps actually
/// performed. In-process transport for the deletion worker's aggregate
/// wrap gauge, deliberately kept off the serialized admin surface: per-key
/// exposure would need its own contract decision and snapshot pin.
#[serde(skip)]
pub wrap_budget_reserved: Option<u64>,
}
impl From<MasterKeyInfo> for KeyInfo {
@@ -277,6 +285,7 @@ impl From<MasterKeyInfo> for KeyInfo {
created_by: master_key.created_by,
rotation_due: false,
rotation_due_reason: None,
wrap_budget_reserved: None,
}
}
}
crates/kms/tests/common/mod.rs
+85 -3
View File
@@ -32,14 +32,14 @@ use std::collections::HashMap;
use std::fmt::Debug;
use std::future::Future;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::{Arc, Mutex};
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use rustfs_kms::backends::BackendCapabilities;
use rustfs_kms::{
CreateKeyRequest, KeyUsage, KmsConfig, KmsError, KmsManager, KmsServiceManager, KmsServiceStatus, ObjectEncryptionService,
Result,
CreateKeyRequest, DeleteKeyRequest, KeyUsage, KmsConfig, KmsError, KmsManager, KmsServiceManager, KmsServiceStatus,
ObjectEncryptionService, Result,
};
use tempfile::TempDir;
@@ -126,6 +126,12 @@ pub struct TestKms {
manager: Arc<KmsServiceManager>,
kind: BackendKind,
config: KmsConfig,
/// Ids of the keys [`TestKms::create_key`] created, so a run against a
/// persistent Vault can remove them afterwards instead of accumulating
/// `behavior-*` keys forever (rustfs/backlog#1774). Shared through an Arc
/// because the harness instance is consumed by the spec while the cleanup
/// runs after it.
created_keys: Arc<Mutex<Vec<String>>>,
/// Held for the harness lifetime so the local key directory outlives a
/// simulated process restart.
_dir: Option<TempDir>,
@@ -147,6 +153,7 @@ impl TestKms {
manager,
kind: BackendKind::Local,
config,
created_keys: Arc::new(Mutex::new(Vec::new())),
_dir: Some(dir),
}
}
@@ -166,6 +173,7 @@ impl TestKms {
manager,
kind: BackendKind::VaultKv2,
config,
created_keys: Arc::new(Mutex::new(Vec::new())),
_dir: None,
}
}
@@ -180,6 +188,7 @@ impl TestKms {
manager,
kind: BackendKind::VaultTransit,
config,
created_keys: Arc::new(Mutex::new(Vec::new())),
_dir: None,
}
}
@@ -192,6 +201,7 @@ impl TestKms {
manager,
kind: BackendKind::Static,
config,
created_keys: Arc::new(Mutex::new(Vec::new())),
_dir: None,
}
}
@@ -253,8 +263,75 @@ impl TestKms {
.await
.unwrap_or_else(|error| panic!("create_key({name}) should succeed on {}: {error:?}", self.kind.name()));
assert_eq!(response.key_id, name, "created key id must be the requested name");
self.created_keys
.lock()
.expect("created-keys lock")
.push(response.key_id.clone());
response.key_id
}
/// Handle to the ids [`Self::create_key`] recorded, for cleanup that runs
/// after a spec consumed the harness instance.
pub fn created_keys_handle(&self) -> Arc<Mutex<Vec<String>>> {
Arc::clone(&self.created_keys)
}
/// Remove this instance's recorded Vault keys; see [`cleanup_vault_keys`].
pub async fn cleanup(&self) {
cleanup_vault_keys(self.kind, &self.config, self.created_keys_handle()).await;
}
}
/// Best-effort removal of the Vault keys a harness instance created, so a
/// persistent dev Vault does not accumulate `behavior-*` keys across runs
/// (rustfs/backlog#1774). A no-op for the Local and Static backends, whose
/// state dies with the per-test temp directory.
///
/// The deletion runs on a fresh manager over the same configuration — the
/// case's own manager is consumed by the spec and may have been stopped by a
/// restart scenario — with the immediate-deletion gate enabled on the cleanup
/// configuration only, so the configuration under test keeps the gate at its
/// production default and specs asserting the gate's refusal stay honest.
///
/// Failures are reported but never panic: cleanup runs after the spec's own
/// assertions, and a Vault hiccup here must not turn a green behavior run red.
pub async fn cleanup_vault_keys(kind: BackendKind, config: &KmsConfig, created_keys: Arc<Mutex<Vec<String>>>) {
if !kind.is_vault() {
return;
}
let key_ids: Vec<String> = created_keys.lock().expect("created-keys lock").drain(..).collect();
if key_ids.is_empty() {
return;
}
let config = config.clone().with_immediate_deletion_allowed();
let manager = start_manager(&config).await;
let kms = manager.get_manager().await.expect("KMS manager should be running");
for key_id in key_ids {
// The Transit backend deletes in two steps (first call parks the key in
// PendingDeletion, the next call destroys it); KV2 destroys on the
// first call and reports KeyNotFound on the second.
for _ in 0..2 {
match kms
.delete_key(DeleteKeyRequest {
key_id: key_id.clone(),
pending_window_in_days: None,
force_immediate: Some(true),
confirm_key_id: Some(key_id.clone()),
})
.await
{
Ok(_) => continue,
Err(KmsError::KeyNotFound { .. }) => break,
Err(error) => {
eprintln!("vault key cleanup: could not delete {key_id}: {error:?}");
break;
}
}
}
}
if let Err(error) = manager.stop().await {
eprintln!("vault key cleanup: could not stop the cleanup manager: {error:?}");
}
}
async fn start_manager(config: &KmsConfig) -> Arc<KmsServiceManager> {
@@ -332,7 +409,12 @@ where
.chain(live_vault_backends());
for kind in kinds {
let case = BackendCase::new(kind).await;
// Captured before the spec consumes the case; keys the spec creates
// through the harness afterwards still land in the shared list.
let config = case.kms.config().clone();
let created_keys = case.kms.created_keys_handle();
spec(case).await;
cleanup_vault_keys(kind, &config, created_keys).await;
}
}
crates/lock/src/distributed_lock.rs
+6 -1
View File
@@ -479,7 +479,7 @@ pub struct DistributedLock {
/// Lock clients for this namespace
clients: Vec<Arc<dyn LockClient>>,
/// Namespace identifier
namespace: String,
namespace: Arc<str>,
/// Quorum size for exclusive/write operations
quorum: usize,
}
@@ -496,6 +496,11 @@ struct LockAcquireQuorumResult {
impl DistributedLock {
/// Create new distributed lock
pub fn new(namespace: String, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
Self::new_shared(namespace.into(), clients, quorum)
}
/// Create a distributed lock that shares an existing namespace allocation.
pub(crate) fn new_shared(namespace: Arc<str>, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
let q = if clients.len() <= 1 {
1
} else {
crates/lock/src/local_lock.rs
+6 -1
View File
@@ -28,12 +28,17 @@ pub struct LocalLock {
/// Global lock manager for fast local locks
manager: Arc<GlobalLockManager>,
/// Namespace identifier
namespace: String,
namespace: Arc<str>,
}
impl LocalLock {
/// Create new local lock
pub fn new(namespace: String, manager: Arc<GlobalLockManager>) -> Self {
Self::new_shared(namespace.into(), manager)
}
/// Create a local lock that shares an existing namespace allocation.
pub(crate) fn new_shared(namespace: Arc<str>, manager: Arc<GlobalLockManager>) -> Self {
Self { namespace, manager }
}
crates/lock/src/namespace/mod.rs
+10
View File
@@ -180,6 +180,11 @@ impl NamespaceLock {
Self::Local(LocalLock::new(namespace, manager))
}
/// Create a local namespace lock that shares an existing namespace allocation.
pub fn with_local_manager_shared(namespace: Arc<str>, manager: Arc<crate::GlobalLockManager>) -> Self {
Self::Local(LocalLock::new_shared(namespace, manager))
}
/// Create namespace lock with clients
/// Uses DistributedLock with appropriate quorum
pub fn with_clients(namespace: String, clients: Vec<Arc<dyn LockClient>>) -> Self {
@@ -195,6 +200,11 @@ impl NamespaceLock {
Self::Distributed(DistributedLock::new(namespace, clients, quorum))
}
/// Create a namespace lock that shares an existing namespace allocation.
pub fn with_clients_and_quorum_shared(namespace: Arc<str>, clients: Vec<Arc<dyn LockClient>>, quorum: usize) -> Self {
Self::Distributed(DistributedLock::new_shared(namespace, clients, quorum))
}
/// Get namespace identifier
pub fn namespace(&self) -> &str {
match self {
crates/lock/src/namespace/tests.rs
+10
View File
@@ -356,6 +356,16 @@ async fn test_namespace_lock_with_local_manager() {
assert_eq!(lock.namespace(), "local-ns");
}
#[tokio::test]
async fn namespace_lock_preserves_shared_namespace_storage() {
let namespace: Arc<str> = Arc::from("shared-namespace");
let namespace_ptr = Arc::as_ptr(&namespace);
let local = LocalLock::new_shared(namespace.clone(), Arc::new(GlobalLockManager::new()));
assert_eq!(local.namespace(), namespace.as_ref());
assert_eq!(local.namespace().as_ptr(), namespace_ptr.cast::<u8>());
}
#[tokio::test]
async fn test_namespace_lock_with_clients() {
let clients = vec![ClientFactory::create_local(), ClientFactory::create_local()];
crates/obs/src/metrics/collectors/scanner.rs
+1 -1
View File
@@ -113,7 +113,7 @@ pub struct ScannerStats {
pub current_cycle_usage_saves: u64,
/// Current scanner mode: 0 unknown or idle, 1 normal, 2 deep bitrot scan
pub current_scan_mode: u64,
/// Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded
/// Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded, 5 deferred
pub last_cycle_result: u64,
/// Last scanner partial cycle reason: 0 unknown, 1 runtime, 2 objects, 3 directories
pub last_cycle_partial_reason: u64,
crates/obs/src/metrics/schema/scanner.rs
+1 -1
View File
@@ -460,7 +460,7 @@ pub static SCANNER_CURRENT_SCAN_MODE_MD: LazyLock<MetricDescriptor> = LazyLock::
pub static SCANNER_LAST_CYCLE_RESULT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ScannerLastCycleResult,
"Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded.",
"Last scanner cycle result: 0 unknown, 1 success, 2 error, 3 partial, 4 superseded, 5 deferred.",
&[],
subsystems::SCANNER,
)
crates/policy/src/policy/action.rs
-31
View File
@@ -355,37 +355,6 @@ pub enum S3Action {
GetBucketQuotaAction,
}
// #[derive(Serialize, Deserialize, Hash, PartialEq, Eq, Clone, EnumString, IntoStaticStr, Debug, Copy)]
// #[serde(try_from = "&str", into = "&str")]
// pub enum AdminAction {
// #[strum(serialize = "admin:*")]
// AllActions,
// #[strum(serialize = "admin:Profiling")]
// ProfilingAdminAction,
// #[strum(serialize = "admin:ServerTrace")]
// TraceAdminAction,
// #[strum(serialize = "admin:ConsoleLog")]
// ConsoleLogAdminAction,
// #[strum(serialize = "admin:ServerInfo")]
// ServerInfoAdminAction,
// #[strum(serialize = "admin:OBDInfo")]
// HealthInfoAdminAction,
// #[strum(serialize = "admin:TopLocksInfo")]
// TopLocksAdminAction,
// #[strum(serialize = "admin:LicenseInfo")]
// LicenseInfoAdminAction,
// #[strum(serialize = "admin:BandwidthMonitor")]
// BandwidthMonitorAction,
// #[strum(serialize = "admin:InspectData")]
// InspectDataAction,
// #[strum(serialize = "admin:Prometheus")]
// PrometheusAdminAction,
// #[strum(serialize = "admin:ListServiceAccounts")]
// ListServiceAccountsAdminAction,
// #[strum(serialize = "admin:CreateServiceAccount")]
// CreateServiceAccountAdminAction,
// }
// AdminAction - admin policy action.
#[derive(Serialize, Deserialize, Hash, PartialEq, Eq, Clone, IntoStaticStr, Debug, Copy, EnumString)]
#[serde(try_from = "&str", into = "&str")]
crates/policy/src/policy/utils/path.rs
+9
View File
@@ -53,6 +53,15 @@ impl<'a> LazyBuf<'a> {
}
/// copy from golang(path.Clean)
///
/// DELIBERATE DUPLICATION — do not replace with `rustfs_utils::path::clean`.
/// This is a faithful port of Go's slash-only `path.Clean`, which is what S3
/// ARN/resource matching requires: policy resource paths are opaque S3 keys,
/// and a backslash in a key is object-name data, never a separator. The utils
/// version is Windows-aware (`filepath.Clean` semantics: converts backslashes
/// to forward slashes), so swapping it in would change policy evaluation on
/// Windows — a security-adjacent behavior change. Mirror note sits on the
/// utils implementation (backlog#1833).
pub fn clean(path: &str) -> String {
if path.is_empty() {
return ".".into();
crates/policy/src/utils.rs
-12
View File
@@ -53,16 +53,4 @@ mod tests {
assert!(!token.is_empty());
}
// #[test]
// fn test_extract_claims() {
// let claims = Claims {
// sub: "user1".to_string(),
// company: "example".to_string(),
// };
// let secret = "my_secret";
// let token = generate_jwt(&claims, secret).unwrap();
// let decoded_claims = extract_claims::<Claims>(&token, secret).unwrap();
// assert_eq!(decoded_claims.claims, claims);
// }
}
crates/protos/src/lib.rs
+1 -1
View File
@@ -2513,7 +2513,7 @@ mod tests {
json_field: "rename_data_resp",
bin_field: "rename_data_resp_bin",
},
json_encoder: "let rename_data_resp_json = compat_response_json(&rename_data_resp, false);",
json_encoder: "let rename_data_resp_json = compat_response_json(rename_data_resp, request_decoded_from_msgpack)",
},
];
crates/replication/src/http.rs
+47
View File
@@ -12,6 +12,26 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! DELIBERATE DUPLICATION — do not merge these declarations into the
//! rustfs-utils http module without a maintainer decision on the crate
//! boundary.
//!
//! The canonical owners of these interop-contract values live in the
//! rustfs-utils crate: `crates/utils/src/http/metadata_compat.rs` (dual
//! x-rustfs-internal-/x-minio-internal- metadata keys),
//! `crates/utils/src/http/header_compat.rs` (x-rustfs-/x-minio- header pairs),
//! and `crates/utils/src/http/headers.rs` (standard S3 header names). This
//! crate keeps a local copy because
//! `rustfs-replication` is a wire-contract crate that must stay free of
//! internal dependencies: `scripts/check_architecture_migration_rules.sh`
//! rejects any `rustfs-utils` import or dependency here ("replication crate
//! HTTP/helper contracts must not import or depend on rustfs-utils"), and the
//! same rule bans `rustfs-filemeta` and `rustfs-storage-api`.
//!
//! Drift protection lives in the test module below: every constant's literal
//! wire value is pinned, so a change on either side that breaks interop fails
//! this crate's tests rather than silently forking the contract.
use std::collections::HashMap;
const RUSTFS_INTERNAL_PREFIX: &str = "x-rustfs-internal-";
@@ -145,4 +165,31 @@ mod tests {
assert!(has_prefix_fold("X-Amz-Meta-Foo", "x-amz-meta-"));
assert!(!has_prefix_fold("X-Amz-Meta-Foo", "amz-meta"));
}
/// Pins every duplicated interop constant to its literal wire value. The
/// canonical owner lives in the rustfs-utils crate (see the module doc);
/// an arch guard forbids depending on it from this crate, so byte-for-byte
/// pinning here is what keeps the two copies from drifting apart.
#[test]
fn duplicated_interop_constants_pin_canonical_wire_values() {
use super::*;
assert_eq!(AMZ_BUCKET_REPLICATION_STATUS, "X-Amz-Replication-Status");
assert_eq!(AMZ_OBJECT_LOCK_LEGAL_HOLD, "X-Amz-Object-Lock-Legal-Hold");
assert_eq!(AMZ_OBJECT_LOCK_MODE, "X-Amz-Object-Lock-Mode");
assert_eq!(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, "X-Amz-Object-Lock-Retain-Until-Date");
assert_eq!(AMZ_OBJECT_TAGGING, "X-Amz-Tagging");
assert_eq!(AMZ_WEBSITE_REDIRECT_LOCATION, "x-amz-website-redirect-location");
assert_eq!(CACHE_CONTROL, "Cache-Control");
assert_eq!(CONTENT_DISPOSITION, "Content-Disposition");
assert_eq!(CONTENT_ENCODING, "Content-Encoding");
assert_eq!(CONTENT_LANGUAGE, "Content-Language");
assert_eq!(EXPIRES, "Expires");
assert_eq!(SSEC_ALGORITHM_HEADER, "x-amz-server-side-encryption-customer-algorithm");
assert_eq!(SSEC_KEY_HEADER, "x-amz-server-side-encryption-customer-key");
assert_eq!(SSEC_KEY_MD5_HEADER, "x-amz-server-side-encryption-customer-key-md5");
assert_eq!(SUFFIX_ACTUAL_SIZE, "actual-size");
assert_eq!(SUFFIX_REPLICATION_STATUS, "replication-status");
assert_eq!(SUFFIX_REPLICATION_RESET_STATUS, "replication-reset-status");
}
}
crates/rio/src/checksum.rs
+9
View File
@@ -30,6 +30,15 @@ pub const RUSTFS_MULTIPART_CHECKSUM: &str = "x-rustfs-multipart-checksum";
pub const RUSTFS_MULTIPART_CHECKSUM_TYPE: &str = "x-rustfs-multipart-checksum-type";
/// Checksum type enumeration with flags
///
/// One of three deliberately separate checksum registries (backlog#1833):
/// this bitset owns the **on-disk xl.meta encoding** — the raw `u32` is
/// varint-serialized into xl.meta (see `append_to`), so bits are append-only
/// and must never be renumbered. `rustfs_checksums::ChecksumAlgorithm`
/// (crates/checksums/src/lib.rs) owns the streaming-hash algorithm registry,
/// and the MinIO-port client keeps its own `ChecksumMode`
/// (crates/ecstore/src/client/checksum.rs). When adding an algorithm, extend
/// all three (or record why not) — they do not derive from each other.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct ChecksumType(pub u32);
crates/s3select-query/src/instance.rs
+2 -2
View File
@@ -292,7 +292,7 @@ mod tests {
}
#[tokio::test]
#[ignore]
#[ignore = "requires a live RustFS store with a pre-seeded test object (bucket 'dandan')"]
async fn test_simple_sql() {
let sql = "select * from S3Object";
let input = SelectObjectContentInput {
@@ -354,7 +354,7 @@ mod tests {
}
#[tokio::test]
#[ignore]
#[ignore = "requires a live RustFS store with a pre-seeded test object (bucket 'dandan')"]
async fn test_func_sql() {
let sql = "SELECT * FROM S3Object s";
let input = SelectObjectContentInput {
crates/scanner/src/scanner.rs
+106 -42
View File
@@ -30,8 +30,8 @@ use crate::runtime_config::{
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason};
use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob};
use crate::scanner_io::{
ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified, dirty_usage_buckets_pending, dirty_usage_generation,
scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation,
ScannerCycleDeferReason, ScannerCycleStatus, ScannerIOCycle, dirty_usage_bucket_notified, dirty_usage_buckets_pending,
dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation,
};
use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed};
use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGuard};
@@ -41,7 +41,8 @@ use chrono::{DateTime, Utc};
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource,
emit_scan_cycle_complete, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, global_metrics,
emit_scan_cycle_complete, emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded,
global_metrics,
};
use rustfs_config::ScannerSpeed;
#[cfg(test)]
@@ -84,7 +85,7 @@ const METRIC_SCANNER_LEADER_LOCK_TOTAL: &str = "rustfs_scanner_leader_lock_total
const CLEAN_IDLE_MAX_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
const MAX_SCANNER_SCHEDULE_DELAY: Duration = Duration::from_secs(365 * 24 * 60 * 60);
const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
/// First-retry delay after a usage snapshot is superseded by concurrent writes.
/// First-retry delay after a scanner cycle cannot publish authoritative usage.
///
/// A superseded cycle is the *expected* outcome of the dirty-usage fast path:
/// a write burst marks buckets dirty, the scanner wakes within milliseconds,
@@ -94,14 +95,15 @@ const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
/// otherwise idle instance whose clean-idle backoff had doubled a 60 s
/// interval), which defeats the fast path it is meant to protect.
///
/// The exponential growth in [`ScannerSupersededBackoff::retry_interval`] is
/// The exponential growth in [`ScannerRetryBackoff::retry_interval`] is
/// what protects against a persistently hot bucket driving an unbroken
/// full-scan loop, so it can start small: 5 s, 10 s, 20 s … capped by
/// [`SUPERSEDED_RETRY_MAX_INTERVAL`]. A one-off race recovers in seconds; a
/// [`SCANNER_RETRY_MAX_INTERVAL`]. A one-off race recovers in seconds; a
/// genuinely hot bucket still reaches minute-scale backoff within a handful of
/// cycles.
const SUPERSEDED_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5);
const SUPERSEDED_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60);
/// cycles. Preflight deferrals use the same bounded schedule so a temporarily
/// unavailable peer cannot drive a tight retry loop.
const SCANNER_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5);
const SCANNER_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60);
const SCANNER_LEADER_LOCK_POLL_INTERVAL: Duration = Duration::from_secs(1);
#[cfg(not(test))]
const SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30);
@@ -338,6 +340,7 @@ pub(crate) enum ScannerCycleOutcome {
CompletedWithPendingMaintenance,
Partial,
Superseded,
Deferred(ScannerCycleDeferReason),
Failed,
}
@@ -382,22 +385,16 @@ struct ScannerCleanIdleBackoff {
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct ScannerSupersededBackoff {
struct ScannerRetryBackoff {
consecutive_cycles: u32,
}
impl ScannerSupersededBackoff {
fn record_cycle(&mut self, outcome: ScannerCycleOutcome) {
match outcome {
ScannerCycleOutcome::Superseded => {
self.consecutive_cycles = self.consecutive_cycles.saturating_add(1);
}
ScannerCycleOutcome::Completed
| ScannerCycleOutcome::CompletedWithPendingMaintenance
| ScannerCycleOutcome::Partial
| ScannerCycleOutcome::Failed => {
self.consecutive_cycles = 0;
}
impl ScannerRetryBackoff {
fn record_retryable_cycle(&mut self, retryable: bool) {
if retryable {
self.consecutive_cycles = self.consecutive_cycles.saturating_add(1);
} else {
self.consecutive_cycles = 0;
}
}
@@ -406,8 +403,8 @@ impl ScannerSupersededBackoff {
let multiplier = 1u32.checked_shl(exponent).unwrap_or(u32::MAX);
let base_interval = configured_interval
.max(Duration::from_secs(1))
.min(SUPERSEDED_RETRY_BASE_INTERVAL);
let cap = SUPERSEDED_RETRY_MAX_INTERVAL.max(configured_interval.max(Duration::from_secs(1)));
.min(SCANNER_RETRY_BASE_INTERVAL);
let cap = SCANNER_RETRY_MAX_INTERVAL.max(configured_interval.max(Duration::from_secs(1)));
Some(base_interval.saturating_mul(multiplier).min(cap))
}
}
@@ -3008,6 +3005,21 @@ async fn run_data_scanner_cycle(
ScannerCycleOutcome::Failed
};
}
ScannerCycleOutcome::Deferred(reason) => {
info!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
cycle = cycle_info.current,
reason = reason.as_str(),
state = "deferred",
"Scanner cycle deferred before usage scanning began"
);
emit_scan_cycle_deferred(cycle_start.elapsed());
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Deferred(reason);
}
ScannerCycleOutcome::Superseded => {
info!(
target: "rustfs::scanner",
@@ -3201,7 +3213,8 @@ async fn run_data_scanner_with_maintenance_state(
let mut dirty_usage_generation_seen = dirty_usage_generation();
let mut runtime_config_generation_seen = scanner_runtime_config_generation();
let mut clean_idle_backoff = ScannerCleanIdleBackoff::default();
let mut superseded_backoff = ScannerSupersededBackoff::default();
let mut superseded_backoff = ScannerRetryBackoff::default();
let mut deferred_backoff = ScannerRetryBackoff::default();
let initial_runtime_config = resolve_scanner_runtime_config();
if clean_idle_topology_supported
&& scanner_clean_idle_backoff_configured(&initial_runtime_config)
@@ -3331,7 +3344,8 @@ async fn run_data_scanner_with_maintenance_state(
)
.await
.unwrap_or(ScannerCycleOutcome::Failed);
superseded_backoff.record_cycle(initial_outcome);
superseded_backoff.record_retryable_cycle(initial_outcome == ScannerCycleOutcome::Superseded);
deferred_backoff.record_retryable_cycle(matches!(initial_outcome, ScannerCycleOutcome::Deferred(_)));
dirty_usage_generation_seen = dirty_generation_before_cycle;
if guard.is_lock_lost() {
record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await;
@@ -3416,7 +3430,9 @@ async fn run_data_scanner_with_maintenance_state(
let mut wait_plan =
scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for);
let superseded_retry_interval = superseded_backoff.retry_interval(runtime_config.cycle_interval);
if let Some(retry_interval) = superseded_retry_interval {
let deferred_retry_interval = deferred_backoff.retry_interval(runtime_config.cycle_interval);
let convergence_retry_interval = superseded_retry_interval.or(deferred_retry_interval);
if let Some(retry_interval) = convergence_retry_interval {
wait_plan.effective_interval = retry_interval;
wait_plan.delay = randomized_cycle_delay_for(retry_interval).min(retry_interval);
}
@@ -3443,6 +3459,8 @@ async fn run_data_scanner_with_maintenance_state(
clean_idle_backoff_enabled = backoff_enabled,
superseded_retry_backoff_enabled = superseded_retry_interval.is_some(),
superseded_cycles = superseded_backoff.consecutive_cycles,
deferred_retry_backoff_enabled = deferred_retry_interval.is_some(),
deferred_cycles = deferred_backoff.consecutive_cycles,
lifecycle_active = maintenance_features.lifecycle,
replication_active = maintenance_features.replication,
feature_inspection_failed = maintenance_features.inspection_failed,
@@ -3457,13 +3475,12 @@ async fn run_data_scanner_with_maintenance_state(
activity_poll_interval,
&mut scanner_activity_seen,
ScannerCycleObservedGenerations {
// A superseded cycle already observed concurrent writes. Hold
// further dirty notifications until the bounded retry timer so
// a hot bucket cannot drive an unbroken full-scan loop.
dirty_usage: superseded_retry_interval.is_none().then_some(dirty_usage_generation_seen),
// A non-converged cycle holds further activity notifications
// until its bounded retry timer to avoid an unbroken scan loop.
dirty_usage: convergence_retry_interval.is_none().then_some(dirty_usage_generation_seen),
runtime_config: runtime_config_generation_seen,
maintenance: maintenance_generation_before_wait,
defer_cluster_activity: superseded_retry_interval.is_some(),
defer_cluster_activity: convergence_retry_interval.is_some(),
},
|| guard.is_lock_lost(),
|| probe_scanner_activity(storeapi.as_ref(), distributed),
@@ -3540,7 +3557,8 @@ async fn run_data_scanner_with_maintenance_state(
)
.await
.unwrap_or(ScannerCycleOutcome::Failed);
superseded_backoff.record_cycle(outcome);
superseded_backoff.record_retryable_cycle(outcome == ScannerCycleOutcome::Superseded);
deferred_backoff.record_retryable_cycle(matches!(outcome, ScannerCycleOutcome::Deferred(_)));
dirty_usage_generation_seen = dirty_generation_before_cycle;
if guard.is_lock_lost() {
record_scanner_leader_lock_lost("Scanner leader lock lost during a scanner cycle").await;
@@ -3710,6 +3728,12 @@ fn scanner_cycle_completion_outcome(
) -> ScannerCycleOutcome {
match (scan_status, usage_persist_outcome) {
(_, DataUsagePersistOutcome::Failed) => ScannerCycleOutcome::Failed,
(ScannerCycleStatus::Deferred(reason), DataUsagePersistOutcome::NoUpdate)
if !has_dirty_usage && !has_failed_dirty_usage =>
{
ScannerCycleOutcome::Deferred(reason)
}
(ScannerCycleStatus::Deferred(_), _) => ScannerCycleOutcome::Failed,
(ScannerCycleStatus::Superseded, _) if !has_failed_dirty_usage => ScannerCycleOutcome::Superseded,
(ScannerCycleStatus::Superseded, _) => ScannerCycleOutcome::Failed,
(
@@ -6553,6 +6577,46 @@ mod tests {
#[test]
fn test_scanner_cycle_completion_prioritizes_persist_failure() {
assert_eq!(
scanner_cycle_completion_outcome(
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
DataUsagePersistOutcome::NoUpdate,
false,
false,
),
ScannerCycleOutcome::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
);
assert_eq!(
scanner_cycle_completion_outcome(
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
DataUsagePersistOutcome::Saved,
false,
false,
),
ScannerCycleOutcome::Failed
);
assert_eq!(
scanner_cycle_completion_outcome(
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
DataUsagePersistOutcome::NoUpdate,
true,
false,
),
ScannerCycleOutcome::Failed
);
assert_eq!(
scanner_cycle_completion_outcome(
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
DataUsagePersistOutcome::Failed,
false,
false,
),
ScannerCycleOutcome::Failed
);
assert_eq!(
scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::NoUpdate, false, false),
ScannerCycleOutcome::Failed
);
assert_eq!(
scanner_cycle_completion_outcome(ScannerCycleStatus::Incomplete, DataUsagePersistOutcome::Failed, true, true),
ScannerCycleOutcome::Failed
@@ -6940,47 +7004,47 @@ mod tests {
#[test]
fn superseded_retry_backoff_grows_caps_and_resets_after_convergence() {
let mut backoff = ScannerSupersededBackoff::default();
let mut backoff = ScannerRetryBackoff::default();
assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None);
for expected in [5, 10, 20, 40, 80, 160, 320] {
backoff.record_cycle(ScannerCycleOutcome::Superseded);
backoff.record_retryable_cycle(true);
assert_eq!(
backoff.retry_interval(Duration::from_secs(24 * 60 * 60)),
Some(Duration::from_secs(expected))
);
}
for _ in 0..20 {
backoff.record_cycle(ScannerCycleOutcome::Superseded);
backoff.record_retryable_cycle(true);
}
assert_eq!(
backoff.retry_interval(Duration::from_secs(24 * 60 * 60)),
Some(Duration::from_secs(24 * 60 * 60))
);
backoff.record_cycle(ScannerCycleOutcome::Completed);
backoff.record_retryable_cycle(false);
assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None);
}
#[test]
fn superseded_retry_backoff_respects_a_faster_configured_cycle() {
let mut backoff = ScannerSupersededBackoff::default();
backoff.record_cycle(ScannerCycleOutcome::Superseded);
let mut backoff = ScannerRetryBackoff::default();
backoff.record_retryable_cycle(true);
// A configured cycle shorter than the base still wins: retrying sooner
// than the operator's own cadence buys nothing.
assert_eq!(backoff.retry_interval(Duration::from_secs(3)), Some(Duration::from_secs(3)));
backoff.record_cycle(ScannerCycleOutcome::Superseded);
backoff.record_retryable_cycle(true);
assert_eq!(backoff.retry_interval(Duration::from_secs(3)), Some(Duration::from_secs(6)));
}
#[test]
fn superseded_retry_backoff_grows_from_the_default_cycle() {
let mut backoff = ScannerSupersededBackoff::default();
let mut backoff = ScannerRetryBackoff::default();
// The first race after a write burst retries in seconds, not a whole
// cycle, while repeated supersedes still climb toward the cap.
for expected in [5, 10, 20, 40] {
backoff.record_cycle(ScannerCycleOutcome::Superseded);
backoff.record_retryable_cycle(true);
assert_eq!(backoff.retry_interval(Duration::from_secs(60)), Some(Duration::from_secs(expected)));
}
}
crates/scanner/src/scanner_io.rs
+106 -15
View File
@@ -2194,11 +2194,45 @@ pub(crate) async fn scanner_set_disk_inventory(set: &SetDisks) -> Vec<Arc<Disk>>
disks
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ScannerCycleDeferReason {
ActivityBaselineUnavailable,
DataMovement,
}
impl ScannerCycleDeferReason {
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::ActivityBaselineUnavailable => "activity_baseline_unavailable",
Self::DataMovement => "data_movement",
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ScannerCycleStatus {
Complete,
Incomplete,
Superseded,
Deferred(ScannerCycleDeferReason),
}
enum ScannerActivityPreflight {
Ready(crate::scanner::ScannerActivitySnapshot),
ActivityBaselineUnavailable(String),
DataMovement,
}
fn scanner_activity_preflight(
activity: std::result::Result<crate::scanner::ScannerActivitySnapshot, String>,
) -> ScannerActivityPreflight {
match activity {
Err(error) => ScannerActivityPreflight::ActivityBaselineUnavailable(error),
Ok(snapshot) if !crate::scanner::scanner_activity_allows_usage_publication(&snapshot) => {
ScannerActivityPreflight::DataMovement
}
Ok(snapshot) => ScannerActivityPreflight::Ready(snapshot),
}
}
#[derive(Debug)]
@@ -2307,9 +2341,9 @@ impl ScannerIOCycle for ECStore {
let child_token = ctx.child_token();
let distributed = self.setup_is_dist_erasure().await;
let activity_before = match crate::scanner::probe_scanner_activity(self, distributed).await {
Ok(snapshot) => snapshot,
Err(err) => {
let activity_before = match scanner_activity_preflight(crate::scanner::probe_scanner_activity(self, distributed).await) {
ScannerActivityPreflight::Ready(snapshot) => snapshot,
ScannerActivityPreflight::ActivityBaselineUnavailable(err) => {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
@@ -2319,20 +2353,26 @@ impl ScannerIOCycle for ECStore {
error = %err,
"Scanner cycle skipped because cluster activity could not be baselined"
);
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
return Ok(ScannerCycleResult::new(
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable),
None,
));
}
ScannerActivityPreflight::DataMovement => {
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "cycle_data_movement_active",
"Scanner cycle deferred while rebalance or decommission data movement is active"
);
return Ok(ScannerCycleResult::new(
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement),
None,
));
}
};
if !crate::scanner::scanner_activity_allows_usage_publication(&activity_before) {
debug!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "cycle_data_movement_active",
"Scanner cycle deferred while rebalance or decommission data movement is active"
);
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None));
}
let dirty_generation_before_bucket_list = dirty_usage_generation();
let bucket_listing = self.list_bucket_for_scanner(&BucketOptions::default()).await?;
let mut bucket_plan_complete = bucket_listing.topology_complete;
@@ -3982,6 +4022,7 @@ mod tests {
use super::*;
use crate::scanner_budget::ScannerCycleBudgetConfig;
use crate::scanner_folder::ScannerItem;
use crate::storage_api::owner::{EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats};
use crate::storage_api::scan::{BucketOperations as _, MakeBucketOptions, ObjectIO as _};
use crate::{
DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerObjectOptions,
@@ -4004,6 +4045,20 @@ mod tests {
}
}
#[test]
fn scanner_activity_preflight_defers_a_temporarily_offline_peer() {
let preflight = scanner_activity_preflight(Err("peer rustfs-node3:9000 is temporarily offline".to_string()));
match preflight {
ScannerActivityPreflight::ActivityBaselineUnavailable(error) => {
assert_eq!(error, "peer rustfs-node3:9000 is temporarily offline");
}
ScannerActivityPreflight::Ready(_) | ScannerActivityPreflight::DataMovement => {
panic!("an unavailable activity baseline must defer the scanner cycle");
}
}
}
async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc<ECStore>) {
init_ecstore_config_for_scanner_tests();
let temp_dir = tempfile::tempdir().expect("multi-pool scanner test directory should be created");
@@ -4096,6 +4151,42 @@ mod tests {
assert!(!second.is_lock_lost());
}
#[tokio::test]
#[serial]
async fn scanner_cycle_is_deferred_while_rebalance_is_active() {
let (_temp_dir, store) = setup_two_pool_scanner_store().await;
let mut pool_stats = vec![EcstoreRebalanceStats::default(); store.pools.len()];
pool_stats[0] = EcstoreRebalanceStats {
participating: true,
info: EcstoreRebalanceInfo {
start_time: Some(OffsetDateTime::now_utc()),
status: EcstoreRebalStatus::Started,
..Default::default()
},
..Default::default()
};
*store.rebalance_meta.write().await = Some(EcstoreRebalanceMeta {
id: Uuid::new_v4().to_string(),
pool_stats,
..Default::default()
});
assert!(store.scanner_data_movement_active().await);
let ctx = CancellationToken::new();
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
let (updates, mut receiver) = mpsc::channel(1);
let result = tokio::time::timeout(
Duration::from_secs(30),
ScannerIOCycle::nsscanner_with_status(store.as_ref(), ctx, budget, updates, 1, 1, HealScanMode::Normal),
)
.await
.expect("rebalance-deferred scanner cycle should finish")
.expect("rebalance-deferred scanner cycle should succeed");
assert_eq!(result.status, ScannerCycleStatus::Deferred(ScannerCycleDeferReason::DataMovement));
assert!(receiver.recv().await.is_none(), "rebalance-deferred cycle must not publish usage");
}
#[tokio::test]
async fn data_usage_publish_fails_when_receiver_is_closed() {
let (updates, receiver) = mpsc::channel(1);
crates/scanner/src/storage_api.rs
+8 -2
View File
@@ -83,6 +83,11 @@ pub(crate) use rustfs_ecstore::api::layout::{
EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rebalance::{
RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta,
RebalanceStats as EcstoreRebalanceStats,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext;
pub(crate) use rustfs_ecstore::api::runtime::{
expiry_state_handle as ecstore_expiry_state_handle, global_tier_config_mgr as ecstore_get_global_tier_config_mgr,
@@ -122,8 +127,9 @@ pub(crate) mod owner {
#[cfg(test)]
pub(crate) use super::{
EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, EcstoreEndpointServerPools, EcstoreEndpoints,
EcstoreInstanceContext, EcstorePoolEndpoints, ecstore_config_init, ecstore_init_bucket_metadata_sys,
ecstore_init_local_disks_with_instance_ctx, ecstore_new_disk,
EcstoreInstanceContext, EcstorePoolEndpoints, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta,
EcstoreRebalanceStats, ecstore_config_init, ecstore_init_bucket_metadata_sys, ecstore_init_local_disks_with_instance_ctx,
ecstore_new_disk,
};
}
crates/targets/src/target/nats/jetstream.rs
+4 -4
View File
@@ -760,7 +760,7 @@ mod tests {
}
#[tokio::test]
#[ignore]
#[ignore = "requires a live NATS JetStream server (docker run nats:2 -js; RUSTFS_TEST_NATS_URL overrides)"]
async fn tls_change_rebuilds_the_context_and_drains_the_old_acker() {
// A TLS fingerprint change on the publish path rebuilds the cached context from the new client and drains the old acker.
let subject = format!("rustfs.tlsrebuild.{}", Uuid::new_v4().simple());
@@ -839,7 +839,7 @@ mod tests {
}
#[tokio::test]
#[ignore]
#[ignore = "requires a live NATS JetStream server (docker run nats:2 -js; RUSTFS_TEST_NATS_URL overrides)"]
async fn tls_change_after_a_failed_reconnect_still_rebuilds_the_context() {
// A rotation detected while the broker is unreachable does not orphan the cached context: a failed reconnect followed by a successful one ends bound to the rebuilt context.
let subject = format!("rustfs.tlsfail.{}", Uuid::new_v4().simple());
@@ -917,7 +917,7 @@ mod tests {
}
#[tokio::test]
#[ignore]
#[ignore = "requires a live NATS JetStream server (docker run nats:2 -js; RUSTFS_TEST_NATS_URL overrides)"]
async fn publish_gate_rejects_an_unsafe_stream_and_heals_after_the_stream_is_fixed() {
// The gate rejects every publish while the stream's duplicate window is below the retry lifetime, and starts publishing once the operator widens it, without a restart.
let subject = format!("rustfs.gate.{}", Uuid::new_v4().simple());
@@ -975,7 +975,7 @@ mod tests {
}
#[tokio::test]
#[ignore]
#[ignore = "requires a live NATS JetStream server (docker run nats:2 -js; RUSTFS_TEST_NATS_URL overrides)"]
async fn a_remapped_subject_is_rejected_by_the_ack_stream_check_and_the_entry_stays_queued() {
// After a subject remap the takeover stream acknowledges, so the ack-stream check rejects it with the mismatch detail, keeps the entry queued, and resets the verdict for re-validation.
let subject = format!("rustfs.remap.{}", Uuid::new_v4().simple());
crates/targets/tests/mysql_integration.rs
+6 -6
View File
@@ -107,7 +107,7 @@ async fn drop_table(dsn: &str, table: &str) {
.await;
}
#[ignore]
#[ignore = "requires a live MySQL 8.0+/TiDB instance (see module docs for the container command)"]
#[tokio::test]
async fn direct_write_and_read() {
let dsn = test_dsn();
@@ -131,7 +131,7 @@ async fn direct_write_and_read() {
drop_table(&dsn, &table).await;
}
#[ignore]
#[ignore = "requires a live MySQL 8.0+/TiDB instance (see module docs for the container command)"]
#[tokio::test]
async fn delete_appends_row_does_not_remove_old() {
let dsn = test_dsn();
@@ -155,7 +155,7 @@ async fn delete_appends_row_does_not_remove_old() {
drop_table(&dsn, &table).await;
}
#[ignore]
#[ignore = "requires a live MySQL 8.0+/TiDB instance (see module docs for the container command)"]
#[tokio::test]
async fn queue_store_saves_entry_and_replays() {
let dsn = test_dsn();
@@ -195,7 +195,7 @@ async fn queue_store_saves_entry_and_replays() {
drop_table(&dsn, &table).await;
}
#[ignore]
#[ignore = "requires a live MySQL 8.0+/TiDB instance (see module docs for the container command)"]
#[tokio::test]
async fn duplicate_replay_produces_duplicate_rows() {
let dsn = test_dsn();
@@ -236,7 +236,7 @@ async fn duplicate_replay_produces_duplicate_rows() {
drop_table(&dsn, &table).await;
}
#[ignore]
#[ignore = "requires a live MySQL 8.0+/TiDB instance (see module docs for the container command)"]
#[tokio::test]
async fn incompatible_schema_init_fails() {
let dsn = test_dsn();
@@ -267,7 +267,7 @@ async fn incompatible_schema_init_fails() {
drop_table(&dsn, &table).await;
}
#[ignore]
#[ignore = "requires a live MySQL 8.0+/TiDB instance (see module docs for the container command)"]
#[tokio::test]
async fn check_mysql_server_available_succeeds_against_existing_table() {
let dsn = test_dsn();
crates/targets/tests/nats_jetstream_regression_guards.rs
+1 -1
View File
@@ -181,7 +181,7 @@ fn jetstream_args(subject: &str, stream_name: &str, queue_dir: &str) -> NATSArgs
///
/// Ignored by default because it needs a running NATS server with JetStream.
#[tokio::test]
#[ignore]
#[ignore = "requires a live NATS JetStream server (see module docs; RUSTFS_TEST_NATS_URL overrides)"]
async fn end_to_end_publish_is_acked_on_the_stream() {
use rustfs_targets::EventName;
use rustfs_targets::Target;
crates/targets/tests/nats_jetstream_validation_integration.rs
+4 -4
View File
@@ -82,7 +82,7 @@ async fn remove_stream(stream_name: &str) {
}
#[tokio::test]
#[ignore]
#[ignore = "requires a live NATS JetStream server (see module docs; RUSTFS_TEST_NATS_URL overrides)"]
async fn missing_stream_fails_the_health_check() {
let stream_name = format!("RUSTFS_TEST_{}", Uuid::new_v4().simple());
let args = jetstream_args("rustfs.events", &stream_name);
@@ -94,7 +94,7 @@ async fn missing_stream_fails_the_health_check() {
}
#[tokio::test]
#[ignore]
#[ignore = "requires a live NATS JetStream server (see module docs; RUSTFS_TEST_NATS_URL overrides)"]
async fn valid_stream_passes_the_health_check() {
let stream_name = format!("RUSTFS_TEST_{}", Uuid::new_v4().simple());
let subject = "rustfs.events";
@@ -106,7 +106,7 @@ async fn valid_stream_passes_the_health_check() {
}
#[tokio::test]
#[ignore]
#[ignore = "requires a live NATS JetStream server (see module docs; RUSTFS_TEST_NATS_URL overrides)"]
async fn stream_not_capturing_the_subject_fails_the_health_check() {
let stream_name = format!("RUSTFS_TEST_{}", Uuid::new_v4().simple());
// The stream binds a different subject than the target publishes to.
@@ -118,7 +118,7 @@ async fn stream_not_capturing_the_subject_fails_the_health_check() {
}
#[tokio::test]
#[ignore]
#[ignore = "requires a live NATS JetStream server (see module docs; RUSTFS_TEST_NATS_URL overrides)"]
async fn too_small_duplicate_window_fails_the_health_check() {
let stream_name = format!("RUSTFS_TEST_{}", Uuid::new_v4().simple());
let subject = "rustfs.events";
crates/utils/src/http/header_compat.rs
+9 -2
View File
@@ -17,14 +17,21 @@
//!
//! Use suffix-based API: `get_header(headers, SUFFIX_FORCE_DELETE)` queries both
//! x-rustfs-force-delete and x-minio-force-delete.
//!
//! This module is the canonical owner of these interop values. One deliberate
//! copy exists: `crates/replication/src/http.rs` re-declares the subset it
//! needs because the wire-contract crate must stay free of internal
//! dependencies (arch guard in `scripts/check_architecture_migration_rules.sh`
//! bans replication -> rustfs-utils). When changing a value here, check the
//! pinned copy there; its tests pin the shared wire values byte-for-byte.
use http::{HeaderMap, HeaderValue};
use std::borrow::Cow;
const RUSTFS_PREFIX: &str = "x-rustfs-";
const MINIO_PREFIX: &str = "x-minio-";
const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";
pub const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
pub const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";
const MINIO_INTERNAL_ENCRYPTION_PREFIX: &str = "x-minio-internal-server-side-encryption-";
const MINIO_INTERNAL_ENCRYPTED_MULTIPART: &str = "x-minio-internal-encrypted-multipart";
const RUSTFS_ENCRYPTION_ORIGINAL_SIZE: &str = super::object_encryption_keys::INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER;
crates/utils/src/http/headers.rs
+5 -1
View File
@@ -156,7 +156,11 @@ pub const REQUEST_ID_HEADER: &str = "x-request-id";
pub const AMZ_REQUEST_ID: &str = "x-amz-request-id";
pub const AMZ_REQUEST_HOST_ID: &str = "x-amz-id-2";
// Content Checksums
// Content Checksums. The standard five x-amz-checksum-* names also exist in
// the zero-internal-dependency rustfs-checksums leaf crate
// (crates/checksums/src/http.rs, which additionally owns the RustFS
// extension names); values are pinned by the S3 wire protocol — keep both
// sides in sync (backlog#1833).
pub const AMZ_CHECKSUM_ALGO: &str = "x-amz-checksum-algorithm";
pub const AMZ_CHECKSUM_CRC32: &str = "x-amz-checksum-crc32";
pub const AMZ_CHECKSUM_CRC32C: &str = "x-amz-checksum-crc32c";
crates/utils/src/http/metadata_compat.rs
+7
View File
@@ -14,6 +14,13 @@
//! System metadata compatibility: write both x-rustfs-internal-* and x-minio-internal-*
//! for MinIO interoperability. Read prefers RustFS, fallback to MinIO.
//!
//! This module is the canonical owner of these interop values. One deliberate
//! copy exists: `crates/replication/src/http.rs` re-declares the subset it
//! needs because the wire-contract crate must stay free of internal
//! dependencies (arch guard in `scripts/check_architecture_migration_rules.sh`
//! bans replication -> rustfs-utils). When changing a value here, check the
//! pinned copy there; its tests pin the shared wire values byte-for-byte.
use std::collections::{BTreeMap, HashMap};
crates/utils/src/http/object_encryption_keys.rs
+21 -4
View File
@@ -30,6 +30,13 @@ use super::headers::{SSEC_ALGORITHM_HEADER, SSEC_KEY_MD5_HEADER};
pub const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id";
pub const INTERNAL_ENCRYPTION_KEY_HEADER: &str = "x-rustfs-encryption-key";
pub const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv";
/// Carries the AEAD algorithm the object was sealed with.
///
/// The S3 `x-amz-server-side-encryption` header records the *SSE mode*
/// (`AES256` / `aws:kms`), not the cipher, so it cannot round-trip
/// `ChaCha20Poly1305`. Without this header a ChaCha-sealed object comes back
/// from the projection claiming `aws:kms` and is then opened with the wrong
/// cipher.
pub const INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "x-rustfs-encryption-algorithm";
pub const INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER: &str = "x-rustfs-encryption-original-size";
pub const INTERNAL_ENCRYPTION_CONTEXT_HEADER: &str = "x-rustfs-encryption-context";
@@ -45,6 +52,15 @@ pub const MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER: &str = "X-Minio-Internal-
pub const MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key";
pub const MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Context";
/// Reserved RustFS-branded twin of the MinIO-internal SSE key family.
///
/// No RustFS writer emits these keys today — the SSE writer persists the
/// MinIO-branded `X-Minio-Internal-Server-Side-Encryption-*` keys verbatim for
/// interoperability — but redaction (`rustfs_filemeta`) and replication
/// stripping treat the family as sensitive so that a future or third-party
/// writer cannot leak sealed material through the reserved names.
pub const RUSTFS_INTERNAL_ENCRYPTION_PREFIX: &str = "x-rustfs-internal-server-side-encryption-";
pub const REPLICATION_SSEC_ALGORITHM_HEADER: &str = "X-Rustfs-Replication-Ssec-Algorithm";
pub const REPLICATION_SSEC_KEY_MD5_HEADER: &str = "X-Rustfs-Replication-Ssec-Key-Md5";
pub const REPLICATION_SSEC_ORIGINAL_SIZE_HEADER: &str = "X-Rustfs-Replication-Ssec-Original-Size";
@@ -125,13 +141,14 @@ pub fn ssec_replication_transport_header(stored_key: &str) -> Option<&'static st
/// SSE-C material. SSE-C passthrough re-adds its keys through the transport
/// mapping instead.
pub fn is_replication_stripped_encryption_key(key: &str) -> bool {
// The dual-key invariant writes an x-rustfs-internal- twin next to every
// x-minio-internal- SSE key; cover it here so this predicate is safe to
// use standalone, without an is_internal_key backstop.
// The x-rustfs-internal- SSE prefix is a reserved name family with no
// writer today (see RUSTFS_INTERNAL_ENCRYPTION_PREFIX); cover it here so
// this predicate is safe to use standalone, without an is_internal_key
// backstop.
super::is_encryption_metadata_key(key)
|| super::is_sse_header(key)
|| key.eq_ignore_ascii_case(SSEC_ORIGINAL_SIZE_HEADER)
|| super::starts_with_ignore_ascii_case(key, "x-rustfs-internal-server-side-encryption-")
|| super::starts_with_ignore_ascii_case(key, RUSTFS_INTERNAL_ENCRYPTION_PREFIX)
}
#[cfg(test)]

Some files were not shown because too many files have changed in this diff Show More