Compare commits

..

2 Commits

Author SHA1 Message Date
overtrue 4ea0934d45 test(utils): allow ephemeral port reuse 2026-08-15 07:13:26 +08:00
overtrue 3c88e8df8a fix(ci): pin Docker images to release source 2026-08-15 06:37:48 +08:00
186 changed files with 2393 additions and 17799 deletions
Generated
-9
View File
@@ -278,7 +278,6 @@ checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c"
dependencies = [ dependencies = [
"bigdecimal", "bigdecimal",
"bon", "bon",
"crc32fast",
"digest 0.11.3", "digest 0.11.3",
"log", "log",
"miniz_oxide 0.9.1", "miniz_oxide 0.9.1",
@@ -290,11 +289,9 @@ dependencies = [
"serde", "serde",
"serde_bytes", "serde_bytes",
"serde_json", "serde_json",
"snap",
"strum", "strum",
"thiserror 2.0.20", "thiserror 2.0.20",
"uuid", "uuid",
"zstd",
] ]
[[package]] [[package]]
@@ -9203,7 +9200,6 @@ dependencies = [
"serial_test", "serial_test",
"sha2 0.11.0", "sha2 0.11.0",
"shadow-rs", "shadow-rs",
"snap",
"socket2", "socket2",
"subtle", "subtle",
"sysinfo", "sysinfo",
@@ -9825,19 +9821,14 @@ name = "rustfs-madmin"
version = "1.0.0-rc.2" version = "1.0.0-rc.2"
dependencies = [ dependencies = [
"hotpath", "hotpath",
"http 1.5.0",
"humantime", "humantime",
"hyper", "hyper",
"jiff", "jiff",
"reqwest",
"rmp-serde", "rmp-serde",
"rustfs-signer",
"s3s",
"serde", "serde",
"serde_json", "serde_json",
"sysinfo", "sysinfo",
"time", "time",
"tokio",
] ]
[[package]] [[package]]
+1 -1
View File
@@ -171,7 +171,7 @@ tower = { version = "0.5.3" }
tower-http = { version = "0.7.0" } tower-http = { version = "0.7.0" }
# Serialization and Data Formats # Serialization and Data Formats
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] } apache-avro = "0.22.0"
bytes = { version = "1.12.1" } bytes = { version = "1.12.1" }
bytesize = "2.7.0" bytesize = "2.7.0"
byteorder = "1.5.0" byteorder = "1.5.0"
+1
View File
@@ -11,6 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#![allow(dead_code)]
use base64_simd::STANDARD; use base64_simd::STANDARD;
+1 -4
View File
@@ -38,10 +38,7 @@ pub const XXHASH_3_HEADER_NAME: &str = "x-amz-checksum-xxhash3";
pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64"; pub const XXHASH_64_HEADER_NAME: &str = "x-amz-checksum-xxhash64";
pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128"; pub const XXHASH_128_HEADER_NAME: &str = "x-amz-checksum-xxhash128";
#[allow( #[allow(dead_code)]
dead_code,
reason = "Content-MD5 wire name, resolved by header_name() below and asserted by this crate's tests (backlog#1823)"
)]
pub(crate) static MD5_HEADER_NAME: &str = "content-md5"; pub(crate) static MD5_HEADER_NAME: &str = "content-md5";
pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] = pub const CHECKSUM_ALGORITHMS_IN_PRIORITY_ORDER: [&str; 5] =
+2 -8
View File
@@ -476,19 +476,13 @@ impl Checksum for Xxhash64 {
} }
} }
#[allow(dead_code)]
#[derive(Debug, Default)] #[derive(Debug, Default)]
#[allow(
dead_code,
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
)]
struct Md5 { struct Md5 {
hasher: md5::Md5, hasher: md5::Md5,
} }
#[allow( #[allow(dead_code)]
dead_code,
reason = "Content-MD5 is not a ChecksumAlgorithm variant and has no arm in into_impl: S3 carries it as its own header, separate from the x-amz-checksum-* family. This impl exists so the two paths share the Checksum trait, and is asserted by this crate's tests (backlog#1823)"
)]
impl Md5 { impl Md5 {
fn update(&mut self, bytes: &[u8]) { fn update(&mut self, bytes: &[u8]) {
use md5::Digest; use md5::Digest;
+1 -91
View File
@@ -76,18 +76,6 @@ const SOURCE_MTIME_HEADERS: [&str; 2] = ["x-rustfs-source-mtime", "x-minio-sourc
const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] = const SOURCE_REPLICATION_REQUEST_HEADERS: [&str; 2] =
["x-rustfs-source-replication-request", "x-minio-source-replication-request"]; ["x-rustfs-source-replication-request", "x-minio-source-replication-request"];
const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"]; const SOURCE_ETAG_HEADERS: [&str; 2] = ["x-rustfs-source-etag", "x-minio-source-etag"];
const SOURCE_TAGGING_TIMESTAMP_HEADERS: [&str; 2] = [
"x-rustfs-source-replication-tagging-timestamp",
"x-minio-source-replication-tagging-timestamp",
];
const SOURCE_RETENTION_TIMESTAMP_HEADERS: [&str; 2] = [
"x-rustfs-source-replication-retention-timestamp",
"x-minio-source-replication-retention-timestamp",
];
const SOURCE_LEGALHOLD_TIMESTAMP_HEADERS: [&str; 2] = [
"x-rustfs-source-replication-legalhold-timestamp",
"x-minio-source-replication-legalhold-timestamp",
];
const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"]; const RESERVED_BUCKET_PREFIXES: [&str; 3] = ["xn--", "sthree-", "amzn-s3-demo-"];
const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"]; const RESERVED_BUCKET_SUFFIXES: [&str; 6] = ["-s3alias", "--ol-s3", ".mrap", "--x-s3", "--table-s3", "-an"];
@@ -130,25 +118,6 @@ pub enum FaultAction {
WrongEtag, WrongEtag,
} }
/// Replication LWW timestamp headers observed on a request, journaled so
/// sender-side tests can assert what a real target would receive.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ReplicationTimestampHeaders {
pub tagging: Option<String>,
pub retention: Option<String>,
pub legalhold: Option<String>,
}
impl ReplicationTimestampHeaders {
fn from_headers(headers: &HeaderMap) -> Self {
Self {
tagging: header_value(headers, &SOURCE_TAGGING_TIMESTAMP_HEADERS).map(bounded_journal_value),
retention: header_value(headers, &SOURCE_RETENTION_TIMESTAMP_HEADERS).map(bounded_journal_value),
legalhold: header_value(headers, &SOURCE_LEGALHOLD_TIMESTAMP_HEADERS).map(bounded_journal_value),
}
}
}
/// Credential-free request metadata retained for deterministic assertions. /// Credential-free request metadata retained for deterministic assertions.
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub struct RequestRecord { pub struct RequestRecord {
@@ -162,7 +131,6 @@ pub struct RequestRecord {
pub part_number: Option<i32>, pub part_number: Option<i32>,
pub content_length: Option<u64>, pub content_length: Option<u64>,
pub consumed_bytes: Option<usize>, pub consumed_bytes: Option<usize>,
pub replication_timestamps: ReplicationTimestampHeaders,
pub fault: Option<FaultAction>, pub fault: Option<FaultAction>,
} }
@@ -568,15 +536,7 @@ impl S3Access for FaultAccess {
.get(CONTENT_LENGTH) .get(CONTENT_LENGTH)
.and_then(|value| value.to_str().ok()) .and_then(|value| value.to_str().ok())
.and_then(|value| value.parse().ok()); .and_then(|value| value.parse().ok());
let replication_timestamps = ReplicationTimestampHeaders::from_headers(context.headers()); let fault = record_request(&self.control, operation, context.method().clone(), parsed, content_length);
let fault = record_request(
&self.control,
operation,
context.method().clone(),
parsed,
content_length,
replication_timestamps,
);
if let Some(RequestFault { if let Some(RequestFault {
action: FaultAction::Status(status), action: FaultAction::Status(status),
.. ..
@@ -629,7 +589,6 @@ fn record_request(
method: Method, method: Method,
parsed: ParsedRequest, parsed: ParsedRequest,
content_length: Option<u64>, content_length: Option<u64>,
replication_timestamps: ReplicationTimestampHeaders,
) -> Option<RequestFault> { ) -> Option<RequestFault> {
let mut state = lock(control); let mut state = lock(control);
let action = parsed let action = parsed
@@ -654,7 +613,6 @@ fn record_request(
part_number: parsed.part_number, part_number: parsed.part_number,
content_length, content_length,
consumed_bytes: None, consumed_bytes: None,
replication_timestamps,
fault: action.clone(), fault: action.clone(),
}); });
action.map(|action| RequestFault { sequence, action }) action.map(|action| RequestFault { sequence, action })
@@ -1741,52 +1699,6 @@ mod tests {
.await?) .await?)
} }
#[tokio::test]
async fn journals_replication_timestamp_headers() -> Result<(), BoxError> {
let target = FakeS3Target::start().await?;
target.create_bucket("target-bucket");
let client = client(&target);
client
.put_object()
.bucket("target-bucket")
.key("plain")
.body(ByteStream::from_static(b"plain"))
.send()
.await?;
client
.put_object()
.bucket("target-bucket")
.key("stamped")
.body(ByteStream::from_static(b"stamped"))
.customize()
.map_request(move |mut request| {
let headers = request.headers_mut();
headers.insert("x-rustfs-source-replication-tagging-timestamp", "2026-01-02T03:04:05Z");
headers.insert("x-minio-source-replication-retention-timestamp", "2026-01-02T03:04:06Z");
headers.insert("x-rustfs-source-replication-legalhold-timestamp", "2026-01-02T03:04:07Z");
Ok::<_, std::convert::Infallible>(request)
})
.send()
.await?;
let requests = target.requests();
let plain = requests
.iter()
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("plain"))
.expect("plain PUT must be journaled");
assert_eq!(plain.replication_timestamps, ReplicationTimestampHeaders::default());
let stamped = requests
.iter()
.find(|record| record.operation == Operation::PutObject && record.key.as_deref() == Some("stamped"))
.expect("stamped PUT must be journaled");
assert_eq!(stamped.replication_timestamps.tagging.as_deref(), Some("2026-01-02T03:04:05Z"));
assert_eq!(stamped.replication_timestamps.retention.as_deref(), Some("2026-01-02T03:04:06Z"));
assert_eq!(stamped.replication_timestamps.legalhold.as_deref(), Some("2026-01-02T03:04:07Z"));
Ok(())
}
macro_rules! assert_sdk_error { macro_rules! assert_sdk_error {
($error:expr, $status:expr, $code:expr) => {{ ($error:expr, $status:expr, $code:expr) => {{
let error = &$error; let error = &$error;
@@ -3073,7 +2985,6 @@ mod tests {
part_number: None, part_number: None,
}, },
Some(0), Some(0),
ReplicationTimestampHeaders::default(),
); );
} }
let records = lock(&control).requests.clone(); let records = lock(&control).requests.clone();
@@ -3095,7 +3006,6 @@ mod tests {
part_number: None, part_number: None,
}, },
None, None,
ReplicationTimestampHeaders::default(),
); );
{ {
let bounded_records = lock(&bounded_control); let bounded_records = lock(&bounded_control);
+14 -16
View File
@@ -135,8 +135,7 @@ pub mod bucket {
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe; pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
pub use crate::bucket::metadata_sys::{ pub use crate::bucket::metadata_sys::{
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock, BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete, capture_bucket_metadata_incarnation, delete, delete_if_incarnation, get, get_accelerate_config, get_bucket_policy,
delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config, get_object_lock_config, get_object_lock_config_state, get_public_access_block_config, get_quota_config,
@@ -185,18 +184,17 @@ pub mod bucket {
mrf_backlog_observability_snapshot, mrf_backlog_observability_snapshot,
}; };
pub use crate::bucket::replication::{ pub use crate::bucket::replication::{
BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats, BucketReplicationResyncStatus, BucketReplicationStats, BucketStats, DeleteReplicationConfigSnapshot,
DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry,
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION,
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo,
ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig, ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigStructureError, ReplicationConfigurationExt,
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput, ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge,
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO, ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission,
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge, ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage,
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError, ReplicationTargetValidationError, ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog,
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus, TargetReplicationResyncStatus, VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
VersionPurgeStatusType, XferStats, commit_force_delete_intent, complete_force_delete_intent,
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool, delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field, get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
@@ -373,8 +371,8 @@ pub mod error {
pub mod erasure { pub mod erasure {
pub use crate::erasure::coding::{ pub use crate::erasure::coding::{
BitrotReader, BitrotSelfTestError, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, ErasureConstructionError, ReedSolomonEncoder,
ReedSolomonEncoder, bitrot_self_test, calc_shard_size, calc_shard_size_legacy, calc_shard_size, calc_shard_size_legacy,
}; };
} }
+5 -70
View File
@@ -58,9 +58,7 @@ use rustfs_utils::http::{
}; };
use rustfs_utils::http::{ use rustfs_utils::http::{
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK,
SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_REPLICATION_REQUEST, SUFFIX_SOURCE_VERSION_ID, insert_header,
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_SOURCE_VERSION_ID,
insert_header,
}; };
use rustls_pki_types::pem::PemObject; use rustls_pki_types::pem::PemObject;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@@ -82,6 +80,7 @@ use tracing::warn;
use url::Url; use url::Url;
use uuid::Uuid; use uuid::Uuid;
const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16; const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
const REDACTED_CREDENTIAL: &str = "<redacted>"; const REDACTED_CREDENTIAL: &str = "<redacted>";
@@ -1477,12 +1476,9 @@ impl Default for AdvancedPutOptions {
replication_status: ReplicationStatusType::Pending, replication_status: ReplicationStatusType::Pending,
source_mtime: OffsetDateTime::now_utc(), source_mtime: OffsetDateTime::now_utc(),
replication_request: false, replication_request: false,
// UNIX_EPOCH means "never modified": header() must not emit a retention_timestamp: OffsetDateTime::now_utc(),
// timestamp header for it, otherwise a receiver would treat an tagging_timestamp: OffsetDateTime::now_utc(),
// unset category as a modification made right now. legalhold_timestamp: OffsetDateTime::now_utc(),
retention_timestamp: OffsetDateTime::UNIX_EPOCH,
tagging_timestamp: OffsetDateTime::UNIX_EPOCH,
legalhold_timestamp: OffsetDateTime::UNIX_EPOCH,
replication_validity_check: false, replication_validity_check: false,
} }
} }
@@ -1679,16 +1675,6 @@ impl PutObjectOptions {
); );
} }
for (suffix, timestamp) in [
(SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, self.internal.tagging_timestamp),
(SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, self.internal.retention_timestamp),
(SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, self.internal.legalhold_timestamp),
] {
if timestamp.unix_timestamp() != 0 {
insert_header(&mut header, suffix, timestamp.format(&Rfc3339).unwrap_or_default());
}
}
if self.internal.replication_request { if self.internal.replication_request {
insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true"); insert_header(&mut header, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
} }
@@ -2856,57 +2842,6 @@ mod tests {
); );
} }
#[test]
fn put_object_headers_carry_replication_timestamp_headers() {
// MinIO receivers resolve concurrent tag/retention/legal-hold edits by
// last-writer-wins on these headers (object-api-options.go parses them
// as RFC3339); a replica without them loses every conflict resolution.
let mut opts = PutObjectOptions::default();
opts.internal.replication_request = true;
let tagging = OffsetDateTime::from_unix_timestamp(1_700_000_001).expect("valid timestamp");
let retention = OffsetDateTime::from_unix_timestamp(1_700_000_002).expect("valid timestamp");
let legalhold = OffsetDateTime::from_unix_timestamp(1_700_000_003).expect("valid timestamp");
opts.internal.tagging_timestamp = tagging;
opts.internal.retention_timestamp = retention;
opts.internal.legalhold_timestamp = legalhold;
let header = opts.header();
for (suffix, expected) in [
("source-replication-tagging-timestamp", tagging),
("source-replication-retention-timestamp", retention),
("source-replication-legalhold-timestamp", legalhold),
] {
assert_eq!(
rustfs_utils::http::get_header(&header, suffix).as_deref(),
Some(expected.format(&Rfc3339).expect("RFC3339 timestamp").as_str()),
"replication put requests must carry the {suffix} header"
);
}
}
#[test]
fn put_object_headers_omit_unset_replication_timestamps() {
// UNIX_EPOCH means "never modified on the source"; sending it would
// make the receiver treat an unset category as a fresh modification.
let mut opts = PutObjectOptions::default();
opts.internal.replication_request = true;
opts.internal.tagging_timestamp = OffsetDateTime::UNIX_EPOCH;
opts.internal.retention_timestamp = OffsetDateTime::UNIX_EPOCH;
opts.internal.legalhold_timestamp = OffsetDateTime::UNIX_EPOCH;
let header = opts.header();
for suffix in [
"source-replication-tagging-timestamp",
"source-replication-retention-timestamp",
"source-replication-legalhold-timestamp",
] {
assert!(
rustfs_utils::http::get_header(&header, suffix).is_none(),
"unset {suffix} must not be sent to replication targets"
);
}
}
#[tokio::test] #[tokio::test]
async fn get_remote_target_client_internal_rejects_loopback_endpoint() { async fn get_remote_target_client_internal_rejects_loopback_endpoint() {
let sys = BucketTargetSys::default(); let sys = BucketTargetSys::default();
@@ -126,23 +126,11 @@ const EVENT_LIFECYCLE_EXPIRED_DETECTED: &str = "lifecycle_expired_detected";
const EVENT_LIFECYCLE_NOT_ENQUEUED: &str = "lifecycle_not_enqueued"; const EVENT_LIFECYCLE_NOT_ENQUEUED: &str = "lifecycle_not_enqueued";
const EVENT_LIFECYCLE_DELETE_DISPATCHED: &str = "lifecycle_delete_dispatched"; const EVENT_LIFECYCLE_DELETE_DISPATCHED: &str = "lifecycle_delete_dispatched";
const EVENT_LIFECYCLE_DELETE_COMPLETED: &str = "lifecycle_delete_completed"; const EVENT_LIFECYCLE_DELETE_COMPLETED: &str = "lifecycle_delete_completed";
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
const EVENT_LIFECYCLE_TIER_AUDIT: &str = "lifecycle_tier_audit"; const EVENT_LIFECYCLE_TIER_AUDIT: &str = "lifecycle_tier_audit";
const EVENT_LIFECYCLE_TIER_OPERATION_FAILED: &str = "lifecycle_tier_operation_failed"; const EVENT_LIFECYCLE_TIER_OPERATION_FAILED: &str = "lifecycle_tier_operation_failed";
const EVENT_LIFECYCLE_DELETE_FAILED: &str = "lifecycle_delete_failed"; const EVENT_LIFECYCLE_DELETE_FAILED: &str = "lifecycle_delete_failed";
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>; pub type TimeFn = Arc<dyn Fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub type TraceFn = pub type TraceFn =
Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>; Arc<dyn Fn(String, HashMap<String, String>) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync + 'static>;
pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>; pub type ExpiryOpType = Box<dyn ExpiryOp + Send + Sync + 'static>;
@@ -152,21 +140,9 @@ static TIER_FREE_VERSION_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new(); static MANUAL_TRANSITION_JOB_RECOVERY_STARTED: OnceLock<()> = OnceLock::new();
pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging"; pub const AMZ_OBJECT_TAGGING: &str = "X-Amz-Tagging";
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count"; pub const AMZ_TAG_COUNT: &str = "x-amz-tagging-count";
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive"; pub const AMZ_TAG_DIRECTIVE: &str = "X-Amz-Tagging-Directive";
pub const AMZ_ENCRYPTION_AES: &str = "AES256"; pub const AMZ_ENCRYPTION_AES: &str = "AES256";
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub const AMZ_ENCRYPTION_KMS: &str = "aws:kms"; pub const AMZ_ENCRYPTION_KMS: &str = "aws:kms";
pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier."; pub const ERR_INVALID_STORAGECLASS: &str = "invalid tier.";
@@ -304,10 +280,6 @@ impl LifecycleSys {
} }
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub fn trace(oi: &ObjectInfo) -> TraceFn { pub fn trace(oi: &ObjectInfo) -> TraceFn {
let bucket = oi.bucket.clone(); let bucket = oi.bucket.clone();
let name = oi.name.clone(); let name = oi.name.clone();
@@ -598,10 +570,6 @@ async fn delete_free_version_remote_object(
Ok(()) Ok(())
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
async fn delete_free_version_remote_object_then<T, F, Fut>( async fn delete_free_version_remote_object_then<T, F, Fut>(
oi: &ObjectInfo, oi: &ObjectInfo,
tier_config_mgr: &Arc<RwLock<TierConfigMgr>>, tier_config_mgr: &Arc<RwLock<TierConfigMgr>>,
@@ -2900,10 +2868,6 @@ fn stale_upload_default_due(initiated: OffsetDateTime, default_expiry: StdDurati
initiated + time::Duration::seconds(default_expiry.as_secs() as i64) initiated + time::Duration::seconds(default_expiry.as_secs() as i64)
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
async fn stale_upload_current_size(set: &Arc<SetDisks>, metadata: &HashMap<String, String>, upload_dir: &str) -> Option<usize> { async fn stale_upload_current_size(set: &Arc<SetDisks>, metadata: &HashMap<String, String>, upload_dir: &str) -> Option<usize> {
stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await stale_upload_current_size_with_opts(set, metadata, upload_dir, false).await
} }
@@ -3388,10 +3352,6 @@ pub async fn validate_transition_tier(lc: &BucketLifecycleConfiguration) -> Resu
Ok(()) Ok(())
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
fn mark_delete_opts_skip_decommissioned_on_remote_success(opts: &mut ObjectOptions, remote_delete_succeeded: bool) { fn mark_delete_opts_skip_decommissioned_on_remote_success(opts: &mut ObjectOptions, remote_delete_succeeded: bool) {
if remote_delete_succeeded { if remote_delete_succeeded {
opts.skip_decommissioned = true; opts.skip_decommissioned = true;
@@ -4379,10 +4339,6 @@ pub async fn expire_transitioned_object(
Ok(dobj) Ok(dobj)
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> { pub fn gen_transition_objname(bucket: &str) -> Result<String, Error> {
let us = Uuid::new_v4().to_string(); let us = Uuid::new_v4().to_string();
let mut hasher = Sha256::new(); let mut hasher = Sha256::new();
@@ -4417,10 +4373,6 @@ pub async fn transition_object(api: Arc<ECStore>, oi: &ObjectInfo, lae: LcAuditE
result result
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn { pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
let tier = _tier.to_string(); let tier = _tier.to_string();
Arc::new(move || { Arc::new(move || {
@@ -4439,10 +4391,6 @@ pub fn audit_tier_actions(_tier: &str, bytes: i64) -> TimeFn {
}) })
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub async fn get_transitioned_object_reader( pub async fn get_transitioned_object_reader(
bucket: &str, bucket: &str,
object: &str, object: &str,
@@ -5197,10 +5145,6 @@ async fn lifecycle_delete_config_snapshot(api: &ECStore, oi: &ObjectInfo) -> Res
ReplicationObjectBridge::delete_request_config(api, &oi.bucket).await ReplicationObjectBridge::delete_request_config(api, &oi.bucket).await
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool { pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
let mut success = false; let mut success = false;
match event.action { match event.action {
@@ -7478,10 +7422,6 @@ mod tests {
// process environment while `env::set_var`/`env::remove_var` is active. // process environment while `env::set_var`/`env::remove_var` is active.
// SAFETY: keep this note adjacent to the allowance for the repository guard. // SAFETY: keep this note adjacent to the allowance for the repository guard.
#[allow(unsafe_code)] #[allow(unsafe_code)]
#[allow(
dead_code,
reason = "transition-queue env fixture kept for tests that scope those vars; no test uses it today (backlog#1823)"
)]
async fn with_transition_queue_env_async<F, Fut>(capacity: Option<&str>, timeout_ms: Option<&str>, test_fn: F) async fn with_transition_queue_env_async<F, Fut>(capacity: Option<&str>, timeout_ms: Option<&str>, test_fn: F)
where where
F: FnOnce() -> Fut, F: FnOnce() -> Fut,
@@ -759,10 +759,6 @@ pub struct ManualTransitionWorkerResultRecord {
} }
impl ManualTransitionWorkerResultRecord { impl ManualTransitionWorkerResultRecord {
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self { pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self {
Self::new_with_reason(job_id, task_key, result, None) Self::new_with_reason(job_id, task_key, result, None)
} }
@@ -1261,10 +1257,6 @@ pub(crate) async fn save_manual_transition_task_if_absent(
} }
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub async fn load_manual_transition_task_record( pub async fn load_manual_transition_task_record(
api: Arc<ECStore>, api: Arc<ECStore>,
job_id: Uuid, job_id: Uuid,
@@ -1328,10 +1320,6 @@ async fn scan_manual_transition_task_journal(api: Arc<ECStore>, job_id: Uuid) ->
} }
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub async fn load_manual_transition_worker_result_stats( pub async fn load_manual_transition_worker_result_stats(
api: Arc<ECStore>, api: Arc<ECStore>,
job_id: Uuid, job_id: Uuid,
@@ -1467,10 +1455,6 @@ async fn scan_manual_transition_worker_result_journal(
} }
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub async fn reconcile_manual_transition_worker_results( pub async fn reconcile_manual_transition_worker_results(
api: Arc<ECStore>, api: Arc<ECStore>,
job_id: Uuid, job_id: Uuid,
@@ -15,35 +15,25 @@
use rustfs_common::metrics::IlmAction; use rustfs_common::metrics::IlmAction;
use crate::bucket::lifecycle::lifecycle::ObjectOpts; use crate::bucket::lifecycle::lifecycle::ObjectOpts;
use crate::bucket::replication::ReplicationLifecycleBridge;
pub(crate) use crate::bucket::replication::ReplicationStatusType; pub(crate) use crate::bucket::replication::ReplicationStatusType;
#[cfg(test)] #[cfg(test)]
pub(crate) use crate::bucket::replication::VersionPurgeStatusType; pub(crate) use crate::bucket::replication::VersionPurgeStatusType;
pub(crate) use crate::bucket::replication::{ pub(crate) use crate::bucket::replication::{
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta, DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta,
}; };
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
use crate::storage_api_contracts::object::DeletedObject; use crate::storage_api_contracts::object::DeletedObject;
#[allow( pub(crate) type LifecycleReplicationConfig = ReplicationLifecycleConfig;
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) fn has_pending_version_purge(obj: &ObjectOpts) -> bool { pub(crate) fn has_pending_version_purge(obj: &ObjectOpts) -> bool {
obj.version_purge_status.is_pending() obj.version_purge_status.is_pending()
} }
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) fn has_pending_object_replication(obj: &ObjectOpts) -> bool { pub(crate) fn has_pending_object_replication(obj: &ObjectOpts) -> bool {
replication_status_blocks_lifecycle(&obj.replication_status) replication_status_blocks_lifecycle(&obj.replication_status)
} }
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) fn has_pending_lifecycle_replication(obj: &ObjectOpts) -> bool { pub(crate) fn has_pending_lifecycle_replication(obj: &ObjectOpts) -> bool {
has_pending_object_replication(obj) || has_pending_version_purge(obj) has_pending_object_replication(obj) || has_pending_version_purge(obj)
} }
@@ -14,10 +14,6 @@
use std::collections::HashMap; use std::collections::HashMap;
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> { pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
crate::bucket::tagging::decode_tags_to_map(tags) crate::bucket::tagging::decode_tags_to_map(tags)
} }
@@ -331,10 +331,6 @@ where
persist_tier_delete_journal_entry(api, &committed).await persist_tier_delete_journal_entry(api, &committed).await
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()> pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
where where
S: ObjectOperations< S: ObjectOperations<
@@ -148,10 +148,6 @@ struct RecoveryCursor {
object: String, object: String,
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub async fn recover_tier_free_versions( pub async fn recover_tier_free_versions(
api: Arc<ECStore>, api: Arc<ECStore>,
limit: usize, limit: usize,
@@ -385,10 +385,6 @@ impl ExpiryOp for Jentry {
} }
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> { pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
let result = delete_object_from_remote_tier_raw(obj_name, rv_id, tier_name).await; let result = delete_object_from_remote_tier_raw(obj_name, rv_id, tier_name).await;
if let Err(err) = &result if let Err(err) = &result
@@ -399,10 +395,6 @@ pub async fn delete_object_from_remote_tier(obj_name: &str, rv_id: &str, tier_na
result result
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> { async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_name: &str) -> Result<(), std::io::Error> {
#[cfg(test)] #[cfg(test)]
if let Some(result) = run_remote_tier_delete_test_hook(obj_name, rv_id, tier_name) { if let Some(result) = run_remote_tier_delete_test_hook(obj_name, rv_id, tier_name) {
@@ -413,10 +405,6 @@ async fn delete_object_from_remote_tier_raw(obj_name: &str, rv_id: &str, tier_na
delete_object_from_remote_tier_raw_with_manager(obj_name, rv_id, tier_name, &tier_config_mgr).await delete_object_from_remote_tier_raw_with_manager(obj_name, rv_id, tier_name, &tier_config_mgr).await
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
async fn delete_object_from_remote_tier_raw_with_manager( async fn delete_object_from_remote_tier_raw_with_manager(
obj_name: &str, obj_name: &str,
rv_id: &str, rv_id: &str,
@@ -497,10 +485,6 @@ pub enum RemoteTierDeleteOutcome {
AlreadyRemoved, AlreadyRemoved,
} }
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
pub async fn delete_object_from_remote_tier_idempotent( pub async fn delete_object_from_remote_tier_idempotent(
obj_name: &str, obj_name: &str,
rv_id: &str, rv_id: &str,
@@ -50,16 +50,8 @@ pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]
pub enum TransitionTransactionError { pub enum TransitionTransactionError {
#[error("transition transaction already exists")] #[error("transition transaction already exists")]
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
AlreadyExists, AlreadyExists,
#[error("transition transaction is not found")] #[error("transition transaction is not found")]
#[allow(
dead_code,
reason = "MinIO-parity tier/lifecycle entry point that this port never wired (backlog#1823)"
)]
NotFound, NotFound,
#[error("transition transaction is corrupt: {0}")] #[error("transition transaction is corrupt: {0}")]
Corrupt(&'static str), Corrupt(&'static str),
-31
View File
@@ -60,14 +60,12 @@ struct ConfigWriteLockProbeState {
static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new(); static CONFIG_WRITE_LOCK_PROBES: std::sync::OnceLock<StdMutex<Vec<Arc<ConfigWriteLockProbeState>>>> = std::sync::OnceLock::new();
#[cfg(any(test, feature = "test-util"))] #[cfg(any(test, feature = "test-util"))]
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
pub struct ConfigWriteLockProbe { pub struct ConfigWriteLockProbe {
state: Arc<ConfigWriteLockProbeState>, state: Arc<ConfigWriteLockProbeState>,
} }
#[cfg(any(test, feature = "test-util"))] #[cfg(any(test, feature = "test-util"))]
impl ConfigWriteLockProbe { impl ConfigWriteLockProbe {
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
pub fn install(bucket: &str) -> Self { pub fn install(bucket: &str) -> Self {
let state = Arc::new(ConfigWriteLockProbeState { let state = Arc::new(ConfigWriteLockProbeState {
bucket: bucket.to_string(), bucket: bucket.to_string(),
@@ -86,7 +84,6 @@ impl ConfigWriteLockProbe {
Self { state } Self { state }
} }
#[allow(dead_code, reason = "installed by tests behind `--features test-util` (backlog#1823)")]
pub async fn wait_until_attempted(&self) { pub async fn wait_until_attempted(&self) {
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
.await .await
@@ -659,16 +656,6 @@ pub async fn update_under_transaction_lock(
update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await update_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file, data).await
} }
/// Clear one config file while the caller holds this bucket's transaction lock.
pub async fn delete_under_transaction_lock(
guard: &BucketMetadataMutationGuard,
bucket: &str,
config_file: &str,
) -> Result<OffsetDateTime> {
guard.ensure_valid(bucket)?;
delete_under_config_write_guard(get_bucket_metadata_sys()?, guard, config_file).await
}
pub async fn update_quota_if_incarnation( pub async fn update_quota_if_incarnation(
bucket: &str, bucket: &str,
data: Vec<u8>, data: Vec<u8>,
@@ -808,14 +795,6 @@ pub async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result<Bu
acquire_config_write_guard(get_bucket_metadata_sys()?, bucket).await acquire_config_write_guard(get_bucket_metadata_sys()?, bucket).await
} }
/// Acquire the bucket transaction lock only if its incarnation still matches.
pub async fn acquire_bucket_metadata_transaction_lock_for_incarnation(
bucket: &str,
expected_incarnation_id: Uuid,
) -> Result<BucketMetadataMutationGuard> {
acquire_config_write_guard_for_incarnation(get_bucket_metadata_sys()?, bucket, Some(expected_incarnation_id)).await
}
pub(crate) async fn acquire_bucket_metadata_transaction_lock_in( pub(crate) async fn acquire_bucket_metadata_transaction_lock_in(
ctx: &crate::runtime::instance::InstanceContext, ctx: &crate::runtime::instance::InstanceContext,
bucket: &str, bucket: &str,
@@ -893,10 +872,6 @@ pub async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTi
bucket_meta_sys.get_bucket_policy_raw(bucket).await bucket_meta_sys.get_bucket_policy_raw(bucket).await
} }
#[allow(
dead_code,
reason = "free-function facade over the live BucketMetadataSys::get_bucket_acl_config; no caller in this port (backlog#1823)"
)]
pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> { pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?; let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await; let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -1111,10 +1086,6 @@ pub async fn get_config_from_disk(bucket: &str) -> Result<BucketMetadata> {
bucket_meta_sys.get_config_from_disk(bucket).await bucket_meta_sys.get_config_from_disk(bucket).await
} }
#[allow(
dead_code,
reason = "ambient-facade variant of the live created_at_in; no caller in this port (backlog#1823)"
)]
pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> { pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?; let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await; let bucket_meta_sys = bucket_meta_sys_lock.read().await;
@@ -1628,7 +1599,6 @@ impl BucketMetadataSys {
/// [`Self::update`], with the payload computed from the loaded metadata /// [`Self::update`], with the payload computed from the loaded metadata
/// instead of supplied up front. Loads through this system's own store so /// instead of supplied up front. Loads through this system's own store so
/// the read and the persisted write target the same instance. /// the read and the persisted write target the same instance.
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
async fn update_config_with<F>(&self, bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime> async fn update_config_with<F>(&self, bucket: &str, config_file: &str, mutate: F) -> Result<OffsetDateTime>
where where
F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send, F: FnOnce(&BucketMetadata) -> Result<Vec<u8>> + Send,
@@ -1733,7 +1703,6 @@ impl BucketMetadataSys {
/// A miss is never published as an authoritative default, and a snapshot /// A miss is never published as an authoritative default, and a snapshot
/// read before delete plus same-name recreation cannot replace the new /// read before delete plus same-name recreation cannot replace the new
/// generation. /// generation.
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(crate) async fn reload_from_store(&self, bucket: &str) -> Result<()> { pub(crate) async fn reload_from_store(&self, bucket: &str) -> Result<()> {
if is_meta_bucketname(bucket) { if is_meta_bucketname(bucket) {
return Err(Error::other("errInvalidArgument")); return Err(Error::other("errInvalidArgument"));
+1
View File
@@ -13,6 +13,7 @@
// limitations under the License. // limitations under the License.
// #730: bucket subsystems still contain staged ECStore migration code. // #730: bucket subsystems still contain staged ECStore migration code.
#![allow(dead_code)]
pub mod bandwidth; pub mod bandwidth;
pub mod bucket_target_sys; pub mod bucket_target_sys;
@@ -136,7 +136,6 @@ pub fn add_years(dt: OffsetDateTime, years: i32) -> OffsetDateTime {
/// Check if an object has legal hold enabled. /// Check if an object has legal hold enabled.
/// Returns true if legal hold is ON. /// Returns true if legal hold is ON.
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool { fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> bool {
let lhold = objectlock::get_object_legalhold_meta(user_defined); let lhold = objectlock::get_object_legalhold_meta(user_defined);
matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON) matches!(lhold.status, Some(ref st) if st.as_str() == ObjectLockLegalHoldStatus::ON)
@@ -152,7 +151,6 @@ fn has_legal_hold(user_defined: &std::collections::HashMap<String, String>) -> b
/// # Returns /// # Returns
/// * `true` if the object is locked (cannot be deleted/modified) /// * `true` if the object is locked (cannot be deleted/modified)
/// * `false` if the object is not locked /// * `false` if the object is not locked
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub fn is_object_locked_by_metadata(user_defined: &std::collections::HashMap<String, String>, is_delete_marker: bool) -> bool { pub fn is_object_locked_by_metadata(user_defined: &std::collections::HashMap<String, String>, is_delete_marker: bool) -> bool {
// Delete markers are never locked // Delete markers are never locked
if is_delete_marker { if is_delete_marker {
-2
View File
@@ -193,7 +193,6 @@ pub enum QuotaError {
} }
#[derive(Debug, Serialize)] #[derive(Debug, Serialize)]
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub struct QuotaErrorResponse { pub struct QuotaErrorResponse {
#[serde(rename = "Code")] #[serde(rename = "Code")]
pub code: String, pub code: String,
@@ -209,7 +208,6 @@ pub struct QuotaErrorResponse {
} }
impl QuotaErrorResponse { impl QuotaErrorResponse {
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub fn new(quota_error: &QuotaError, request_id: &str, host_id: &str) -> Self { pub fn new(quota_error: &QuotaError, request_id: &str, host_id: &str) -> Self {
match quota_error { match quota_error {
QuotaError::QuotaExceeded { .. } => Self { QuotaError::QuotaExceeded { .. } => Self {
@@ -899,7 +899,6 @@ async fn save_ledger_locked(
} }
#[cfg(any(test, feature = "test-util"))] #[cfg(any(test, feature = "test-util"))]
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub fn fail_next_quota_ledger_save_for_test() { pub fn fail_next_quota_ledger_save_for_test() {
FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst); FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst);
} }
+2 -2
View File
@@ -60,7 +60,7 @@ pub use replication_filemeta_boundary::{
pub(crate) use replication_filemeta_boundary::{ pub(crate) use replication_filemeta_boundary::{
replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta, replication_state_from_filemeta, replication_status_from_filemeta, version_purge_status_from_filemeta,
}; };
pub(crate) use replication_lifecycle_bridge::ReplicationLifecycleBridge; pub(crate) use replication_lifecycle_bridge::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
pub(crate) use replication_migration_bridge::ReplicationMigrationBridge; pub(crate) use replication_migration_bridge::ReplicationMigrationBridge;
pub use replication_object_bridge::ReplicationObjectBridge; pub use replication_object_bridge::ReplicationObjectBridge;
pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig}; pub use replication_object_config::{DeleteReplicationConfigSnapshot, ReplicationConfig};
@@ -81,6 +81,6 @@ pub use replication_queue_boundary::{
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus}; pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
pub use replication_scanner_bridge::ReplicationScannerBridge; pub use replication_scanner_bridge::ReplicationScannerBridge;
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog}; pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
pub use replication_stats_boundary::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats}; pub use replication_stats_boundary::{BucketReplicationStats, BucketStats};
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage}; pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge; pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
@@ -37,10 +37,6 @@ impl ReplicationConfigStore {
com::read_config_limited(api, file, max_bytes).await com::read_config_limited(api, file, max_bytes).await
} }
#[allow(
dead_code,
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
)]
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>> pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
where where
S: ReplicationObjectIO, S: ReplicationObjectIO,
@@ -24,27 +24,15 @@ use super::replication_storage_boundary::{
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication, DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
}; };
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) type ReplicationLifecycleConfig = ReplicationConfig; pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
pub(crate) struct ReplicationLifecycleBridge; pub(crate) struct ReplicationLifecycleBridge;
impl ReplicationLifecycleBridge { impl ReplicationLifecycleBridge {
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig { pub(crate) fn new_config(config: ReplicationConfiguration) -> ReplicationLifecycleConfig {
ReplicationConfig::new(Some(config), None) ReplicationConfig::new(Some(config), None)
} }
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) fn has_pending_version_purge( pub(crate) fn has_pending_version_purge(
config: &ReplicationLifecycleConfig, config: &ReplicationLifecycleConfig,
object_name: &str, object_name: &str,
@@ -57,10 +45,6 @@ impl ReplicationLifecycleBridge {
.is_some_and(|config| config.has_active_rules(object_name, true)) .is_some_and(|config| config.has_active_rules(object_name, true))
} }
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) async fn check_delete_replication( pub(crate) async fn check_delete_replication(
bucket: &str, bucket: &str,
object: &ObjectToDelete, object: &ObjectToDelete,
@@ -70,10 +54,6 @@ impl ReplicationLifecycleBridge {
check_replicate_delete(bucket, object, source, opts, None).await check_replicate_delete(bucket, object, source, opts, None).await
} }
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) fn version_delete_replication_state(decision: &ReplicateDecision) -> ReplicationState { pub(crate) fn version_delete_replication_state(decision: &ReplicateDecision) -> ReplicationState {
let pending_status = decision.pending_status(); let pending_status = decision.pending_status();
ReplicationState { ReplicationState {
@@ -19,33 +19,17 @@ use time::OffsetDateTime;
use super::replication_error_boundary::Result; use super::replication_error_boundary::Result;
use crate::bucket::msgp_decode; use crate::bucket::msgp_decode;
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) struct ReplicationMsgpCodec; pub(crate) struct ReplicationMsgpCodec;
impl ReplicationMsgpCodec { impl ReplicationMsgpCodec {
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) fn read_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> { pub(crate) fn read_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
msgp_decode::read_msgp_ext8_time(rd) msgp_decode::read_msgp_ext8_time(rd)
} }
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) fn skip_value<R: Read>(rd: &mut R) -> Result<()> { pub(crate) fn skip_value<R: Read>(rd: &mut R) -> Result<()> {
msgp_decode::skip_msgp_value(rd) msgp_decode::skip_msgp_value(rd)
} }
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) fn write_time<W: Write>(wr: &mut W, time: OffsetDateTime) -> Result<()> { pub(crate) fn write_time<W: Write>(wr: &mut W, time: OffsetDateTime) -> Result<()> {
msgp_decode::write_msgp_time(wr, time) msgp_decode::write_msgp_time(wr, time)
} }
@@ -77,10 +77,6 @@ impl ReplicationObjectBridge {
load_delete_request_config_in(ctx, bucket).await load_delete_request_config_in(ctx, bucket).await
} }
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) async fn delete_config_snapshot_in( pub(crate) async fn delete_config_snapshot_in(
ctx: &ReplicationInstanceContext, ctx: &ReplicationInstanceContext,
bucket: &str, bucket: &str,
@@ -231,10 +231,6 @@ pub(crate) async fn load_delete_replication_config(
delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?) delete_snapshot_from_metadata(ReplicationMetadataStore::delete_metadata(bucket).await?)
} }
#[allow(
dead_code,
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
)]
pub(crate) async fn load_delete_replication_config_in( pub(crate) async fn load_delete_replication_config_in(
ctx: &ReplicationInstanceContext, ctx: &ReplicationInstanceContext,
bucket: &str, bucket: &str,
@@ -217,10 +217,6 @@ impl DurableMrfBacklogTracker {
} }
} }
#[allow(
dead_code,
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
)]
fn durable_mrf_backlog_tracker_from_entries(entries: &[MrfReplicateEntry]) -> DurableMrfBacklogTracker { fn durable_mrf_backlog_tracker_from_entries(entries: &[MrfReplicateEntry]) -> DurableMrfBacklogTracker {
let mut tracker = DurableMrfBacklogTracker { let mut tracker = DurableMrfBacklogTracker {
available: true, available: true,
@@ -716,10 +712,6 @@ pub struct ReplicationPool<S: ReplicationStorage> {
// MRF worker lifecycle // MRF worker lifecycle
mrf_worker_cancellations: Mutex<Vec<CancellationToken>>, mrf_worker_cancellations: Mutex<Vec<CancellationToken>>,
#[allow(
dead_code,
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
)]
mrf_stop_tx: Sender<()>, mrf_stop_tx: Sender<()>,
// Worker size tracking // Worker size tracking
@@ -948,10 +940,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
} }
/// Resizes worker priority and counts /// Resizes worker priority and counts
#[allow(
dead_code,
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
)]
pub async fn resize_worker_priority( pub async fn resize_worker_priority(
&self, &self,
pri: ReplicationPriority, pri: ReplicationPriority,
@@ -1192,10 +1180,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
} }
/// Queues an MRF save operation /// Queues an MRF save operation
#[allow(
dead_code,
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
)]
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) { async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await; let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
} }
@@ -1667,10 +1651,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
} }
/// Worker function for handling regular replication operations /// Worker function for handling regular replication operations
#[allow(
dead_code,
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
)]
async fn add_worker( async fn add_worker(
&self, &self,
mut rx: Receiver<ReplicationOperation>, mut rx: Receiver<ReplicationOperation>,
@@ -1684,10 +1664,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
} }
/// Worker function for handling large object replication operations /// Worker function for handling large object replication operations
#[allow(
dead_code,
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
)]
async fn add_large_worker( async fn add_large_worker(
&self, &self,
mut rx: Receiver<ReplicationOperation>, mut rx: Receiver<ReplicationOperation>,
@@ -1702,10 +1678,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
} }
/// Worker function for handling MRF (Most Recent Failures) operations /// Worker function for handling MRF (Most Recent Failures) operations
#[allow(
dead_code,
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
)]
async fn add_mrf_worker( async fn add_mrf_worker(
&self, &self,
mut rx: Receiver<ReplicationOperation>, mut rx: Receiver<ReplicationOperation>,
@@ -1719,10 +1691,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
} }
/// Delete resync metadata from replication resync state in memory /// Delete resync metadata from replication resync state in memory
#[allow(
dead_code,
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
)]
pub async fn delete_resync_metadata(&self, bucket: &str) { pub async fn delete_resync_metadata(&self, bucket: &str) {
let mut status_map = self.resyncer.status_map.write().await; let mut status_map = self.resyncer.status_map.write().await;
status_map.remove(bucket); status_map.remove(bucket);
@@ -21,31 +21,11 @@ pub(crate) use rustfs_replication::{
should_count_head_proxy_failure, should_count_head_proxy_failure,
}; };
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) const RESYNC_META_FORMAT: u16 = rustfs_replication::resync::RESYNC_META_FORMAT; pub(crate) const RESYNC_META_FORMAT: u16 = rustfs_replication::resync::RESYNC_META_FORMAT;
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) const RESYNC_META_VERSION: u16 = rustfs_replication::resync::RESYNC_META_VERSION; pub(crate) const RESYNC_META_VERSION: u16 = rustfs_replication::resync::RESYNC_META_VERSION;
pub(crate) const RESYNC_FILE_MAX_BYTES: usize = rustfs_replication::RESYNC_FILE_MAX_BYTES; pub(crate) const RESYNC_FILE_MAX_BYTES: usize = rustfs_replication::RESYNC_FILE_MAX_BYTES;
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) const WIRE_ZERO_TIME_UNIX: i64 = rustfs_replication::resync::WIRE_ZERO_TIME_UNIX; pub(crate) const WIRE_ZERO_TIME_UNIX: i64 = rustfs_replication::resync::WIRE_ZERO_TIME_UNIX;
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) const MRF_META_FORMAT: u16 = rustfs_replication::mrf::MRF_META_FORMAT; pub(crate) const MRF_META_FORMAT: u16 = rustfs_replication::mrf::MRF_META_FORMAT;
#[allow(
dead_code,
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
)]
pub(crate) const MRF_META_VERSION: u16 = rustfs_replication::mrf::MRF_META_VERSION; pub(crate) const MRF_META_VERSION: u16 = rustfs_replication::mrf::MRF_META_VERSION;
fn map_replication_error(err: rustfs_replication::Error) -> Error { fn map_replication_error(err: rustfs_replication::Error) -> Error {
@@ -122,10 +122,6 @@ const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
"tcp connect error", "tcp connect error",
]; ];
#[allow(
dead_code,
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
)]
const RESYNC_TIME_INTERVAL: TokioDuration = TokioDuration::from_secs(60); const RESYNC_TIME_INTERVAL: TokioDuration = TokioDuration::from_secs(60);
static WARNED_MONITOR_UNINIT: std::sync::Once = std::sync::Once::new(); static WARNED_MONITOR_UNINIT: std::sync::Once = std::sync::Once::new();
@@ -332,10 +328,6 @@ fn bounded_resync_max_jobs(value: usize) -> usize {
#[derive(Debug)] #[derive(Debug)]
pub struct ReplicationResyncer { pub struct ReplicationResyncer {
pub status_map: Arc<RwLock<HashMap<String, BucketReplicationResyncStatus>>>, pub status_map: Arc<RwLock<HashMap<String, BucketReplicationResyncStatus>>>,
#[allow(
dead_code,
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
)]
pub worker_size: usize, pub worker_size: usize,
pub(crate) cancel_tokens: Arc<RwLock<HashMap<ResyncCancelKey, CancellationToken>>>, pub(crate) cancel_tokens: Arc<RwLock<HashMap<ResyncCancelKey, CancellationToken>>>,
resync_admission: Arc<Semaphore>, resync_admission: Arc<Semaphore>,
@@ -552,10 +544,6 @@ impl ReplicationResyncer {
.is_some_and(|status| status.failed_count > 0) .is_some_and(|status| status.failed_count > 0)
} }
#[allow(
dead_code,
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
)]
pub async fn persist_to_disk<S>(&self, cancel_token: CancellationToken, api: Arc<S>) pub async fn persist_to_disk<S>(&self, cancel_token: CancellationToken, api: Arc<S>)
where where
S: ReplicationObjectIO, S: ReplicationObjectIO,
@@ -340,10 +340,6 @@ impl ReplicationStats {
} }
/// Site replication update replica statistics /// Site replication update replica statistics
#[allow(
dead_code,
reason = "MinIO-parity replication surface with no caller in this port (backlog#1823)"
)]
fn sr_update_replica_stat(&self, size: i64) { fn sr_update_replica_stat(&self, size: i64) {
self.sr_stats.replica_size.fetch_add(size, Ordering::Relaxed); self.sr_stats.replica_size.fetch_add(size, Ordering::Relaxed);
self.sr_stats.replica_count.fetch_add(1, Ordering::Relaxed); self.sr_stats.replica_count.fetch_add(1, Ordering::Relaxed);
@@ -708,12 +704,6 @@ impl ReplicationStats {
} else { } else {
BucketReplicationStats::new() BucketReplicationStats::new()
}; };
// Stamp the serializable failure windows from the live samples: the
// samples themselves do not cross the peer-RPC wire, so this snapshot
// is what cluster aggregation and the metrics endpoints see.
for stat in replication_stats.stats.values_mut() {
stat.fail_stats.refresh_windows();
}
let uptime = if cache.contains_key(bucket) { let uptime = if cache.contains_key(bucket) {
SystemTime::now() SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH) .duration_since(SystemTime::UNIX_EPOCH)
@@ -15,9 +15,7 @@
#[cfg(test)] #[cfg(test)]
pub(crate) use rustfs_replication::FailStats; pub(crate) use rustfs_replication::FailStats;
pub(crate) use rustfs_replication::{ pub(crate) use rustfs_replication::{
ActiveWorkerStat, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope, SRMetricsSummary, ActiveWorkerStat, BucketReplicationStat, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope,
SRMetricsSummary, XferStats,
}; };
// Public so the admin wire DTOs (rustfs/src/admin/replication_metrics_wire.rs) pub use rustfs_replication::{BucketReplicationStats, BucketStats};
// can project the internal stats onto the minio-go response shapes through
// the storage_api facade chain.
pub use rustfs_replication::{BucketReplicationStat, BucketReplicationStats, BucketStats, InQueueMetric, XferStats};
@@ -27,10 +27,8 @@ use rustfs_utils::http::{
AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID,
AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE,
HeaderExt as _, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, HeaderExt as _, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_TAGGING_TIMESTAMP, get_str, insert_header_map,
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP, SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key, ssec_replication_transport_header,
get_str, insert_header_map, is_internal_key, is_object_encryption_marker, is_replication_stripped_encryption_key,
ssec_replication_transport_header,
}; };
use time::OffsetDateTime; use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339; use time::format_description::well_known::Rfc3339;
@@ -121,27 +119,6 @@ fn classify_replication_source_encryption(metadata: &HashMap<String, String>) ->
} }
} }
fn is_legacy_source_replication_timestamp_key(key: &str) -> bool {
fn has_prefix_and_suffix(key: &str, prefix: &str, suffix: &str) -> bool {
let key = key.as_bytes();
key.len() == prefix.len() + suffix.len()
&& key[..prefix.len()].eq_ignore_ascii_case(prefix.as_bytes())
&& key[prefix.len()..].eq_ignore_ascii_case(suffix.as_bytes())
}
[
SUFFIX_SOURCE_REPLICATION_TAGGING_TIMESTAMP,
SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP,
SUFFIX_SOURCE_REPLICATION_LEGALHOLD_TIMESTAMP,
]
.iter()
.any(|suffix| {
["x-rustfs-", "x-minio-"]
.iter()
.any(|prefix| has_prefix_and_suffix(key, prefix, suffix))
})
}
pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool { pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
rustfs_replication::is_ssec_encrypted(user_defined) rustfs_replication::is_ssec_encrypted(user_defined)
} }
@@ -199,11 +176,6 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
continue; continue;
} }
if is_legacy_source_replication_timestamp_key(key) {
meta.insert(format!("x-amz-meta-{key}"), value.to_string());
continue;
}
if is_internal_key(key) || is_standard_header(key) { if is_internal_key(key) || is_standard_header(key) {
continue; continue;
} }
@@ -287,23 +259,15 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
if !tags.is_empty() { if !tags.is_empty() {
put_options.user_tags = tags; put_options.user_tags = tags;
} put_options.internal.tagging_timestamp =
} if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP) {
// Load the stored tagging timestamp independently of whether any tags
// remain: DeleteObjectTagging leaves the object tagless but stamps this
// key, and the deletion's LWW timestamp must still reach the replica.
// With no stored key, fall back to mod_time only while tags exist
// (MinIO parity); a tagless object without the key was never tagged and
// keeps the epoch default (no header).
put_options.internal.tagging_timestamp = if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_TAGGING_TIMESTAMP)
{
OffsetDateTime::parse(&timestamp, &Rfc3339) OffsetDateTime::parse(&timestamp, &Rfc3339)
.map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))? .map_err(|err| Error::other(format!("Failed to parse tagging timestamp: {err}")))?
} else if !put_options.user_tags.is_empty() {
object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
} else { } else {
OffsetDateTime::UNIX_EPOCH object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
}; };
}
}
let metadata = &*object_info.user_defined; let metadata = &*object_info.user_defined;
@@ -319,15 +283,13 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
put_options.cache_control = cache_control.to_string(); put_options.cache_control = cache_control.to_string();
} }
if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE).filter(|mode| !mode.is_empty()) { if let Some(mode) = metadata.lookup(AMZ_OBJECT_LOCK_MODE) {
put_options.mode = Some(ObjectLockRetentionMode::from(mode.to_uppercase().as_str())); put_options.mode = Some(ObjectLockRetentionMode::from(mode.to_uppercase().as_str()));
} }
if let Some(retain_until_date) = metadata.lookup(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE) { if let Some(retain_until_date) = metadata.lookup(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE) {
if !retain_until_date.is_empty() {
put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339) put_options.retain_until_date = OffsetDateTime::parse(retain_until_date, &Rfc3339)
.map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?; .map_err(|err| Error::other(format!("Failed to parse retain until date: {err}")))?;
}
put_options.internal.retention_timestamp = put_options.internal.retention_timestamp =
if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) { if let Some(timestamp) = get_str(&object_info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP) {
OffsetDateTime::parse(&timestamp, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH) OffsetDateTime::parse(&timestamp, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH)
@@ -732,110 +694,6 @@ mod tests {
assert!(options.internal.replication_request); assert!(options.internal.replication_request);
} }
/// DeleteObjectTagging leaves the object tagless but stamps the
/// tagging-timestamp internal key; the deletion's LWW timestamp must
/// still be loaded (and therefore sent) so the replica can order the
/// deletion against concurrent tag edits.
#[test]
fn replication_put_options_carry_tagging_timestamp_after_tag_deletion() {
let mut metadata = std::collections::HashMap::new();
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_TAGGING_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
let object_info = ObjectInfo {
user_defined: Arc::new(metadata),
user_tags: Arc::new(String::new()),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
version_id: Some(Uuid::nil()),
..Default::default()
};
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
assert!(options.user_tags.is_empty());
assert_eq!(
options.internal.tagging_timestamp,
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp"),
"the stored tagging timestamp must load independently of remaining tags"
);
// A tagless object without the stored key was never tagged: the epoch
// default keeps the header unsent.
let untagged = ObjectInfo {
user_tags: Arc::new(String::new()),
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp")),
version_id: Some(Uuid::nil()),
..Default::default()
};
let (options, _) = replication_put_object_options("", &untagged).expect("build put options");
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
}
#[test]
fn replication_put_options_do_not_promote_legacy_user_timestamp_metadata() {
let legacy_keys = [
"x-rustfs-source-replication-tagging-timestamp",
"x-rustfs-source-replication-retention-timestamp",
"x-rustfs-source-replication-legalhold-timestamp",
"x-minio-source-replication-tagging-timestamp",
"x-minio-source-replication-retention-timestamp",
"x-minio-source-replication-legalhold-timestamp",
];
let object_info = ObjectInfo {
user_defined: Arc::new(
legacy_keys
.iter()
.map(|key| (key.to_string(), "2099-01-02T03:04:05Z".to_string()))
.collect(),
),
..Default::default()
};
let (options, _) = replication_put_object_options("", &object_info).expect("build put options");
for legacy_key in legacy_keys {
assert!(!options.user_metadata.contains_key(legacy_key));
assert_eq!(
options
.user_metadata
.get(&format!("x-amz-meta-{legacy_key}"))
.map(String::as_str),
Some("2099-01-02T03:04:05Z")
);
}
assert_eq!(options.internal.tagging_timestamp, OffsetDateTime::UNIX_EPOCH);
assert_eq!(options.internal.retention_timestamp, OffsetDateTime::UNIX_EPOCH);
assert_eq!(options.internal.legalhold_timestamp, OffsetDateTime::UNIX_EPOCH);
}
#[test]
fn replication_put_options_carry_retention_timestamp_after_clear() {
let mut metadata = HashMap::from([
(AMZ_OBJECT_LOCK_MODE.to_string(), String::new()),
(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.to_string(), String::new()),
]);
rustfs_utils::http::insert_str(&mut metadata, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, "2026-01-02T03:04:05Z".to_string());
let object_info = ObjectInfo {
user_defined: Arc::new(metadata),
..Default::default()
};
let (options, _) = replication_put_object_options("", &object_info).expect("retention clear must replicate");
assert!(options.mode.is_none());
assert_eq!(options.retain_until_date, OffsetDateTime::UNIX_EPOCH);
assert_eq!(
options.internal.retention_timestamp,
OffsetDateTime::parse("2026-01-02T03:04:05Z", &Rfc3339).expect("valid timestamp")
);
let headers = options.header();
assert!(!headers.contains_key(AMZ_OBJECT_LOCK_MODE));
assert!(!headers.contains_key(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE));
assert_eq!(
rustfs_utils::http::get_header(&headers, SUFFIX_SOURCE_REPLICATION_RETENTION_TIMESTAMP).as_deref(),
Some("2026-01-02T03:04:05Z")
);
}
#[test] #[test]
fn replication_put_options_strip_encryption_metadata_from_plaintext_objects() { fn replication_put_options_strip_encryption_metadata_from_plaintext_objects() {
use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, SSEC_ORIGINAL_SIZE_HEADER}; use rustfs_utils::http::object_encryption_keys::{INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER, SSEC_ORIGINAL_SIZE_HEADER};
+4 -52
View File
@@ -40,14 +40,7 @@ impl ARN {
impl Display for ARN { impl Display for ARN {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
// The `minio` partition is deliberate: madmin-go's ParseARN write!(f, "arn:rustfs:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
// hard-rejects any other partition, so native mc/madmin tooling can
// only decode remote-target ARNs minted in this form (backlog#1675
// P1-7). Legacy `arn:rustfs:` ARNs persisted by older releases stay
// readable via the FromStr whitelist below; runtime matching between
// targets and replication rules is by full-string equality, so mixed
// partitions coexist safely.
write!(f, "arn:minio:{}:{}:{}:{}", self.arn_type, self.region, self.id, self.bucket)
} }
} }
@@ -55,12 +48,7 @@ impl FromStr for ARN {
type Err = std::io::Error; type Err = std::io::Error;
fn from_str(s: &str) -> Result<Self, Self::Err> { fn from_str(s: &str) -> Result<Self, Self::Err> {
// Partition whitelist, not just an `arn:` check: `BucketTargetType:: if !s.starts_with("arn:rustfs:") {
// from_str(...).unwrap_or_default()` below never fails, so this is
// the only structural gate rejecting foreign ARNs. `arn:rustfs:` is
// the legacy partition and must stay accepted forever (persisted
// bucket-targets.json / replication configs from older releases).
if !s.starts_with("arn:minio:") && !s.starts_with("arn:rustfs:") {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format")); return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid ARN format"));
} }
@@ -113,50 +101,14 @@ mod tests {
} }
/// RustFS commonly generates ARNs with an empty region: /// RustFS commonly generates ARNs with an empty region:
/// `arn:minio:replication::<deployment_id>:<bucket>`. /// `arn:rustfs:replication::<deployment_id>:<bucket>`.
#[test] #[test]
fn from_str_handles_empty_region_segment() { fn from_str_handles_empty_region_segment() {
let parsed = ARN::from_str("arn:minio:replication::depl-123:bucket-a").expect("valid ARN must parse"); let parsed = ARN::from_str("arn:rustfs:replication::depl-123:bucket-a").expect("valid ARN must parse");
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService); assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
assert_eq!(parsed.region, "", "region segment is empty in this form"); assert_eq!(parsed.region, "", "region segment is empty in this form");
assert_eq!(parsed.id, "depl-123"); assert_eq!(parsed.id, "depl-123");
assert_eq!(parsed.bucket, "bucket-a"); assert_eq!(parsed.bucket, "bucket-a");
} }
/// madmin-go's `ParseARN` hard-rejects anything that does not start with
/// `arn:minio:`, so generated ARNs must use the `minio` partition or the
/// native mc/madmin tooling cannot decode remote-target listings.
#[test]
fn display_emits_minio_partition() {
let arn = ARN::new(
BucketTargetType::ReplicationService,
"depl-123".to_string(),
String::new(),
"bucket-a".to_string(),
);
assert_eq!(arn.to_string(), "arn:minio:replication::depl-123:bucket-a");
}
/// Persisted bucket-targets.json files from older RustFS releases carry
/// `arn:rustfs:` ARNs; the legacy partition must stay parseable forever.
#[test]
fn from_str_accepts_legacy_rustfs_partition() {
let parsed = ARN::from_str("arn:rustfs:replication:us-east-1:depl-123:bucket-a").expect("legacy ARN must parse");
assert_eq!(parsed.arn_type, BucketTargetType::ReplicationService);
assert_eq!(parsed.region, "us-east-1");
assert_eq!(parsed.id, "depl-123");
assert_eq!(parsed.bucket, "bucket-a");
}
/// The partition whitelist is the only structural gate: `BucketTargetType::
/// from_str(...).unwrap_or_default()` never fails, so any 6-segment string
/// would otherwise parse as `type=None`.
#[test]
fn from_str_rejects_unknown_partition() {
assert!(ARN::from_str("arn:aws:replication::depl-123:bucket-a").is_err());
assert!(ARN::from_str("not-an-arn").is_err());
}
} }
@@ -59,10 +59,6 @@ impl fmt::Debug for Credentials {
} }
#[derive(Debug, Deserialize, Serialize, Default, Clone)] #[derive(Debug, Deserialize, Serialize, Default, Clone)]
#[allow(
dead_code,
reason = "MinIO-parity bucket-target service discriminator with no caller in this port (backlog#1823)"
)]
pub enum ServiceType { pub enum ServiceType {
#[default] #[default]
Replication, Replication,
+17 -20
View File
@@ -73,6 +73,23 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
check_bucket_name_common(bucket_name, true) check_bucket_name_common(bucket_name, true)
} }
pub fn check_valid_object_name_prefix(object_name: &str) -> Result<()> {
if object_name.len() > 1024 {
return Err(Error::other("Object name cannot be longer than 1024 characters"));
}
if !object_name.is_ascii() {
return Err(Error::other("Object name with non-UTF-8 strings are not supported"));
}
Ok(())
}
pub fn check_valid_object_name(object_name: &str) -> Result<()> {
if object_name.trim().is_empty() {
return Err(Error::other("Object name cannot be empty"));
}
check_valid_object_name_prefix(object_name)
}
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T> pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
where where
T: for<'xml> xml::Deserialize<'xml>, T: for<'xml> xml::Deserialize<'xml>,
@@ -83,10 +100,6 @@ where
Ok(ans) Ok(ans)
} }
#[allow(
dead_code,
reason = "xml serialize helper with no caller in this port; the live sibling is deserialize (backlog#1823)"
)]
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> { pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
let mut buf = Vec::with_capacity(256); let mut buf = Vec::with_capacity(256);
{ {
@@ -173,27 +186,15 @@ pub fn is_valid_object_name(object: &str) -> bool {
/// Client-facing reason attached to rejections of object keys that Win32/NTFS /// Client-facing reason attached to rejections of object keys that Win32/NTFS
/// cannot represent as file paths (issue #3299). Deployments on Linux/macOS /// cannot represent as file paths (issue #3299). Deployments on Linux/macOS
/// accept the full S3 key character set. /// accept the full S3 key character set.
#[allow(
dead_code,
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
)]
pub const WINDOWS_RESERVED_CHARACTERS_REASON: &str = pub const WINDOWS_RESERVED_CHARACTERS_REASON: &str =
"object key contains characters unsupported on Windows hosts (one of ':', '*', '?', '\"', '|', '<', '>')"; "object key contains characters unsupported on Windows hosts (one of ':', '*', '?', '\"', '|', '<', '>')";
/// Client-facing reason for path segments Windows can store but not address /// Client-facing reason for path segments Windows can store but not address
/// afterwards (issue #3449): trailing dot/space or reserved DOS device names. /// afterwards (issue #3449): trailing dot/space or reserved DOS device names.
#[allow(
dead_code,
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
)]
pub const WINDOWS_RESERVED_SEGMENT_REASON: &str = "object key contains a path segment unsupported on Windows hosts (trailing dot or space, or a reserved device name such as NUL/CON/COM1)"; pub const WINDOWS_RESERVED_SEGMENT_REASON: &str = "object key contains a path segment unsupported on Windows hosts (trailing dot or space, or a reserved device name such as NUL/CON/COM1)";
/// Reserved DOS device names that shadow regular files on Windows, even when /// Reserved DOS device names that shadow regular files on Windows, even when
/// an extension is appended (e.g. `NUL.txt` resolves to the `NUL` device). /// an extension is appended (e.g. `NUL.txt` resolves to the `NUL` device).
#[allow(
dead_code,
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
)]
const WINDOWS_RESERVED_NAMES: &[&str] = &[ const WINDOWS_RESERVED_NAMES: &[&str] = &[
"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3",
"LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
@@ -203,10 +204,6 @@ const WINDOWS_RESERVED_NAMES: &[&str] = &[
/// the Win32 API cannot address afterwards (issue #3449): segments ending in a /// the Win32 API cannot address afterwards (issue #3449): segments ending in a
/// dot or a space, and reserved DOS device names — bare or with an extension /// dot or a space, and reserved DOS device names — bare or with an extension
/// (`NUL.txt`), matching classic Win32 path resolution semantics. /// (`NUL.txt`), matching classic Win32 path resolution semantics.
#[allow(
dead_code,
reason = "live on Windows: callers sit inside the #[cfg(target_os = \"windows\")] block in check_object_name_for_length_and_slash (backlog#1823)"
)]
pub fn object_name_has_windows_incompatible_segment(object: &str) -> bool { pub fn object_name_has_windows_incompatible_segment(object: &str) -> bool {
object.split(['/', '\\']).any(|segment| { object.split(['/', '\\']).any(|segment| {
if segment.ends_with('.') || segment.ends_with(' ') { if segment.ends_with('.') || segment.ends_with(' ') {
@@ -90,10 +90,6 @@ impl BucketVersioningSys {
/// caller's own instance context so a second in-process store never /// caller's own instance context so a second in-process store never
/// answers with the first instance's versioning state; falls back to the /// answers with the first instance's versioning state; falls back to the
/// ambient system when the instance cell is not initialized. /// ambient system when the instance cell is not initialized.
#[allow(
dead_code,
reason = "instance-scoped seam (backlog#1052) with no caller in this port (backlog#1823)"
)]
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<VersioningConfiguration> { pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<VersioningConfiguration> {
if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) { if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) {
return Ok(VersioningConfiguration::default()); return Ok(VersioningConfiguration::default());
@@ -15,7 +15,6 @@
use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout}; use crate::disk::disk_store::{get_drive_walkdir_peek_timeout, get_drive_walkdir_stall_timeout};
use crate::disk::error::DiskError; use crate::disk::error::DiskError;
use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions}; use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
use futures::future::join_all;
use metrics::counter; use metrics::counter;
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof}; use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
use std::{ use std::{
@@ -656,7 +655,6 @@ async fn list_path_raw_inner(
errs.push(None); errs.push(None);
} }
let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()]; let mut pending_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
let mut peek_outcomes: Vec<Option<PeekOutcome>> = std::iter::repeat_with(|| None).take(readers.len()).collect();
loop { loop {
let mut current = MetaCacheEntry::default(); let mut current = MetaCacheEntry::default();
@@ -678,21 +676,6 @@ async fn list_path_raw_inner(
let mut has_err = 0; let mut has_err = 0;
let mut agree = 0; let mut agree = 0;
// Start every missing head read in the same round so one stalled
// disk cannot multiply the wait budget by the erasure-set width.
// Outcomes are still consumed below in stable disk-index order.
let concurrent_peeks = readers.iter_mut().enumerate().filter_map(|(i, reader)| {
if errs[i].is_some() || pending_entries[i].is_some() {
return None;
}
let cancel = &revjob_rx;
Some(async move { (i, peek_with_timeout(cancel, reader, peek_timeout).await) })
});
for (i, outcome) in join_all(concurrent_peeks).await {
peek_outcomes[i] = Some(outcome);
}
for (i, r) in readers.iter_mut().enumerate() { for (i, r) in readers.iter_mut().enumerate() {
if errs[i].is_some() { if errs[i].is_some() {
has_err += 1; has_err += 1;
@@ -702,10 +685,7 @@ async fn list_path_raw_inner(
let entry = if let Some(entry) = pending_entries[i].take() { let entry = if let Some(entry) = pending_entries[i].take() {
entry entry
} else { } else {
let Some(outcome) = peek_outcomes[i].take() else { match peek_with_timeout(&revjob_rx, r, peek_timeout).await {
return Err(DiskError::Unexpected);
};
match outcome {
PeekOutcome::Ready(res) => { PeekOutcome::Ready(res) => {
if let Some(entry) = res { if let Some(entry) = res {
// info!("read entry disk: {}, name: {}", i, entry.name); // info!("read entry disk: {}, name: {}", i, entry.name);
@@ -1315,36 +1295,6 @@ mod tests {
assert_eq!(err, DiskError::Timeout); assert_eq!(err, DiskError::Timeout);
} }
#[tokio::test(start_paused = true)]
async fn list_path_raw_bounds_multiple_stalled_readers_by_one_peek_deadline() {
let peek_timeout = Duration::from_millis(20);
let started = tokio::time::Instant::now();
let err = list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![None, None, None, None],
min_disks: 1,
test_reader_behaviors: vec![
TestReaderBehavior::Stall,
TestReaderBehavior::Stall,
TestReaderBehavior::Stall,
TestReaderBehavior::Stall,
],
peek_timeout: Some(peek_timeout),
..Default::default()
},
)
.await
.expect_err("all stalled readers should fail the listing");
assert_eq!(err, DiskError::Timeout);
assert_eq!(
started.elapsed(),
peek_timeout,
"reader deadlines must overlap instead of accumulating once per disk"
);
}
#[tokio::test] #[tokio::test]
async fn list_path_raw_waits_past_producer_stall_for_slow_progressing_reader() { async fn list_path_raw_waits_past_producer_stall_for_slow_progressing_reader() {
let entry = MetaCacheEntry { let entry = MetaCacheEntry {
@@ -229,6 +229,17 @@ pub fn http_resp_to_error_response(
err_resp err_resp
} }
pub fn err_transfer_acceleration_bucket(bucket_name: &str) -> ErrorResponse {
ErrorResponse {
status_code: StatusCode::BAD_REQUEST,
code: S3ErrorCode::InvalidArgument,
message: "The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods .."
.to_string(),
bucket_name: bucket_name.to_string(),
..Default::default()
}
}
pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse { pub fn err_entity_too_large(total_size: i64, max_object_size: i64, bucket_name: &str, object_name: &str) -> ErrorResponse {
let msg = format!( let msg = format!(
"Your proposed upload size {} exceeds the maximum allowed object size {} for single PUT operation.", "Your proposed upload size {} exceeds the maximum allowed object size {} for single PUT operation.",
@@ -284,6 +295,16 @@ pub fn err_invalid_argument(message: &str) -> ErrorResponse {
} }
} }
pub fn err_api_not_supported(message: &str) -> ErrorResponse {
ErrorResponse {
status_code: StatusCode::NOT_IMPLEMENTED,
code: S3ErrorCode::Custom("APINotSupported".into()),
message: message.to_string(),
request_id: "rustfs".to_string(),
..Default::default()
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@@ -135,10 +135,6 @@ impl Object {
Self { ..Default::default() } Self { ..Default::default() }
} }
#[allow(
dead_code,
reason = "MinIO-parity reader surface with no caller in this port (backlog#1823)"
)]
fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> { fn do_get_request(&self, request: &GetRequest) -> Result<GetResponse, std::io::Error> {
let _ = request.did_offset_change; let _ = request.did_offset_change;
let _ = request.offset; let _ = request.offset;
@@ -154,20 +150,12 @@ impl Object {
)) ))
} }
#[allow(
dead_code,
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
)]
fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> { fn set_offset(&mut self, bytes_read: i64) -> Result<(), std::io::Error> {
self.curr_offset += bytes_read; self.curr_offset += bytes_read;
Ok(()) Ok(())
} }
#[allow(
dead_code,
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
)]
fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> { fn read(&mut self, b: &[u8]) -> Result<i64, std::io::Error> {
let mut read_req = GetRequest { let mut read_req = GetRequest {
is_read_op: true, is_read_op: true,
@@ -192,10 +180,6 @@ impl Object {
Ok(response.size) Ok(response.size)
} }
#[allow(
dead_code,
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
)]
fn stat(&self) -> Result<ObjectInfo, std::io::Error> { fn stat(&self) -> Result<ObjectInfo, std::io::Error> {
if !self.is_started || !self.object_info_set { if !self.is_started || !self.object_info_set {
let _ = self.do_get_request(&GetRequest { let _ = self.do_get_request(&GetRequest {
@@ -208,10 +192,6 @@ impl Object {
Ok(self.object_info.clone()) Ok(self.object_info.clone())
} }
#[allow(
dead_code,
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
)]
fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> { fn read_at(&mut self, b: &[u8], offset: i64) -> Result<i64, std::io::Error> {
self.curr_offset = offset; self.curr_offset = offset;
@@ -239,10 +219,6 @@ impl Object {
Ok(response.size) Ok(response.size)
} }
#[allow(
dead_code,
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
)]
fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> { fn seek(&mut self, offset: i64, whence: i64) -> Result<i64, std::io::Error> {
if !self.is_started || !self.object_info_set { if !self.is_started || !self.object_info_set {
let seek_req = GetRequest { let seek_req = GetRequest {
@@ -277,10 +253,6 @@ impl Object {
Ok(self.curr_offset) Ok(self.curr_offset)
} }
#[allow(
dead_code,
reason = "MinIO-parity Object reader method with no caller in this port (backlog#1823)"
)]
fn close(&mut self) -> Result<(), std::io::Error> { fn close(&mut self) -> Result<(), std::io::Error> {
self.is_closed = true; self.is_closed = true;
Ok(()) Ok(())
+1 -1
View File
@@ -37,7 +37,7 @@ use crate::client::{
api_put_object_common::optimal_part_info, api_put_object_common::optimal_part_info,
api_put_object_multipart::UploadPartParams, api_put_object_multipart::UploadPartParams,
api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart}, api_s3_datatypes::{CompleteMultipartUpload, CompletePart, ObjectPart},
constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE}, constants::{ISO8601_DATEFORMAT, MAX_MULTIPART_PUT_OBJECT_SIZE, MIN_PART_SIZE, TOTAL_WORKERS},
credentials::SignatureType, credentials::SignatureType,
transition_api::{ReaderImpl, TransitionClient, UploadInfo}, transition_api::{ReaderImpl, TransitionClient, UploadInfo},
utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header}, utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
@@ -30,6 +30,10 @@ pub fn is_object(reader: &ReaderImpl) -> bool {
matches!(reader, ReaderImpl::ObjectBody(_)) matches!(reader, ReaderImpl::ObjectBody(_))
} }
pub fn is_read_at(reader: ReaderImpl) -> bool {
matches!(reader, ReaderImpl::ObjectBody(_))
}
pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> { pub fn optimal_part_info(object_size: i64, configured_part_size: u64) -> Result<(i64, i64, i64), std::io::Error> {
let unknown_size; let unknown_size;
let mut object_size = object_size; let mut object_size = object_size;
@@ -81,6 +81,18 @@ async fn read_multipart_part(reader: &mut ReaderImpl, want: usize) -> Result<Vec
} }
} }
pub struct UploadedPartRes {
pub error: std::io::Error,
pub part_num: i64,
pub size: i64,
pub part: ObjectPart,
}
pub struct UploadPartReq {
pub part_num: i64,
pub part: ObjectPart,
}
impl TransitionClient { impl TransitionClient {
pub async fn put_object_multipart_stream( pub async fn put_object_multipart_stream(
self: Arc<Self>, self: Arc<Self>,
+38 -18
View File
@@ -29,6 +29,10 @@ use crate::client::utils::base64_decode;
use super::transition_api; use super::transition_api;
pub struct ListAllMyBucketsResult {
pub owner: Owner,
}
#[derive(Debug, Default, Serialize, Deserialize)] #[derive(Debug, Default, Serialize, Deserialize)]
pub struct CommonPrefix { pub struct CommonPrefix {
pub prefix: String, pub prefix: String,
@@ -85,10 +89,6 @@ pub struct ListVersionsResult {
pub next_version_id_marker: String, pub next_version_id_marker: String,
} }
#[allow(
dead_code,
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
)]
pub struct ListBucketResult { pub struct ListBucketResult {
common_prefixes: Vec<CommonPrefix>, common_prefixes: Vec<CommonPrefix>,
contents: Vec<transition_api::ObjectInfo>, contents: Vec<transition_api::ObjectInfo>,
@@ -102,10 +102,6 @@ pub struct ListBucketResult {
prefix: String, prefix: String,
} }
#[allow(
dead_code,
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
)]
pub struct ListMultipartUploadsResult { pub struct ListMultipartUploadsResult {
bucket: String, bucket: String,
key_marker: String, key_marker: String,
@@ -121,15 +117,16 @@ pub struct ListMultipartUploadsResult {
common_prefixes: Vec<CommonPrefix>, common_prefixes: Vec<CommonPrefix>,
} }
#[allow(
dead_code,
reason = "fields of a MinIO-parity list result that this port builds but never reads back (backlog#1823)"
)]
pub struct Initiator { pub struct Initiator {
id: String, id: String,
display_name: String, display_name: String,
} }
pub struct CopyObjectResult {
pub etag: String,
pub last_modified: OffsetDateTime,
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct ObjectPart { pub struct ObjectPart {
pub etag: String, pub etag: String,
@@ -263,7 +260,6 @@ pub struct CompletePart {
} }
impl CompletePart { impl CompletePart {
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
fn checksum(&self, t: &ChecksumMode) -> String { fn checksum(&self, t: &ChecksumMode) -> String {
match t { match t {
ChecksumMode::ChecksumCRC32C => { ChecksumMode::ChecksumCRC32C => {
@@ -288,6 +284,11 @@ impl CompletePart {
} }
} }
pub struct CopyObjectPartResult {
pub etag: String,
pub last_modified: OffsetDateTime,
}
#[derive(Debug, Default, serde::Serialize)] #[derive(Debug, Default, serde::Serialize)]
#[serde(rename = "CompleteMultipartUpload")] #[serde(rename = "CompleteMultipartUpload")]
pub struct CompleteMultipartUpload { pub struct CompleteMultipartUpload {
@@ -356,10 +357,10 @@ impl CompleteMultipartUpload {
} }
} }
#[allow( pub struct CreateBucketConfiguration {
dead_code, pub location: String,
reason = "live via quick_xml::de::from_str in bucket_cache.rs; serde deserialization is not a construction (backlog#1823)" }
)]
#[derive(serde::Serialize)] #[derive(serde::Serialize)]
pub struct DeleteObject { pub struct DeleteObject {
//api has //api has
@@ -367,6 +368,21 @@ pub struct DeleteObject {
pub version_id: String, pub version_id: String,
} }
pub struct DeletedObject {
//s3s has
pub key: String,
pub version_id: String,
pub deletemarker: bool,
pub deletemarker_version_id: String,
}
pub struct NonDeletedObject {
pub key: String,
pub code: String,
pub message: String,
pub version_id: String,
}
#[derive(serde::Serialize)] #[derive(serde::Serialize)]
pub struct DeleteMultiObjects { pub struct DeleteMultiObjects {
pub quiet: bool, pub quiet: bool,
@@ -386,7 +402,6 @@ impl DeleteMultiObjects {
Ok(buf) Ok(buf)
} }
#[allow(dead_code, reason = "MinIO-parity XML helper with no caller in this port (backlog#1823)")]
pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> { pub fn unmarshal(buf: &[u8]) -> Result<Self, std::io::Error> {
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
struct WireDeleteObject { struct WireDeleteObject {
@@ -421,3 +436,8 @@ impl DeleteMultiObjects {
}) })
} }
} }
pub struct DeleteMultiObjectsResult {
pub deleted_objects: Vec<DeletedObject>,
pub undeleted_objects: Vec<NonDeletedObject>,
}
-4
View File
@@ -365,10 +365,6 @@ mod tests {
pub struct Checksum { pub struct Checksum {
checksum_type: ChecksumMode, checksum_type: ChecksumMode,
r: Vec<u8>, r: Vec<u8>,
#[allow(
dead_code,
reason = "checksum bookkeeping field kept beside the value it guards (backlog#1823)"
)]
computed: bool, computed: bool,
} }
+3
View File
@@ -32,5 +32,8 @@ pub const MAX_MULTIPART_PUT_OBJECT_SIZE: i64 = 1024 * 1024 * 1024 * 1024 * 5;
pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"; pub const UNSIGNED_PAYLOAD_TRAILER: &str = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
pub const TOTAL_WORKERS: i64 = 4;
pub const SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] = pub const ISO8601_DATEFORMAT: &[FormatItem<'_>] =
format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z"); format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]Z");
+19 -12
View File
@@ -67,10 +67,6 @@ impl<P: Provider + Default> Credentials<P> {
Ok(self.creds.clone()) Ok(self.creds.clone())
} }
#[allow(
dead_code,
reason = "MinIO-parity credential surface with no caller in this port (backlog#1823)"
)]
fn expire(&mut self) { fn expire(&mut self) {
self.force_refresh = true; self.force_refresh = true;
} }
@@ -137,10 +133,6 @@ impl Provider for Static {
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
pub struct STSError { pub struct STSError {
#[allow(
dead_code,
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
)]
pub r#type: String, pub r#type: String,
pub code: String, pub code: String,
pub message: String, pub message: String,
@@ -149,10 +141,6 @@ pub struct STSError {
#[derive(Debug, Clone, thiserror::Error)] #[derive(Debug, Clone, thiserror::Error)]
pub struct ErrorResponse { pub struct ErrorResponse {
pub sts_error: STSError, pub sts_error: STSError,
#[allow(
dead_code,
reason = "MinIO-parity STS error detail that this port never reads back (backlog#1823)"
)]
pub request_id: String, pub request_id: String,
} }
@@ -170,3 +158,22 @@ impl ErrorResponse {
return self.sts_error.message.clone(); return self.sts_error.message.clone();
} }
} }
pub fn xml_decoder<T>(body: &[u8]) -> Result<T, Error>
where
for<'de> T: Deserialize<'de>,
{
match std::str::from_utf8(body) {
Ok(xml_body) => quick_xml::de::from_str::<T>(xml_body).map_err(|err| Error::new(ErrorKind::InvalidData, err.to_string())),
Err(err) => Err(Error::new(ErrorKind::InvalidData, err.to_string())),
}
}
pub fn xml_decode_and_body<T>(body_reader: &[u8]) -> Result<(Vec<u8>, T), std::io::Error>
where
for<'de> T: Deserialize<'de>,
{
let body = body_reader.to_vec();
let parsed = xml_decoder(&body)?;
Ok((body, parsed))
}
+1
View File
@@ -13,6 +13,7 @@
// limitations under the License. // limitations under the License.
// #730: S3 client compatibility models are kept while ECStore callers move to narrower facades. // #730: S3 client compatibility models are kept while ECStore callers move to narrower facades.
#![allow(dead_code)]
pub mod admin_handler_utils; pub mod admin_handler_utils;
pub mod api_error_response; pub mod api_error_response;
@@ -77,6 +77,39 @@ fn part_number_to_rangespec(oi: ObjectInfo, part_number: usize) -> Option<HTTPRa
}) })
} }
fn get_compressed_offsets(oi: ObjectInfo, offset: i64) -> (i64, i64, i64, i64, u64) {
let mut skip_length: i64 = 0;
let mut cumulative_actual_size: i64 = 0;
let mut first_part_idx: i64 = 0;
let mut compressed_offset: i64 = 0;
let mut part_skip: i64 = 0;
let mut decrypt_skip: i64 = 0;
let mut seq_num: u64 = 0;
for (i, part) in oi.parts.iter().enumerate() {
cumulative_actual_size += part.actual_size as i64;
if cumulative_actual_size <= offset {
compressed_offset += part.size as i64;
} else {
first_part_idx = i as i64;
skip_length = cumulative_actual_size - part.actual_size as i64;
break;
}
}
skip_length = offset - skip_length;
let parts: &[ObjectPartInfo] = &oi.parts;
if skip_length > 0
&& parts.len() > first_part_idx as usize
&& parts[first_part_idx as usize].index.as_ref().is_some_and(|idx| idx.len() > 0)
{
let _ = part_skip;
let _ = decrypt_skip;
let _ = seq_num;
}
(compressed_offset, part_skip, first_part_idx, decrypt_skip, seq_num)
}
pub fn new_getobjectreader<'a>( pub fn new_getobjectreader<'a>(
rs: &Option<HTTPRangeSpec>, rs: &Option<HTTPRangeSpec>,
oi: &'a ObjectInfo, oi: &'a ObjectInfo,
@@ -23,7 +23,6 @@ const X_OBS_VERSION_ID: &str = "x-obs-version-id";
const MAX_REMOTE_VERSION_ID_LEN: usize = 1024; const MAX_REMOTE_VERSION_ID_LEN: usize = 1024;
#[derive(Clone, Copy, Debug, Eq, PartialEq)] #[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[allow(dead_code, reason = "bucket versioning states kept as a complete vocabulary (backlog#1823)")]
pub(crate) enum BucketVersioningState { pub(crate) enum BucketVersioningState {
Unknown, Unknown,
Disabled, Disabled,
@@ -48,7 +47,6 @@ impl RemoteVersion {
} }
} }
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
pub(crate) fn exact_request_id(&self) -> Result<Option<&str>, Error> { pub(crate) fn exact_request_id(&self) -> Result<Option<&str>, Error> {
match self { match self {
Self::Unknown => Err(Error::new( Self::Unknown => Err(Error::new(
@@ -101,10 +101,6 @@ where
const C_UNKNOWN: i32 = -1; const C_UNKNOWN: i32 = -1;
const C_OFFLINE: i32 = 0; const C_OFFLINE: i32 = 0;
#[allow(
dead_code,
reason = "reachable only from the unused transition client methods below (backlog#1823)"
)]
const C_ONLINE: i32 = 1; const C_ONLINE: i32 = 1;
fn invalid_utf8_header_error(scope: &str, header_name: &str) -> std::io::Error { fn invalid_utf8_header_error(scope: &str, header_name: &str) -> std::io::Error {
@@ -324,10 +320,6 @@ impl TransitionClient {
Ok(client) Ok(client)
} }
#[allow(
dead_code,
reason = "MinIO-parity transition client surface with no caller in this port (backlog#1823)"
)]
fn endpoint_url(&self) -> Url { fn endpoint_url(&self) -> Url {
self.endpoint_url.clone() self.endpoint_url.clone()
} }
@@ -356,20 +348,12 @@ impl TransitionClient {
.to_string()) .to_string())
} }
#[allow(
dead_code,
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
)]
fn trace_errors_only_off(&self) { fn trace_errors_only_off(&self) {
if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() { if let Ok(mut trace_errors_only) = self.trace_errors_only.lock() {
*trace_errors_only = false; *trace_errors_only = false;
} }
} }
#[allow(
dead_code,
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
)]
fn trace_off(&self) { fn trace_off(&self) {
if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() { if let Ok(mut is_trace_enabled) = self.is_trace_enabled.lock() {
*is_trace_enabled = false; *is_trace_enabled = false;
@@ -379,20 +363,12 @@ impl TransitionClient {
} }
} }
#[allow(
dead_code,
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
)]
fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) { fn set_s3_transfer_accelerate(&self, accelerate_endpoint: &str) {
if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() { if let Ok(mut endpoint) = self.s3_accelerate_endpoint.lock() {
*endpoint = accelerate_endpoint.to_string(); *endpoint = accelerate_endpoint.to_string();
} }
} }
#[allow(
dead_code,
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
)]
fn set_s3_enable_dual_stack(&self, enabled: bool) { fn set_s3_enable_dual_stack(&self, enabled: bool) {
if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() { if let Ok(mut dual_stack) = self.s3_dual_stack_enabled.lock() {
*dual_stack = enabled; *dual_stack = enabled;
@@ -422,18 +398,10 @@ impl TransitionClient {
(hash_algos, hash_sums) (hash_algos, hash_sums)
} }
#[allow(
dead_code,
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
)]
fn is_online(&self) -> bool { fn is_online(&self) -> bool {
!self.is_offline() !self.is_offline()
} }
#[allow(
dead_code,
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
)]
fn mark_offline(&self) { fn mark_offline(&self) {
self.health_status self.health_status
.compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst); .compare_exchange(C_ONLINE, C_OFFLINE, Ordering::SeqCst, Ordering::SeqCst);
@@ -443,18 +411,10 @@ impl TransitionClient {
self.health_status.load(Ordering::SeqCst) == C_OFFLINE self.health_status.load(Ordering::SeqCst) == C_OFFLINE
} }
#[allow(
dead_code,
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
)]
fn health_check(hc_duration: Duration) { fn health_check(hc_duration: Duration) {
let _ = hc_duration; let _ = hc_duration;
} }
#[allow(
dead_code,
reason = "MinIO-parity transition client method with no caller in this port (backlog#1823)"
)]
fn dump_http(&self, req: &Request<s3s::Body>, resp: &Response<Incoming>) -> Result<(), std::io::Error> { fn dump_http(&self, req: &Request<s3s::Body>, resp: &Response<Incoming>) -> Result<(), std::io::Error> {
let mut resp_trace: Vec<u8>; let mut resp_trace: Vec<u8>;
@@ -1142,7 +1102,6 @@ impl Default for ObjectInfo {
} }
impl ObjectInfo { impl ObjectInfo {
#[allow(dead_code, reason = "MinIO-parity accessor with no caller in this port (backlog#1823)")]
pub(crate) fn remote_version( pub(crate) fn remote_version(
&self, &self,
capabilities: ProviderVersionCapabilities, capabilities: ProviderVersionCapabilities,
+4
View File
@@ -48,6 +48,10 @@ lazy_static! {
}; };
} }
pub fn is_standard_query_value(qs_key: &str) -> bool {
SUPPORTED_QUERY_VALUES[qs_key]
}
pub fn is_storageclass_header(header_key: &str) -> bool { pub fn is_storageclass_header(header_key: &str) -> bool {
header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase() header_key.to_lowercase() == X_AMZ_STORAGE_CLASS.as_str().to_lowercase()
} }
-37
View File
@@ -190,17 +190,6 @@ pub(crate) const GET_METADATA_CACHE_REASON_VERSION_SUSPENDED: &str = "version_su
pub(crate) const GET_METADATA_CACHE_REASON_VERSIONED: &str = "versioned"; pub(crate) const GET_METADATA_CACHE_REASON_VERSIONED: &str = "versioned";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY: &str = "data_read_inline_body_verify";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED: &str = "data_read_inline_deleted";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY: &str = "data_read_inline_geometry";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH: &str = "data_read_inline_identity_mismatch";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD: &str = "data_read_inline_missing_payload";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD: &str = "data_read_inline_missing_shard";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE: &str = "data_read_inline_not_inline";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE: &str = "data_read_inline_part_shape";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE: &str = "data_read_inline_remote";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE: &str = "data_read_inline_size";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED: &str = "data_read_inline_transformed";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM: &str = "insufficient_quorum"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM: &str = "insufficient_quorum";
pub(crate) const GET_METADATA_EARLY_STOP_REASON_NOT_FOUND: &str = "not_found"; pub(crate) const GET_METADATA_EARLY_STOP_REASON_NOT_FOUND: &str = "not_found";
@@ -562,32 +551,6 @@ mod tests {
assert_eq!(GET_METADATA_CACHE_REASON_VERSIONED, "versioned"); assert_eq!(GET_METADATA_CACHE_REASON_VERSIONED, "versioned");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker");
assert_eq!(
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY,
"data_read_inline_body_verify"
);
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, "data_read_inline_deleted");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, "data_read_inline_geometry");
assert_eq!(
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH,
"data_read_inline_identity_mismatch"
);
assert_eq!(
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD,
"data_read_inline_missing_payload"
);
assert_eq!(
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD,
"data_read_inline_missing_shard"
);
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, "data_read_inline_not_inline");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, "data_read_inline_part_shape");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, "data_read_inline_remote");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, "data_read_inline_size");
assert_eq!(
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED,
"data_read_inline_transformed"
);
assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, "insufficient_quorum"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, "insufficient_quorum");
assert_eq!(GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, "not_found"); assert_eq!(GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, "not_found");
+33 -13
View File
@@ -637,23 +637,14 @@ impl Default for DiskOperationMetrics {
} }
impl DiskOperationMetrics { impl DiskOperationMetrics {
#[allow(
dead_code,
reason = "internal metrics recorder reached only from record() below (backlog#1823)"
)]
fn record_call(&mut self) { fn record_call(&mut self) {
self.lifetime_calls.fetch_add(1, Ordering::Relaxed); self.lifetime_calls.fetch_add(1, Ordering::Relaxed);
} }
#[allow(
dead_code,
reason = "internal metrics recorder reached only from record() below (backlog#1823)"
)]
fn record_latency(&mut self, now_sec: u64, elapsed: Duration) { fn record_latency(&mut self, now_sec: u64, elapsed: Duration) {
self.record_latency_atomic(now_sec, elapsed); self.record_latency_atomic(now_sec, elapsed);
} }
#[allow(dead_code, reason = "metrics roll-up with no caller in this port (backlog#1823)")]
fn record(&mut self, now_sec: u64, elapsed: Duration) { fn record(&mut self, now_sec: u64, elapsed: Duration) {
self.record_call(); self.record_call();
self.record_latency(now_sec, elapsed); self.record_latency(now_sec, elapsed);
@@ -779,7 +770,6 @@ impl DiskHealthTracker {
} }
/// Set disk as faulty /// Set disk as faulty
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub fn set_faulty(&self) { pub fn set_faulty(&self) {
self.status.store(DISK_HEALTH_FAULTY, Ordering::Release); self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
} }
@@ -860,7 +850,6 @@ impl DiskHealthTracker {
became_offline became_offline
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool { pub fn mark_offline(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
let current = self.runtime_state(); let current = self.runtime_state();
if current == RuntimeDriveHealthState::Offline { if current == RuntimeDriveHealthState::Offline {
@@ -991,13 +980,11 @@ impl DiskHealthTracker {
} }
/// Get waiting operations count /// Get waiting operations count
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub fn waiting_count(&self) -> u32 { pub fn waiting_count(&self) -> u32 {
self.waiting.load(Ordering::Relaxed) self.waiting.load(Ordering::Relaxed)
} }
/// Get last success timestamp /// Get last success timestamp
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub fn last_success(&self) -> i64 { pub fn last_success(&self) -> i64 {
self.last_success.load(Ordering::Acquire) self.last_success.load(Ordering::Acquire)
} }
@@ -1039,6 +1026,21 @@ impl Default for DiskHealthTracker {
} }
} }
/// Health check context key for tracking disk operations
#[derive(Debug, Clone)]
struct HealthDiskCtxKey;
#[derive(Debug)]
struct HealthDiskCtxValue {
last_success: Arc<AtomicI64>,
}
impl HealthDiskCtxValue {
fn log_success(&self) {
self.last_success.store(current_unix_nanos(), Ordering::Relaxed);
}
}
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities. /// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
/// This is similar to Go's xlStorageDiskIDCheck. /// This is similar to Go's xlStorageDiskIDCheck.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@@ -1070,6 +1072,10 @@ impl LocalDiskWrapper {
) )
} }
pub(crate) fn new_with_health(disk: Arc<LocalDisk>, health_check: bool, health: Arc<DiskHealthTracker>) -> Self {
Self::new_with_health_and_metrics(disk, health_check, health, Arc::new(DiskHealthMetricEpoch::default()))
}
pub(crate) fn new_with_reconnect_state( pub(crate) fn new_with_reconnect_state(
disk: Arc<LocalDisk>, disk: Arc<LocalDisk>,
health_check: bool, health_check: bool,
@@ -1432,6 +1438,20 @@ impl LocalDiskWrapper {
} }
} }
async fn check_id(&self, want_id: Option<Uuid>) -> Result<()> {
if want_id.is_none() {
return Ok(());
}
let stored_disk_id = self.disk.get_disk_id().await?;
if stored_disk_id != want_id {
return Err(Error::other(format!("Disk ID mismatch wanted {want_id:?}, got {stored_disk_id:?}")));
}
Ok(())
}
/// Check if disk ID is stale /// Check if disk ID is stale
async fn check_disk_stale(&self) -> Result<()> { async fn check_disk_stale(&self) -> Result<()> {
let Some(current_disk_id) = *self.disk_id.read().await else { let Some(current_disk_id) = *self.disk_id.read().await else {
-1
View File
@@ -48,7 +48,6 @@ pub fn to_volume_error(io_err: std::io::Error) -> std::io::Error {
} }
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error { pub fn to_disk_error(io_err: std::io::Error) -> std::io::Error {
match io_err.kind() { match io_err.kind() {
std::io::ErrorKind::NotFound => DiskError::DiskNotFound.into(), std::io::ErrorKind::NotFound => DiskError::DiskNotFound.into(),
-1
View File
@@ -178,7 +178,6 @@ pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
} }
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> { pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
// Try remove_file first; fall back to remove_dir_all if it's a directory // Try remove_file first; fall back to remove_dir_all if it's a directory
match fs::remove_file(path.as_ref()).await { match fs::remove_file(path.as_ref()).await {
-69
View File
@@ -665,7 +665,6 @@ async fn remove_empty_directory_tree_under_mount_lease(
} }
#[cfg(unix)] #[cfg(unix)]
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
async fn remove_empty_directory_tree_with( async fn remove_empty_directory_tree_with(
root: &Path, root: &Path,
before_descend: impl FnMut(&Path) -> std::io::Result<()>, before_descend: impl FnMut(&Path) -> std::io::Result<()>,
@@ -1017,29 +1016,13 @@ fn record_direct_read_page_fault_delta(path: &'static str, stage: &'static str,
/// When enabled, shard reads bypass the page cache using O_DIRECT flag. /// When enabled, shard reads bypass the page cache using O_DIRECT flag.
/// Requires aligned buffers (typically 512 bytes or 4096 bytes). /// Requires aligned buffers (typically 512 bytes or 4096 bytes).
/// Default: false (uses page cache via mmap/pread). /// Default: false (uses page cache via mmap/pread).
#[allow(
dead_code,
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
)]
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE"; const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE";
#[allow(
dead_code,
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
)]
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: bool = false; const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE: bool = false;
/// Minimum shard size threshold for O_DIRECT reads. /// Minimum shard size threshold for O_DIRECT reads.
/// Only shards larger than this threshold will use O_DIRECT. /// Only shards larger than this threshold will use O_DIRECT.
/// Default: 4MB. /// Default: 4MB.
#[allow(
dead_code,
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
)]
const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD"; const ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: &str = "RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD";
#[allow(
dead_code,
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
)]
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024; const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
/// Enable O_DIRECT for erasure shard / multipart part data writes (Linux only). /// Enable O_DIRECT for erasure shard / multipart part data writes (Linux only).
@@ -1053,15 +1036,7 @@ const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD: usize = 4 * 1024 * 1024;
/// EINVAL/EOPNOTSUPP (tmpfs, overlayfs, 9p, ...) latch the path off and fall /// EINVAL/EOPNOTSUPP (tmpfs, overlayfs, 9p, ...) latch the path off and fall
/// back to buffered writes for the whole disk. Non-Linux always falls back. /// back to buffered writes for the whole disk. Non-Linux always falls back.
/// Default: false (buffered writes via the page cache, as before). /// Default: false (buffered writes via the page cache, as before).
#[allow(
dead_code,
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
)]
const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE"; const ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: &str = "RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE";
#[allow(
dead_code,
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
)]
const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false; const DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE: bool = false;
const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE"; const ENV_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: &str = "RUSTFS_OBJECT_MMAP_POPULATE_ENABLE";
const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false; const DEFAULT_RUSTFS_OBJECT_MMAP_POPULATE_ENABLE: bool = false;
@@ -1120,14 +1095,12 @@ macro_rules! cached_read_env {
cached_read_env! { cached_read_env! {
/// Check if O_DIRECT reads are enabled. /// Check if O_DIRECT reads are enabled.
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
fn is_direct_io_read_enabled() -> bool = fn is_direct_io_read_enabled() -> bool =
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE); rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_ENABLE);
} }
cached_read_env! { cached_read_env! {
/// Check if O_DIRECT shard/part data writes are enabled. /// Check if O_DIRECT shard/part data writes are enabled.
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
fn is_direct_io_write_enabled() -> bool = fn is_direct_io_write_enabled() -> bool =
rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE); rustfs_utils::get_env_bool(ENV_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE);
} }
@@ -1483,7 +1456,6 @@ pub(crate) fn effective_durability(volume: &str) -> DurabilityMode {
cached_read_env! { cached_read_env! {
/// Get the O_DIRECT read threshold size. /// Get the O_DIRECT read threshold size.
#[allow(dead_code, reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)")]
fn get_direct_io_read_threshold() -> usize = fn get_direct_io_read_threshold() -> usize =
rustfs_utils::get_env_usize(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD); rustfs_utils::get_env_usize(ENV_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD, DEFAULT_RUSTFS_OBJECT_DIRECT_IO_READ_THRESHOLD);
} }
@@ -1701,20 +1673,12 @@ impl DirectIoWriteState {
/// Target staging size for O_DIRECT writes, rounded up to the DIO alignment. /// Target staging size for O_DIRECT writes, rounded up to the DIO alignment.
/// Bounds the per-writer aligned bounce buffer and batches many shard blocks /// Bounds the per-writer aligned bounce buffer and batches many shard blocks
/// into one positioned write to keep the syscall count low. /// into one positioned write to keep the syscall count low.
#[allow(
dead_code,
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
)]
const DIRECT_WRITE_STAGING_BYTES: usize = 1024 * 1024; const DIRECT_WRITE_STAGING_BYTES: usize = 1024 * 1024;
/// Aligned bounce-buffer capacity for a given DIO alignment: the target staging /// Aligned bounce-buffer capacity for a given DIO alignment: the target staging
/// size rounded up to a whole multiple of `align` so the buffer address, every /// size rounded up to a whole multiple of `align` so the buffer address, every
/// flushed batch length, and every write offset stay alignment-correct. /// flushed batch length, and every write offset stay alignment-correct.
/// Platform-independent (no O_DIRECT), so it is unit-tested on any host. /// Platform-independent (no O_DIRECT), so it is unit-tested on any host.
#[allow(
dead_code,
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
)]
fn direct_write_staging_capacity(align: usize) -> usize { fn direct_write_staging_capacity(align: usize) -> usize {
debug_assert!(align.is_power_of_two() && align >= 512); debug_assert!(align.is_power_of_two() && align >= 512);
DIRECT_WRITE_STAGING_BYTES.div_ceil(align) * align DIRECT_WRITE_STAGING_BYTES.div_ceil(align) * align
@@ -1723,10 +1687,6 @@ fn direct_write_staging_capacity(align: usize) -> usize {
/// Split `filled` staged bytes into the alignment-sized prefix written with /// Split `filled` staged bytes into the alignment-sized prefix written with
/// O_DIRECT and the sub-alignment tail written buffered. Platform-independent, /// O_DIRECT and the sub-alignment tail written buffered. Platform-independent,
/// so the tail-boundary math is unit-tested on any host. /// so the tail-boundary math is unit-tested on any host.
#[allow(
dead_code,
reason = "platform-conditional: production callers are inside #[cfg(target_os = \"linux\")] blocks, so this reads as dead on non-Linux hosts (backlog#1823)"
)]
fn direct_write_tail_split(filled: usize, align: usize) -> (usize, usize) { fn direct_write_tail_split(filled: usize, align: usize) -> (usize, usize) {
let aligned = filled - (filled % align); let aligned = filled - (filled % align);
(aligned, filled - aligned) (aligned, filled - aligned)
@@ -2182,7 +2142,6 @@ fn set_delete_version_fail_after_data_staged(path: &str) {
} }
#[cfg(test)] #[cfg(test)]
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(crate) fn set_delete_version_fail_after_commit(root: &Path, path: &str) { pub(crate) fn set_delete_version_fail_after_commit(root: &Path, path: &str) {
DELETE_VERSION_FAIL_AFTER_COMMIT DELETE_VERSION_FAIL_AFTER_COMMIT
.lock() .lock()
@@ -2488,10 +2447,6 @@ enum SyncMode {
FileOnly, FileOnly,
} }
#[allow(
dead_code,
reason = "reclaim bookkeeping fields written by Drop but never read back (backlog#1823)"
)]
struct FileCacheReclaimWriter { struct FileCacheReclaimWriter {
inner: File, inner: File,
reclaim_len: usize, reclaim_len: usize,
@@ -2499,10 +2454,6 @@ struct FileCacheReclaimWriter {
reclaimed: bool, reclaimed: bool,
} }
#[allow(
dead_code,
reason = "reclaim bookkeeping fields written by Drop but never read back (backlog#1823)"
)]
struct FileCacheReclaimReader { struct FileCacheReclaimReader {
inner: File, inner: File,
reclaim_offset: u64, reclaim_offset: u64,
@@ -2568,10 +2519,6 @@ impl<R: AsyncRead + Unpin> AsyncRead for StallTimeoutReader<R> {
} }
} }
#[allow(
dead_code,
reason = "reclaim metrics emitter reached only from the Linux-gated reclaim paths (backlog#1823)"
)]
fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) { fn record_file_cache_reclaim_success(kind: &'static str, reclaim_len: usize, started: std::time::Instant) {
// Runs per read-stream page-cache reclaim window; skip the whole emission // Runs per read-stream page-cache reclaim window; skip the whole emission
// (three metric-key constructions) when general metrics are disabled. // (three metric-key constructions) when general metrics are disabled.
@@ -3124,7 +3071,6 @@ impl LocalIoBackend for StdBackend {
use memmap2::MmapOptions; use memmap2::MmapOptions;
use std::time::{Duration as StdDuration, Instant as StdInstant}; use std::time::{Duration as StdDuration, Instant as StdInstant};
#[allow(dead_code, reason = "mmap copy result slot kept beside the mapping it owns (backlog#1823)")]
struct MmapCopyReadResult { struct MmapCopyReadResult {
bytes: Bytes, bytes: Bytes,
access_check_duration: StdDuration, access_check_duration: StdDuration,
@@ -4758,10 +4704,6 @@ fn build_local_io_backend(root: PathBuf) -> Arc<dyn LocalIoBackend> {
Arc::new(StdBackend::new(root)) Arc::new(StdBackend::new(root))
} }
#[allow(
dead_code,
reason = "path cache and cwd slots retained beside the disk root they derive from (backlog#1823)"
)]
pub struct LocalDisk { pub struct LocalDisk {
pub root: PathBuf, pub root: PathBuf,
publication_root: os::PublicationRoot, publication_root: os::PublicationRoot,
@@ -5548,7 +5490,6 @@ impl LocalDisk {
Ok(Self::resolve_abs_path_from(&self.root, path.as_ref())) Ok(Self::resolve_abs_path_from(&self.root, path.as_ref()))
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn io_resolve_abs_path(&self, path: impl AsRef<Path>) -> PathBuf { fn io_resolve_abs_path(&self, path: impl AsRef<Path>) -> PathBuf {
let path_ref = path.as_ref(); let path_ref = path.as_ref();
let path_str = path_ref.to_string_lossy(); let path_str = path_ref.to_string_lossy();
@@ -5626,24 +5567,15 @@ impl LocalDisk {
} }
// Check if a path is valid // Check if a path is valid
#[allow(
dead_code,
reason = "method wrapper over the live free function check_local_disk_valid_path; no caller in this port (backlog#1823)"
)]
fn check_valid_path<P: AsRef<Path>>(&self, path: P) -> Result<()> { fn check_valid_path<P: AsRef<Path>>(&self, path: P) -> Result<()> {
check_local_disk_valid_path(self.io_root(), path) check_local_disk_valid_path(self.io_root(), path)
} }
#[allow(
dead_code,
reason = "method wrapper over the live free function reject_local_disk_symlink_components; no caller in this port (backlog#1823)"
)]
fn reject_symlink_components(&self, path: &Path) -> Result<()> { fn reject_symlink_components(&self, path: &Path) -> Result<()> {
reject_local_disk_symlink_components(self.io_root(), path) reject_local_disk_symlink_components(self.io_root(), path)
} }
// Batch path generation with single lock acquisition // Batch path generation with single lock acquisition
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn get_object_paths_batch(&self, requests: &[(String, String)]) -> Result<Vec<PathBuf>> { fn get_object_paths_batch(&self, requests: &[(String, String)]) -> Result<Vec<PathBuf>> {
let mut results = Vec::with_capacity(requests.len()); let mut results = Vec::with_capacity(requests.len());
let mut cache_misses = Vec::new(); let mut cache_misses = Vec::new();
@@ -6556,7 +6488,6 @@ impl LocalDisk {
Ok(f) Ok(f)
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
async fn open_file_read_only(&self, path: impl AsRef<Path>) -> Result<File> { async fn open_file_read_only(&self, path: impl AsRef<Path>) -> Result<File> {
let f = super::fs::open_file(path.as_ref(), O_RDONLY).await.map_err(to_file_error)?; let f = super::fs::open_file(path.as_ref(), O_RDONLY).await.map_err(to_file_error)?;
Ok(f) Ok(f)
+1 -5
View File
@@ -13,6 +13,7 @@
// limitations under the License. // limitations under the License.
// #730: disk abstractions still carry staged health and direct-I/O migration paths. // #730: disk abstractions still carry staged health and direct-I/O migration paths.
#![allow(dead_code)]
pub mod disk_store; pub mod disk_store;
pub mod endpoint; pub mod endpoint;
@@ -1113,10 +1114,6 @@ pub struct DiskInfo {
} }
#[derive(Clone, Debug, Default)] #[derive(Clone, Debug, Default)]
#[allow(
dead_code,
reason = "MinIO-parity disk info shape with no constructor in this port (backlog#1823)"
)]
pub struct Info { pub struct Info {
pub total: u64, pub total: u64,
pub free: u64, pub free: u64,
@@ -1375,7 +1372,6 @@ pub fn conv_part_err_to_int(err: &Option<Error>) -> usize {
} }
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub fn has_part_err(part_errs: &[usize]) -> bool { pub fn has_part_err(part_errs: &[usize]) -> bool {
part_errs.iter().any(|err| *err != CHECK_PART_SUCCESS) part_errs.iter().any(|err| *err != CHECK_PART_SUCCESS)
} }
+11 -5
View File
@@ -571,10 +571,6 @@ fn regular_files(dir: &Path) -> io::Result<Vec<PathBuf>> {
/// Fdatasync every regular file directly inside `dir`, then fsync the directory /// Fdatasync every regular file directly inside `dir`, then fsync the directory
/// itself. /// itself.
#[allow(
dead_code,
reason = "reached only through sync_dir_files, whose callers are tests (backlog#1823)"
)]
pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> { pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
for entry in std::fs::read_dir(dir.as_ref())? { for entry in std::fs::read_dir(dir.as_ref())? {
let entry = entry?; let entry = entry?;
@@ -587,7 +583,6 @@ pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
/// Async wrapper around [`sync_dir_files_std`]. Large directories flush files /// Async wrapper around [`sync_dir_files_std`]. Large directories flush files
/// concurrently, bounded both per directory and process-wide. /// concurrently, bounded both per directory and process-wide.
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub async fn sync_dir_files(dir: impl AsRef<Path>) -> io::Result<()> { pub async fn sync_dir_files(dir: impl AsRef<Path>) -> io::Result<()> {
sync_dir_files_with_limiter(dir, Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))).await sync_dir_files_with_limiter(dir, Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))).await
} }
@@ -1814,6 +1809,10 @@ impl RenameCommitGuard {
}) })
} }
pub(crate) fn lock_destination_directory_for_path_access(&self, directory: &Path) -> io::Result<RenameDestinationPathGuard> {
self.destination_directory_guard(directory, false)
}
pub(crate) fn create_destination_directory_for_path_access( pub(crate) fn create_destination_directory_for_path_access(
&self, &self,
directory: &Path, directory: &Path,
@@ -2859,6 +2858,13 @@ pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>
Ok(()) Ok(())
} }
/// Check if a file exists.
/// Returns true if the file exists, false otherwise.
#[tracing::instrument(level = "debug", skip_all)]
pub fn file_exists(path: impl AsRef<Path>) -> bool {
std::fs::metadata(path.as_ref()).map(|_| true).unwrap_or(false)
}
/// Whether an [`io::Error`] means "the directory is not empty". /// Whether an [`io::Error`] means "the directory is not empty".
/// ///
/// POSIX lets `rmdir`/`rename` report a non-empty directory as either /// POSIX lets `rmdir`/`rename` report a non-empty directory as either
+12 -291
View File
@@ -820,263 +820,10 @@ impl BitrotWriterWrapper {
} }
} }
// --- startup bitrot self-test (rustfs/backlog#1873, MinIO bitrotSelfTest parity) ---
//
// A broken hash implementation (bad SIMD feature combination, platform drift, a
// key-handling regression) fails silently: every shard reads back "corrupt",
// heal rewrites data that was fine, and cross-platform clusters disagree about
// which copy is healthy. The self-test below pins the algorithms the moment a
// process starts, so a drifted build announces itself instead of quietly
// rewriting objects. See docs/rustfs-heal-scanner-vs-minio-comprehensive-
// analysis-2026-08-16.md §6 HS-11.
/// Length of the deterministic self-test payload.
pub const BITROT_SELF_TEST_PAYLOAD_LEN: usize = 4096;
/// Known-answer digest of [`bitrot_self_test_payload`] under `HighwayHash256S`
/// (the production default). Pinned so any platform or build where the
/// implementation drifts fails startup instead of mis-hashing shards.
const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S: [u8; 32] = [
0xb9, 0x32, 0xa2, 0xaa, 0x4a, 0xb7, 0x33, 0x6a, 0xa3, 0xca, 0x7e, 0x61, 0x9d, 0x86, 0x52, 0x14, 0x6e, 0x7f, 0xd8, 0x9e, 0xea,
0x08, 0xd9, 0x8c, 0x33, 0x85, 0x87, 0x19, 0x30, 0xd6, 0xed, 0x06,
];
/// Known-answer digest of the same payload under `HighwayHash256SLegacy`.
const BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY: [u8; 32] = [
0x98, 0x24, 0x71, 0x4f, 0x16, 0xbb, 0x48, 0x39, 0xed, 0x68, 0xfa, 0x63, 0x5e, 0xd9, 0x07, 0x61, 0xdf, 0x0a, 0xff, 0xcf, 0x7d,
0x8c, 0xa8, 0xc7, 0xc0, 0xb6, 0x6f, 0x05, 0xdb, 0xda, 0x5a, 0x22,
];
/// FIPS 180-2 test vector: SHA-256 of the ASCII string "abc". Unlike the
/// Highway digests above this one is externally verifiable, so it guards the
/// whole `HashAlgorithm` plumbing even for readers who distrust pinned
/// self-computed constants.
const BITROT_SELF_TEST_KAT_SHA256_ABC: [u8; 32] = [
0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96,
0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad,
];
/// Deterministic self-test payload: xorshift64* from a fixed seed, so every
/// platform and every run hashes the same 4096 bytes.
fn bitrot_self_test_payload() -> [u8; BITROT_SELF_TEST_PAYLOAD_LEN] {
let mut state = 0x9E37_79B9_7F4A_7C15u64;
let mut payload = [0u8; BITROT_SELF_TEST_PAYLOAD_LEN];
for byte in payload.iter_mut() {
state ^= state >> 12;
state ^= state << 25;
state ^= state >> 27;
*byte = state.wrapping_mul(0x2545_F491_4F6C_DD1D) as u8;
}
payload
}
/// Why a bitrot self-test failed.
#[derive(Debug)]
pub enum BitrotSelfTestError {
/// A known-answer digest mismatched the pinned constant.
KnownAnswerMismatch {
algorithm: &'static str,
got: String,
want: String,
},
/// A freshly encoded shard failed `bitrot_verify`.
RoundtripVerify { algorithm: &'static str, detail: String },
/// A verified roundtrip read back different bytes than were written.
RoundtripReadback { algorithm: &'static str },
/// A deliberately tampered shard was not rejected by `bitrot_verify`.
TamperNotRejected {
algorithm: &'static str,
tampered: &'static str,
},
}
impl std::fmt::Display for BitrotSelfTestError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::KnownAnswerMismatch { algorithm, got, want } => {
write!(f, "known-answer mismatch for {algorithm}: got {got}, want {want}")
}
Self::RoundtripVerify { algorithm, detail } => write!(f, "{algorithm} roundtrip shard failed verification: {detail}"),
Self::RoundtripReadback { algorithm } => write!(f, "{algorithm} roundtrip read back different bytes"),
Self::TamperNotRejected { algorithm, tampered } => {
write!(f, "{algorithm} tampered shard ({tampered}) was not rejected")
}
}
}
}
impl std::error::Error for BitrotSelfTestError {}
fn self_test_hex(bytes: &[u8]) -> String {
rustfs_utils::hex(bytes)
}
// (kept as a named one-liner so every KAT failure site reads the same; the
// underlying formatter is the shared `rustfs_utils::hex`)
/// Compare a digest against its pinned constant. Split out so a test can drive
/// it with a wrong constant and prove the mismatch path fires.
fn bitrot_kat_check(
algorithm: &'static str,
algo: &HashAlgorithm,
payload: &[u8],
expected: &[u8; 32],
) -> Result<(), BitrotSelfTestError> {
let digest = algo.hash_encode(payload);
let digest = digest.as_ref();
if digest.len() != expected.len() || digest != expected.as_slice() {
return Err(BitrotSelfTestError::KnownAnswerMismatch {
algorithm,
got: self_test_hex(digest),
want: self_test_hex(expected),
});
}
Ok(())
}
/// Encode `payload` with `shard_size` blocks, verify it end to end, and read
/// every block back through `BitrotReader` comparing bytes.
async fn bitrot_roundtrip_check(
algorithm: &'static str,
algo: HashAlgorithm,
payload: &[u8],
shard_size: usize,
) -> Result<(), BitrotSelfTestError> {
let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
for chunk in payload.chunks(shard_size) {
writer
.write(chunk)
.await
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
algorithm,
detail: format!("encode failed: {err}"),
})?;
}
let encoded = writer.into_inner().into_inner();
let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone());
if encoded.len() != on_disk {
return Err(BitrotSelfTestError::RoundtripVerify {
algorithm,
detail: format!("encoded {} bytes, size formula says {on_disk}", encoded.len()),
});
}
bitrot_verify(std::io::Cursor::new(encoded.clone()), on_disk, payload.len(), algo.clone(), shard_size)
.await
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
algorithm,
detail: err.to_string(),
})?;
let mut reader = BitrotReader::new(std::io::Cursor::new(encoded), shard_size, algo, false);
let mut offset = 0usize;
while offset < payload.len() {
let want = shard_size.min(payload.len() - offset);
let mut buf = vec![0u8; want];
let read = reader
.read(&mut buf)
.await
.map_err(|err| BitrotSelfTestError::RoundtripVerify {
algorithm,
detail: format!("read back failed at offset {offset}: {err}"),
})?;
if read != want || buf[..read] != payload[offset..offset + read] {
return Err(BitrotSelfTestError::RoundtripReadback { algorithm });
}
offset += read;
}
Ok(())
}
/// Flip one byte and require `bitrot_verify` to reject the result.
async fn bitrot_tamper_check(
algorithm: &'static str,
algo: HashAlgorithm,
payload: &[u8],
shard_size: usize,
tampered: &'static str,
flip_at: usize,
) -> Result<(), BitrotSelfTestError> {
let mut writer = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
for chunk in payload.chunks(shard_size) {
writer.write(chunk).await.expect("self-test encode should not fail");
}
let mut corrupt = writer.into_inner().into_inner();
let flip_index = flip_at % corrupt.len();
corrupt[flip_index] ^= 0x80;
let on_disk = bitrot_shard_file_size(payload.len(), shard_size, algo.clone());
match bitrot_verify(std::io::Cursor::new(corrupt), on_disk, payload.len(), algo, shard_size).await {
// The flipped byte must be rejected as a hash mismatch specifically, not
// by any incidental read error: an in-memory cursor cannot fail reads,
// so accepting any other failure here would mask a verify path that
// errors out before it ever compares hashes.
Err(err) if err.to_string().contains("hash mismatch") => Ok(()),
Ok(()) => Err(BitrotSelfTestError::TamperNotRejected { algorithm, tampered }),
Err(err) => Err(BitrotSelfTestError::RoundtripVerify {
algorithm,
detail: format!("tampered shard rejected with an unexpected error: {err}"),
}),
}
}
/// Verify every bitrot algorithm this crate can write or verify in production:
/// both streaming Highway variants roundtrip end to end (encode → size formula
/// → `bitrot_verify` → read back) and reject a flipped byte in both the data
/// and the leading hash, while all three hashed algorithms reproduce their
/// pinned known-answer digests.
///
/// Runs in well under a millisecond on 4 KiB of data; callers may run it inline
/// at startup. Pure CPU, no allocation beyond a few KiB of scratch.
pub async fn bitrot_self_test() -> Result<(), BitrotSelfTestError> {
let payload = bitrot_self_test_payload();
// Externally verifiable vector first: it guards the HashAlgorithm plumbing
// itself, before any self-pinned constants are consulted.
let abc = HashAlgorithm::SHA256.hash_encode(b"abc");
if abc.as_ref() != BITROT_SELF_TEST_KAT_SHA256_ABC.as_slice() {
return Err(BitrotSelfTestError::KnownAnswerMismatch {
algorithm: "SHA256",
got: self_test_hex(abc.as_ref()),
want: self_test_hex(&BITROT_SELF_TEST_KAT_SHA256_ABC),
});
}
bitrot_kat_check(
"HighwayHash256S",
&HashAlgorithm::HighwayHash256S,
&payload,
&BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S,
)?;
bitrot_kat_check(
"HighwayHash256SLegacy",
&HashAlgorithm::HighwayHash256SLegacy,
&payload,
&BITROT_SELF_TEST_KAT_HIGHWAY_HASH256S_LEGACY,
)?;
for (algorithm, algo) in [
("HighwayHash256S", HashAlgorithm::HighwayHash256S),
("HighwayHash256SLegacy", HashAlgorithm::HighwayHash256SLegacy),
] {
// Full blocks plus a partial tail, exactly like a real part stripe.
let tail_len = 2 * 1024 + 333;
bitrot_roundtrip_check(algorithm, algo.clone(), &payload, 1024).await?;
bitrot_roundtrip_check(algorithm, algo.clone(), &payload[..tail_len], 1024).await?;
// One flipped byte in the final data block, one in the first leading
// hash: both must fail verification.
bitrot_tamper_check(algorithm, algo.clone(), &payload, 1024, "final data byte", payload.len() - 1).await?;
bitrot_tamper_check(algorithm, algo, &payload, 1024, "leading hash byte", 0).await?;
}
Ok(())
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::{ use super::{
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_kat_check, bitrot_self_test, BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
bitrot_self_test_payload, bitrot_shard_file_size, bitrot_verify, write_all_vectored,
}; };
use super::{MAX_RETAINED_CHUNKS_PER_BLOCK, ShardChunkRead, ShardSource}; use super::{MAX_RETAINED_CHUNKS_PER_BLOCK, ShardChunkRead, ShardSource};
use bytes::Bytes; use bytes::Bytes;
@@ -1343,32 +1090,6 @@ mod tests {
} }
} }
#[test]
fn bitrot_self_test_payload_is_deterministic() {
// Two independent builds of the payload must agree byte for byte, or
// the pinned known-answer digests below would be meaningless.
assert_eq!(bitrot_self_test_payload(), bitrot_self_test_payload());
}
#[test]
fn bitrot_self_test_rejects_a_wrong_known_answer_digest() {
let payload = bitrot_self_test_payload();
let wrong = [0u8; 32];
let err = bitrot_kat_check("HighwayHash256S", &HashAlgorithm::HighwayHash256S, &payload, &wrong)
.expect_err("a zeroed digest must never match");
match err {
super::BitrotSelfTestError::KnownAnswerMismatch { algorithm, .. } => assert_eq!(algorithm, "HighwayHash256S"),
other => panic!("expected KnownAnswerMismatch, got {other:?}"),
}
}
#[tokio::test]
async fn bitrot_self_test_passes() {
bitrot_self_test()
.await
.expect("the pinned digests and roundtrip checks must all pass on this platform");
}
#[tokio::test] #[tokio::test]
async fn vectored_test_writers_cover_fallback_flush_and_shutdown_paths() { async fn vectored_test_writers_cover_fallback_flush_and_shutdown_paths() {
let mut counting = VectoredCountingWriter::default(); let mut counting = VectoredCountingWriter::default();
@@ -1468,7 +1189,7 @@ mod tests {
let last = corrupt.len() - 1; let last = corrupt.len() - 1;
corrupt[last] ^= 0x80; corrupt[last] ^= 0x80;
let err = bitrot_verify( let err = bitrot_verify(
std::io::Cursor::new(corrupt), Cursor::new(corrupt),
super::bitrot_shard_file_size(data.len(), shard_size, algo.clone()), super::bitrot_shard_file_size(data.len(), shard_size, algo.clone()),
data.len(), data.len(),
algo, algo,
@@ -1561,7 +1282,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn bitrot_reader_rejects_output_buffers_larger_than_shard_size() { async fn bitrot_reader_rejects_output_buffers_larger_than_shard_size() {
let mut reader = BitrotReader::new(std::io::Cursor::new(Vec::<u8>::new()), 4, HashAlgorithm::None, false); let mut reader = BitrotReader::new(Cursor::new(Vec::<u8>::new()), 4, HashAlgorithm::None, false);
let mut out = [0u8; 5]; let mut out = [0u8; 5];
let err = reader let err = reader
.read(&mut out) .read(&mut out)
@@ -1686,7 +1407,7 @@ mod tests {
(HashAlgorithm::HighwayHash256, true), (HashAlgorithm::HighwayHash256, true),
] { ] {
let label = format!("{algo:?}"); let label = format!("{algo:?}");
let writer = std::io::Cursor::new(Vec::<u8>::new()); let writer = Cursor::new(Vec::<u8>::new());
let mut w = BitrotWriter::new(writer, shard_size, algo.clone()); let mut w = BitrotWriter::new(writer, shard_size, algo.clone());
w.write(&[7u8; 16]).await.unwrap(); w.write(&[7u8; 16]).await.unwrap();
let written = w.into_inner().into_inner(); let written = w.into_inner().into_inner();
@@ -1771,7 +1492,7 @@ mod tests {
} }
async fn encode_one_block(payload: &[u8], shard_size: usize, algo: HashAlgorithm) -> Vec<u8> { async fn encode_one_block(payload: &[u8], shard_size: usize, algo: HashAlgorithm) -> Vec<u8> {
let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo); let mut w = BitrotWriter::new(Cursor::new(Vec::<u8>::new()), shard_size, algo);
w.write(payload).await.unwrap(); w.write(payload).await.unwrap();
w.into_inner().into_inner() w.into_inner().into_inner()
} }
@@ -1879,7 +1600,7 @@ mod tests {
for algo in [HashAlgorithm::HighwayHash256S, HashAlgorithm::HighwayHash256SLegacy] { for algo in [HashAlgorithm::HighwayHash256S, HashAlgorithm::HighwayHash256SLegacy] {
for &size in &[1usize, 16, 17, 32, 40, 48] { for &size in &[1usize, 16, 17, 32, 40, 48] {
let payload: Vec<u8> = (0..size).map(|i| i as u8).collect(); let payload: Vec<u8> = (0..size).map(|i| i as u8).collect();
let mut w = BitrotWriter::new(std::io::Cursor::new(Vec::<u8>::new()), shard_size, algo.clone()); let mut w = BitrotWriter::new(Cursor::new(Vec::<u8>::new()), shard_size, algo.clone());
for chunk in payload.chunks(shard_size) { for chunk in payload.chunks(shard_size) {
w.write(chunk).await.unwrap(); w.write(chunk).await.unwrap();
} }
@@ -1953,14 +1674,14 @@ mod tests {
w.write(&data).await.expect("write shard"); w.write(&data).await.expect("write shard");
let mut via_read = vec![0u8; SHARD]; let mut via_read = vec![0u8; SHARD];
let n1 = BitrotReader::new(std::io::Cursor::new(encoded.clone()), SHARD, algo.clone(), false) let n1 = BitrotReader::new(Cursor::new(encoded.clone()), SHARD, algo.clone(), false)
.read(&mut via_read) .read(&mut via_read)
.await .await
.expect("read"); .expect("read");
// A buffer with only capacity — no initialized bytes at all. // A buffer with only capacity — no initialized bytes at all.
let mut via_append: Vec<u8> = Vec::with_capacity(SHARD); let mut via_append: Vec<u8> = Vec::with_capacity(SHARD);
let n2 = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false) let n2 = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false)
.read_appending(&mut via_append, SHARD) .read_appending(&mut via_append, SHARD)
.await .await
.expect("read_appending"); .expect("read_appending");
@@ -1985,7 +1706,7 @@ mod tests {
encoded.truncate(encoded.len() - 1); encoded.truncate(encoded.len() - 1);
let mut out: Vec<u8> = Vec::with_capacity(SHARD); let mut out: Vec<u8> = Vec::with_capacity(SHARD);
let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo.clone(), false) let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo.clone(), false)
.read_appending(&mut out, SHARD) .read_appending(&mut out, SHARD)
.await .await
.expect_err("a truncated shard must not succeed"); .expect_err("a truncated shard must not succeed");
@@ -2011,7 +1732,7 @@ mod tests {
encoded[last] ^= 0xff; encoded[last] ^= 0xff;
let mut out: Vec<u8> = Vec::with_capacity(SHARD); let mut out: Vec<u8> = Vec::with_capacity(SHARD);
let err = BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false) let err = BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
.read_appending(&mut out, SHARD) .read_appending(&mut out, SHARD)
.await .await
.expect_err("a corrupt shard must not verify"); .expect_err("a corrupt shard must not verify");
@@ -2123,7 +1844,7 @@ mod tests {
"Cursor<Bytes> must be able to hand out a block, otherwise the fast path is dead code" "Cursor<Bytes> must be able to hand out a block, otherwise the fast path is dead code"
); );
assert_eq!(mem.position(), 8, "taking a block must advance like a read of the same length"); assert_eq!(mem.position(), 8, "taking a block must advance like a read of the same length");
let mut streamed = std::io::Cursor::new(encoded.clone()); let mut streamed = Cursor::new(encoded.clone());
assert!( assert!(
ShardSource::try_take_block(&mut streamed, 8).is_none(), ShardSource::try_take_block(&mut streamed, 8).is_none(),
"a non-Bytes source must stay on the streaming path" "a non-Bytes source must stay on the streaming path"
@@ -2151,7 +1872,7 @@ mod tests {
); );
let mut via_stream: Vec<u8> = Vec::with_capacity(SHARD); let mut via_stream: Vec<u8> = Vec::with_capacity(SHARD);
BitrotReader::new(std::io::Cursor::new(encoded), SHARD, algo, false) BitrotReader::new(Cursor::new(encoded), SHARD, algo, false)
.read_appending(&mut via_stream, SHARD) .read_appending(&mut via_stream, SHARD)
.await .await
.expect("streaming read"); .expect("streaming read");
-1
View File
@@ -704,7 +704,6 @@ pub(crate) async fn create_bitrot_reader_from_bytes_with_stage_metrics(
} }
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub fn create_deferred_bitrot_reader( pub fn create_deferred_bitrot_reader(
inline_data: Option<Bytes>, inline_data: Option<Bytes>,
disk: Option<DiskStore>, disk: Option<DiskStore>,
-6
View File
@@ -277,12 +277,6 @@ pub struct ObjectOptions {
/// fence avoids recursively acquiring the read lock behind a queued writer. /// fence avoids recursively acquiring the read lock behind a queued writer.
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>, pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
pub replication_request: bool, pub replication_request: bool,
/// Source-cluster LWW timestamps carried by an authorized replication
/// request; None when the source never modified the category. Only the
/// replication-authorized options builders may set these.
pub replication_tagging_timestamp: Option<OffsetDateTime>,
pub replication_retention_timestamp: Option<OffsetDateTime>,
pub replication_legalhold_timestamp: Option<OffsetDateTime>,
/// Authorized SSE-C replication passthrough: the body is already /// Authorized SSE-C replication passthrough: the body is already
/// ciphertext, so the write path must not encrypt or compress it and /// ciphertext, so the write path must not encrypt or compress it and
/// stores the restored encryption metadata verbatim. Only the /// stores the restored encryption metadata verbatim. Only the
+68 -626
View File
@@ -32,22 +32,15 @@ use crate::diagnostics::get::{
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER, GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER,
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID,
GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED, GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED,
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING,
GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK,
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY,
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE,
GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure, GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
@@ -180,13 +173,11 @@ pub(in crate::set_disk) enum GetCodecStreamingReaderBuildOutcome {
Fallback(GetCodecStreamingFallbackReason), Fallback(GetCodecStreamingFallbackReason),
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(in crate::set_disk) struct MultipartCodecStreamingReader { pub(in crate::set_disk) struct MultipartCodecStreamingReader {
pub(in crate::set_disk) readers: VecDeque<Box<dyn AsyncRead + Unpin + Send + Sync>>, pub(in crate::set_disk) readers: VecDeque<Box<dyn AsyncRead + Unpin + Send + Sync>>,
} }
impl MultipartCodecStreamingReader { impl MultipartCodecStreamingReader {
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(in crate::set_disk) fn new(readers: Vec<Box<dyn AsyncRead + Unpin + Send + Sync>>) -> Self { pub(in crate::set_disk) fn new(readers: Vec<Box<dyn AsyncRead + Unpin + Send + Sync>>) -> Self {
Self { Self {
readers: VecDeque::from(readers), readers: VecDeque::from(readers),
@@ -661,15 +652,36 @@ pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo
&& left.erasure.distribution == right.erasure.distribution && left.erasure.distribution == right.erasure.distribution
} }
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason( pub(in crate::set_disk) async fn data_read_early_stop_inline_body_verified(
bucket: &str, bucket: &str,
object: &str, object: &str,
candidate: &FileInfo, candidate: &FileInfo,
parts_metadata: &[FileInfo], parts_metadata: &[FileInfo],
disks: &[Option<DiskStore>], disks: &[Option<DiskStore>],
) -> Option<&'static str> { ) -> bool {
if let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(candidate) { if !candidate.inline_data()
return Some(reason); || candidate.is_compressed()
|| candidate
.metadata
.keys()
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|| candidate.is_remote()
|| candidate.deleted
|| candidate.size <= 0
|| candidate.parts.len() != 1
|| !candidate.has_valid_erasure_geometry()
{
return false;
}
let Ok(object_size) = usize::try_from(candidate.size) else {
return false;
};
if candidate.parts.first().is_none_or(|part| part.size != object_size) {
return false;
}
if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) {
return false;
} }
let Ok(erasure) = coding::Erasure::try_new_with_options( let Ok(erasure) = coding::Erasure::try_new_with_options(
@@ -678,21 +690,18 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
candidate.erasure.block_size, candidate.erasure.block_size,
candidate.uses_legacy_checksum, candidate.uses_legacy_checksum,
) else { ) else {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY); return false;
}; };
let data_files = let Some(data_files) =
match collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, candidate, erasure.data_shards, |index| { collect_inline_data_shard_fileinfos_by_index(parts_metadata, candidate, erasure.data_shards, |index| {
disks.get(index).is_some_and(Option::is_some) disks.get(index).is_some_and(Option::is_some)
}) { })
Ok(data_files) => data_files, else {
Err(reason) => return Some(reason), return false;
}; };
let Some(part) = candidate.parts.first() else { let Some(part) = candidate.parts.first() else {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE); return false;
};
let Ok(object_size) = usize::try_from(candidate.size) else {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
}; };
let checksum_info = candidate.erasure.get_checksum_info(part.number); let checksum_info = candidate.erasure.get_checksum_info(part.number);
let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S { let checksum_algo = if candidate.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
@@ -712,111 +721,12 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
let Ok(mut readers) = let Ok(mut readers) =
build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
else { else {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
};
match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await {
Some(body) if body.len() == object_size => None,
_ => Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
}
}
fn data_read_early_stop_inline_candidate_miss_reason(candidate: &FileInfo) -> Option<&'static str> {
// `inline_data` excludes remote objects; this diagnostic reports them separately.
if !rustfs_utils::http::contains_key_str(&candidate.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA) {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE);
}
if candidate.is_compressed()
|| candidate
.metadata
.keys()
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
{
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED);
}
if candidate.is_remote() {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE);
}
if candidate.deleted {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED);
}
if candidate.size <= 0 {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
}
if candidate.parts.len() != 1 {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
}
if !candidate.has_valid_erasure_geometry() {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
}
let Ok(object_size) = usize::try_from(candidate.size) else {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
};
if candidate.parts.first().is_none_or(|part| part.size != object_size) {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
}
if !can_try_inline_data_shards_direct(object_size, candidate.erasure.block_size) {
return Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE);
}
None
}
fn data_read_inline_missing_shards_are_pending(
candidate: &FileInfo,
parts_metadata: &[FileInfo],
errors: &[Option<DiskError>],
disks: &[Option<DiskStore>],
fanout_order: &[usize],
scheduled_fanout_len: usize,
) -> bool {
let Ok(erasure) = coding::Erasure::try_new_with_options(
candidate.erasure.data_blocks,
candidate.erasure.parity_blocks,
candidate.erasure.block_size,
candidate.uses_legacy_checksum,
) else {
return false; return false;
}; };
let distribution = &candidate.erasure.distribution;
let mut data_shards_seen_or_pending = vec![false; erasure.data_shards];
let mut missing_pending_data_shards = 0usize;
for (disk_index, file_info) in parts_metadata.iter().enumerate() { try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size)
let Some(&block_index) = distribution.get(disk_index) else { .await
return false; .is_some_and(|body| body.len() == object_size)
};
if block_index == 0 || block_index > erasure.data_shards {
continue;
}
if !disks.get(disk_index).is_some_and(Option::is_some) {
return false;
}
let data_slot = block_index - 1;
if file_info.name.is_empty() {
let scheduled_and_not_failed = fanout_order
.get(..scheduled_fanout_len)
.is_some_and(|scheduled_disks| scheduled_disks.contains(&disk_index))
&& errors.get(disk_index).is_some_and(Option::is_none);
if scheduled_and_not_failed {
data_shards_seen_or_pending[data_slot] = true;
missing_pending_data_shards = missing_pending_data_shards.saturating_add(1);
continue;
}
return false;
}
if file_info.erasure.index != block_index
|| !file_info.has_valid_erasure_geometry()
|| !metadata_early_stop_candidate_matches(file_info, candidate)
|| file_info.data.as_ref().is_none_or(|data| data.is_empty())
{
return false;
}
data_shards_seen_or_pending[data_slot] = true;
}
missing_pending_data_shards > 0 && data_shards_seen_or_pending.into_iter().all(|seen_or_pending| seen_or_pending)
} }
pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str { pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) -> &'static str {
@@ -1848,7 +1758,6 @@ pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum_all_shards(
} }
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum( pub(in crate::set_disk) async fn create_bitrot_readers_until_quorum(
files: &[FileInfo], files: &[FileInfo],
disks: &[Option<DiskStore>], disks: &[Option<DiskStore>],
@@ -2139,7 +2048,6 @@ pub(in crate::set_disk) async fn create_data_block_bitrot_readers(
setup setup
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(in crate::set_disk) async fn collect_read_multiple_results<F>( pub(in crate::set_disk) async fn collect_read_multiple_results<F>(
tasks: Vec<F>, tasks: Vec<F>,
read_quorum: usize, read_quorum: usize,
@@ -2456,7 +2364,6 @@ impl SetDisks {
let bucket: Arc<str> = Arc::from(bucket); let bucket: Arc<str> = Arc::from(bucket);
let object: Arc<str> = Arc::from(object); let object: Arc<str> = Arc::from(object);
let version_id: Arc<str> = Arc::from(version_id); let version_id: Arc<str> = Arc::from(version_id);
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
let futures = disks.iter().enumerate().map(|(disk_index, disk)| { let futures = disks.iter().enumerate().map(|(disk_index, disk)| {
let disk = disk.clone(); let disk = disk.clone();
let task_opts = opts; let task_opts = opts;
@@ -2464,14 +2371,10 @@ impl SetDisks {
let bucket = bucket.clone(); let bucket = bucket.clone();
let object = object.clone(); let object = object.clone();
let version_id = version_id.clone(); let version_id = version_id.clone();
let slowtail_fault = slowtail_fault.clone();
tokio::spawn(async move { tokio::spawn(async move {
let response_start = observe.then(Instant::now); let response_start = observe.then(Instant::now);
let result = if let Some(disk) = disk { let result = if let Some(disk) = disk {
Self::record_read_version_call(&object, disk_index); Self::record_read_version_call(&object, disk_index);
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
tokio::time::sleep(delay).await;
}
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts) disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
.await .await
} else { } else {
@@ -2566,8 +2469,6 @@ impl SetDisks {
let mut next_fanout_index = 0usize; let mut next_fanout_index = 0usize;
let mut scheduled_count = 0usize; let mut scheduled_count = 0usize;
let mut force_full_wait = false; let mut force_full_wait = false;
let mut final_miss_reason_override = None;
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
let spawn_read_version = let spawn_read_version =
|join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| { |join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| {
let task_opts = opts; let task_opts = opts;
@@ -2575,7 +2476,6 @@ impl SetDisks {
let bucket = bucket.clone(); let bucket = bucket.clone();
let object = object.clone(); let object = object.clone();
let version_id = version_id.clone(); let version_id = version_id.clone();
let slowtail_fault = slowtail_fault.clone();
join_set.spawn(async move { join_set.spawn(async move {
let response_start = Instant::now(); let response_start = Instant::now();
let result = if let Some(disk) = disk { let result = if let Some(disk) = disk {
@@ -2584,9 +2484,6 @@ impl SetDisks {
Self::record_read_version_call(&object, index); Self::record_read_version_call(&object, index);
#[cfg(test)] #[cfg(test)]
Self::read_version_fanout_barrier(&object, index).await; Self::read_version_fanout_barrier(&object, index).await;
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
tokio::time::sleep(delay).await;
}
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts) disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
.await .await
} else { } else {
@@ -2614,20 +2511,11 @@ impl SetDisks {
} }
while let Some(result) = join_set.join_next().await { while let Some(result) = join_set.join_next().await {
let mut defer_pending_inline_data_shard = false;
match result { match result {
Ok((index, res, elapsed)) => match res { Ok((index, res, elapsed)) => match res {
Ok(file_info) => { Ok(file_info) => {
observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed)); observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed));
accumulator.observe_file_info(&file_info); accumulator.observe_file_info(&file_info);
if bounded_fanout
&& read_data
&& !force_full_wait
&& let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(&file_info)
{
force_full_wait = true;
final_miss_reason_override.get_or_insert(reason);
}
if let Some(slot) = ress.get_mut(index) { if let Some(slot) = ress.get_mut(index) {
*slot = file_info; *slot = file_info;
} }
@@ -2653,43 +2541,17 @@ impl SetDisks {
.or_else(|| accumulator.version_early_stop_decision()) .or_else(|| accumulator.version_early_stop_decision())
{ {
let should_return_early = if read_data { let should_return_early = if read_data {
match accumulator.candidate.as_ref() { let allow_data_read_early_stop = match accumulator.candidate.as_ref() {
Some(candidate) => match data_read_early_stop_inline_body_miss_reason( Some(candidate) => {
bucket.as_ref(), data_read_early_stop_inline_body_verified(bucket.as_ref(), object.as_ref(), candidate, &ress, disks)
object.as_ref(),
candidate,
&ress,
disks,
)
.await .await
{ }
None => true, None => false,
Some(reason) => { };
final_miss_reason_override = Some(reason); if !allow_data_read_early_stop {
if bounded_fanout
&& reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD
&& data_read_inline_missing_shards_are_pending(
candidate,
&ress,
&errors,
disks,
&fanout_order,
next_fanout_index,
)
{
defer_pending_inline_data_shard = true;
} else {
force_full_wait = true; force_full_wait = true;
} }
false allow_data_read_early_stop
}
},
None => {
force_full_wait = true;
final_miss_reason_override = Some(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM);
false
}
}
} else { } else {
true true
}; };
@@ -2726,7 +2588,6 @@ impl SetDisks {
let pending_responses = join_set.len(); let pending_responses = join_set.len();
let should_hedge_single_pending_data_read = read_data let should_hedge_single_pending_data_read = read_data
&& !force_full_wait && !force_full_wait
&& !defer_pending_inline_data_shard
&& pending_responses == 1 && pending_responses == 1
&& accumulator.can_still_reach_early_stop_with_pending(pending_responses); && accumulator.can_still_reach_early_stop_with_pending(pending_responses);
if bounded_fanout && force_full_wait { if bounded_fanout && force_full_wait {
@@ -2739,7 +2600,6 @@ impl SetDisks {
next_fanout_index = next_fanout_index.saturating_add(1); next_fanout_index = next_fanout_index.saturating_add(1);
} }
} else if bounded_fanout } else if bounded_fanout
&& !defer_pending_inline_data_shard
&& next_fanout_index < disks.len() && next_fanout_index < disks.len()
&& (!accumulator.can_still_reach_early_stop_with_pending(pending_responses) && (!accumulator.can_still_reach_early_stop_with_pending(pending_responses)
|| should_hedge_single_pending_data_read) || should_hedge_single_pending_data_read)
@@ -2753,12 +2613,7 @@ impl SetDisks {
} }
} }
let accumulator_miss_reason = accumulator.final_miss_reason(); rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, accumulator.final_miss_reason());
let final_miss_reason = match (final_miss_reason_override, accumulator_miss_reason) {
(Some(reason), GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM) => reason,
_ => accumulator_miss_reason,
};
rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, final_miss_reason);
rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0); rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0);
rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0); rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0);
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations); let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
@@ -2987,7 +2842,6 @@ impl SetDisks {
(meta_file_infos, errs) (meta_file_infos, errs)
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(in crate::set_disk) async fn read_multiple_files( pub(in crate::set_disk) async fn read_multiple_files(
disks: &[Option<DiskStore>], disks: &[Option<DiskStore>],
req: ReadMultipleReq, req: ReadMultipleReq,
@@ -3021,11 +2875,14 @@ impl SetDisks {
}); });
} }
let (ress, _errors) = match collect_read_multiple_results(futures, read_quorum).await { let (ress, errors) = match collect_read_multiple_results(futures, read_quorum).await {
Ok(collected) => collected, Ok(collected) => collected,
Err(()) => return empty_quorum_result(), Err(()) => return empty_quorum_result(),
}; };
// debug!("ReadMultipleResp ress {:?}", ress);
// debug!("ReadMultipleResp errors {:?}", errors);
let mut ret = Vec::with_capacity(req.files.len()); let mut ret = Vec::with_capacity(req.files.len());
for want in req.files.iter() { for want in req.files.iter() {
@@ -3164,7 +3021,6 @@ pub(in crate::set_disk) struct RenameDataCommit {
pub(in crate::set_disk) committed_file_info: FileInfo, pub(in crate::set_disk) committed_file_info: FileInfo,
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
type RenameDataLegacyTuple = ( type RenameDataLegacyTuple = (
Vec<Option<DiskStore>>, Vec<Option<DiskStore>>,
RenameConvergence, RenameConvergence,
@@ -3174,7 +3030,6 @@ type RenameDataLegacyTuple = (
); );
impl RenameDataCommit { impl RenameDataCommit {
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn into_legacy_tuple(self) -> RenameDataLegacyTuple { fn into_legacy_tuple(self) -> RenameDataLegacyTuple {
( (
self.online_disks, self.online_disks,
@@ -3293,7 +3148,6 @@ impl SetDisks {
#[tracing::instrument(level = "debug", skip(disks, file_infos))] #[tracing::instrument(level = "debug", skip(disks, file_infos))]
#[allow(clippy::type_complexity)] #[allow(clippy::type_complexity)]
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(in crate::set_disk) async fn rename_data( pub(in crate::set_disk) async fn rename_data(
disks: &[Option<DiskStore>], disks: &[Option<DiskStore>],
src_bucket: &str, src_bucket: &str,
@@ -5106,7 +4960,6 @@ fn is_cleanup_not_found(e: &DiskError) -> bool {
/// normalized to `DiskNotFound`: a panic is not a "disk absent" condition and /// normalized to `DiskNotFound`: a panic is not a "disk absent" condition and
/// must not be silently swallowed as an ignorable error (fixes the historical /// must not be silently swallowed as an ignorable error (fixes the historical
/// `Unexpected`/`DiskNotFound` misclassification). /// `Unexpected`/`DiskNotFound` misclassification).
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn map_cleanup_join_result(joined: std::result::Result<Option<DiskError>, tokio::task::JoinError>) -> Option<DiskError> { fn map_cleanup_join_result(joined: std::result::Result<Option<DiskError>, tokio::task::JoinError>) -> Option<DiskError> {
match joined { match joined {
Ok(res) => res, Ok(res) => res,
@@ -5331,7 +5184,6 @@ pub(in crate::set_disk) mod rename_fanout_barrier_phase {
/// The per-disk old-data-dir cleanup phase of the commit fan-out. /// The per-disk old-data-dir cleanup phase of the commit fan-out.
pub const CLEANUP: &str = "cleanup"; pub const CLEANUP: &str = "cleanup";
/// The per-disk `read_version` phase of metadata read fan-out. /// The per-disk `read_version` phase of metadata read fan-out.
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub const READ_VERSION: &str = "read_version"; pub const READ_VERSION: &str = "read_version";
} }
@@ -5769,130 +5621,6 @@ mod tests {
(dirs, disks) (dirs, disks)
} }
#[test]
fn metadata_slowtail_fault_delay_parses_and_filters_request() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,3")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some("bench-bucket")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
],
|| {
assert_eq!(
get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, true),
Some(Duration::from_millis(25))
);
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 2, true).is_none());
assert!(get_metadata_slowtail_fault_delay("other-bucket", "objects/000001", 3, true).is_none());
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "other/000001", 3, true).is_none());
assert!(get_metadata_slowtail_fault_delay("bench-bucket", "objects/000001", 3, false).is_none());
},
);
}
#[test]
fn metadata_slowtail_fault_delay_disables_invalid_disk_list() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("25")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("1,nope")),
],
|| {
assert!(get_metadata_slowtail_fault_delay("bucket", "object", 1, true).is_none());
},
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn metadata_slowtail_fault_delays_only_data_read_metadata_task() {
const DISKS: usize = 4;
let bucket = "metadata-slowtail-fault-bucket";
let object = "objects/metadata-slowtail-fault-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
temp_env::async_with_vars(
[
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("false")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
],
async {
let read_without_data =
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", false, false, false, true, 2);
tokio::time::timeout(Duration::from_millis(100), read_without_data)
.await
.expect("non-data metadata fanout must not be delayed by the data-read slowtail hook")
.expect("metadata fanout without read_data should resolve");
let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed(
&disks, bucket, bucket, object, "", true, false, false, true, 2,
));
assert!(
tokio::time::timeout(Duration::from_millis(40), &mut read_with_data)
.await
.is_err(),
"data-read metadata fanout must wait for the injected slow read_version response"
);
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data)
.await
.expect("injected slowtail should eventually complete")
.expect("data-read metadata fanout should resolve");
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
assert_eq!(diagnostics.total_responses(), DISKS);
},
)
.await;
drop(dirs);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn metadata_slowtail_fault_delays_early_stop_metadata_task() {
const DISKS: usize = 4;
let bucket = "metadata-slowtail-early-stop-bucket";
let object = "objects/metadata-slowtail-early-stop-object";
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
temp_env::async_with_vars(
[
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("true")),
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some("3")),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
],
async {
let mut read_with_data = Box::pin(SetDisks::read_all_fileinfo_observed(
&disks, bucket, bucket, object, "", true, false, false, true, 2,
));
assert!(
tokio::time::timeout(Duration::from_millis(40), &mut read_with_data)
.await
.is_err(),
"early-stop metadata fanout must still wait for the injected slow response after fallback to full wait"
);
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_secs(2), read_with_data)
.await
.expect("injected early-stop slowtail should eventually complete")
.expect("early-stop metadata fanout should resolve");
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
assert!(errs.iter().all(Option::is_none));
assert_eq!(diagnostics.total_responses(), DISKS);
},
)
.await;
drop(dirs);
}
/// Demo / regression guard for the backlog#1325 per-disk call counters. /// Demo / regression guard for the backlog#1325 per-disk call counters.
/// ///
/// The metadata fan-out issues each `read_version` inside its own /// The metadata fan-out issues each `read_version` inside its own
@@ -6024,20 +5752,9 @@ mod tests {
object: &str, object: &str,
payload: &[u8], payload: &[u8],
uses_legacy_checksum: bool, uses_legacy_checksum: bool,
) -> Vec<FileInfo> {
inline_metadata_fanout_fileinfos_with_geometry(bucket, object, payload, uses_legacy_checksum, 2, 2).await
}
async fn inline_metadata_fanout_fileinfos_with_geometry(
bucket: &str,
object: &str,
payload: &[u8],
uses_legacy_checksum: bool,
data_shards: usize,
parity_shards: usize,
) -> Vec<FileInfo> { ) -> Vec<FileInfo> {
let distribution_key = metadata_distribution_key(bucket, object); let distribution_key = metadata_distribution_key(bucket, object);
let mut base = FileInfo::new(&distribution_key, data_shards, parity_shards); let mut base = FileInfo::new(&distribution_key, 2, 2);
base.volume = bucket.to_string(); base.volume = bucket.to_string();
base.name = object.to_string(); base.name = object.to_string();
base.size = i64::try_from(payload.len()).expect("test payload should fit i64"); base.size = i64::try_from(payload.len()).expect("test payload should fit i64");
@@ -6100,21 +5817,6 @@ mod tests {
install_inline_metadata_fanout_files(disks, bucket, object, files).await; install_inline_metadata_fanout_files(disks, bucket, object, files).await;
} }
async fn install_inline_metadata_fanout_fileinfo_with_geometry(
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
payload: &[u8],
data_shards: usize,
parity_shards: usize,
mutate: impl FnOnce(&mut [FileInfo]),
) {
let mut files =
inline_metadata_fanout_fileinfos_with_geometry(bucket, object, payload, false, data_shards, parity_shards).await;
mutate(&mut files);
install_inline_metadata_fanout_files(disks, bucket, object, files).await;
}
async fn install_inline_metadata_fanout_files(disks: &[Option<DiskStore>], bucket: &str, object: &str, files: Vec<FileInfo>) { async fn install_inline_metadata_fanout_files(disks: &[Option<DiskStore>], bucket: &str, object: &str, files: Vec<FileInfo>) {
let distribution = files let distribution = files
.first() .first()
@@ -6335,118 +6037,6 @@ mod tests {
drop(dirs); drop(dirs);
} }
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn bounded_metadata_early_stop_waits_for_pending_inline_data_shard() {
const DISKS: usize = 6;
const DATA_SHARDS: usize = 4;
const PARITY_SHARDS: usize = 2;
let bucket = "bounded-inline-data-get-pending-shard-bucket";
let object =
object_with_initial_data_shards(bucket, "bounded-inline-data-get-pending-shard-object", DATA_SHARDS, DATA_SHARDS);
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
install_inline_metadata_fanout_fileinfo_with_geometry(
&disks,
bucket,
&object,
b"verified inline payload",
DATA_SHARDS,
PARITY_SHARDS,
|_| {},
)
.await;
temp_env::async_with_vars(
[
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
],
async {
let fanout_order = bounded_metadata_fanout_order(bucket, &object, DISKS, PARITY_SHARDS);
let distribution_key = metadata_distribution_key(bucket, &object);
let distribution = FileInfo::new(&distribution_key, DATA_SHARDS, PARITY_SHARDS)
.erasure
.distribution;
let paused_data_disk = *fanout_order
.iter()
.take(DATA_SHARDS)
.find(|disk_index| {
distribution
.get(**disk_index)
.is_some_and(|block_index| (1..=DATA_SHARDS).contains(block_index))
})
.expect("initial fanout should include a data shard to pause");
let hedged_parity_disk = fanout_order[DATA_SHARDS];
let unscheduled_parity_disk = fanout_order[DATA_SHARDS + 1];
let barrier = rename_fanout_barrier::arm(&object, paused_data_disk, rename_fanout_barrier::PHASE_READ_VERSION);
let tracker = rename_fanout_barrier::observe_tasks(&object);
let calls = disk_call_counters::observe(&object);
let disks_for_read = disks.clone();
let object_for_read = object.clone();
let mut read = tokio::spawn(async move {
SetDisks::read_all_fileinfo_observed(
&disks_for_read,
bucket,
bucket,
&object_for_read,
"",
true,
false,
false,
true,
PARITY_SHARDS,
)
.await
});
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("initial data shard should pause before returning");
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
while calls.for_disk(disk_call_counters::KIND_READ_VERSION, hedged_parity_disk) == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("bounded fanout should hedge one parity disk while the data shard is pending");
assert!(
tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await.is_err(),
"inline data-read early-stop must wait for a scheduled missing data shard instead of forcing full wait"
);
barrier.release();
let (parts_metadata, errs, diagnostics) = read
.await
.expect("metadata read task should not panic")
.expect("pending data shard should let the inline verifier finish");
assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION),
5,
"pending data-shard defer should not schedule the final parity disk"
);
assert_eq!(
calls.for_disk(disk_call_counters::KIND_READ_VERSION, unscheduled_parity_disk),
0,
"the remaining parity disk must stay unissued when pending data verification succeeds"
);
assert_eq!(
tracker.running(),
0,
"early-stop should drain spawned read_version tasks before returning"
);
assert_eq!(diagnostics.total_responses(), 5);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), 5);
assert!(errs.iter().all(Option::is_none));
},
)
.await;
drop(dirs);
}
#[tokio::test] #[tokio::test]
async fn data_read_early_stop_verifies_legacy_inline_checksum_payload() { async fn data_read_early_stop_verifies_legacy_inline_checksum_payload() {
let bucket = "legacy-inline-data-get-fanout-bucket"; let bucket = "legacy-inline-data-get-fanout-bucket";
@@ -6477,133 +6067,11 @@ mod tests {
.clone(); .clone();
assert!( assert!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &parts_metadata, &disks) data_read_early_stop_inline_body_verified(bucket, object, &candidate, &parts_metadata, &disks).await,
.await
.is_none(),
"legacy inline metadata must use the legacy bitrot shard sizing and checksum algorithm" "legacy inline metadata must use the legacy bitrot shard sizing and checksum algorithm"
); );
} }
#[tokio::test]
async fn data_read_early_stop_reports_inline_miss_reasons() {
let bucket = "inline-data-get-miss-reason-bucket";
let object = "inline-data-get-miss-reason-object";
let payload = b"verified inline payload";
let (_dirs, disks) = call_counter_local_disks(bucket, 4).await;
let files = inline_metadata_fanout_fileinfos_with_mode(bucket, object, payload, false).await;
let distribution = files
.first()
.map(|file| file.erasure.distribution.clone())
.expect("fixture should include metadata");
let order = bounded_metadata_fanout_order(bucket, object, 4, 2);
let mut parts_metadata = vec![FileInfo::default(); 4];
for disk_index in order.into_iter().take(3) {
let block_index = distribution
.get(disk_index)
.copied()
.expect("fixture distribution should cover every disk");
parts_metadata[disk_index] = files
.get(block_index.checked_sub(1).expect("erasure block indexes are one-based"))
.expect("fixture should include every distributed shard")
.clone();
}
let candidate = parts_metadata
.iter()
.find(|file| file.name == object)
.expect("fixture should include observed metadata")
.clone();
let data_disk = distribution
.iter()
.position(|block_index| *block_index == 1)
.expect("fixture distribution should include first data shard");
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &parts_metadata, &disks).await,
None
);
let mut not_inline = candidate.clone();
rustfs_utils::http::remove_str(&mut not_inline.metadata, rustfs_utils::http::SUFFIX_INLINE_DATA);
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &not_inline, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE)
);
let mut remote = candidate.clone();
remote.transition_status = TRANSITION_COMPLETE.to_string();
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &remote, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE)
);
let mut transformed = candidate.clone();
rustfs_utils::http::insert_str(&mut transformed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &transformed, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED)
);
let mut deleted = candidate.clone();
deleted.deleted = true;
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &deleted, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED)
);
let mut zero_size = candidate.clone();
zero_size.size = 0;
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &zero_size, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE)
);
let mut multipart = candidate.clone();
multipart.parts.push(multipart.parts[0].clone());
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &multipart, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE)
);
let mut invalid_geometry = candidate.clone();
invalid_geometry.erasure.data_blocks = 0;
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &invalid_geometry, &parts_metadata, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY)
);
let mut missing_shard = parts_metadata.clone();
missing_shard[data_disk] = FileInfo::default();
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &missing_shard, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD)
);
let mut missing_payload = parts_metadata.clone();
missing_payload[data_disk].data = None;
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &missing_payload, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD)
);
let mut identity_mismatch = parts_metadata.clone();
identity_mismatch[data_disk].version_id = Some(Uuid::new_v4());
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &identity_mismatch, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH)
);
let mut corrupt = parts_metadata.clone();
if let Some(data) = corrupt[data_disk].data.as_mut() {
let mut corrupt_data = data.to_vec();
corrupt_data[0] ^= 0x01;
*data = Bytes::from(corrupt_data);
}
assert_eq!(
data_read_early_stop_inline_body_miss_reason(bucket, object, &candidate, &corrupt, &disks).await,
Some(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY)
);
}
#[test] #[test]
#[serial_test::serial] #[serial_test::serial]
fn metadata_fanout_lifecycle_records_real_early_stop_abort() { fn metadata_fanout_lifecycle_records_real_early_stop_abort() {
@@ -6693,7 +6161,7 @@ mod tests {
&[ &[
("path", GET_OBJECT_PATH_INTERNAL_META), ("path", GET_OBJECT_PATH_INTERNAL_META),
("decision", "miss"), ("decision", "miss"),
("reason", GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY), ("reason", GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM),
], ],
), ),
1, 1,
@@ -6705,7 +6173,7 @@ mod tests {
&[ &[
("path", GET_OBJECT_PATH_LEGACY_DUPLEX), ("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
("decision", "miss"), ("decision", "miss"),
("reason", GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY), ("reason", GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM),
], ],
), ),
0, 0,
@@ -7240,7 +6708,7 @@ mod tests {
} }
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn bounded_non_inline_data_get_immediately_forces_full_fanout() { async fn bounded_non_inline_data_get_hedges_then_waits_for_full_fanout() {
const DISKS: usize = 4; const DISKS: usize = 4;
let bucket = "bounded-data-get-hedge-bucket"; let bucket = "bounded-data-get-hedge-bucket";
let object = "bounded-data-get-hedge-object"; let object = "bounded-data-get-hedge-object";
@@ -7271,7 +6739,7 @@ mod tests {
} }
}) })
.await .await
.expect("bounded non-inline data-read fanout should immediately schedule the spare disk"); .expect("bounded data-read fanout should hedge by starting the spare disk");
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await; let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
assert!( assert!(
@@ -7287,7 +6755,7 @@ mod tests {
assert_eq!( assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION), calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64, DISKS as u64,
"bounded non-inline data-read fanout should issue the paused disk plus the remaining spare" "bounded data-read fanout should issue the paused disk plus one spare hedge"
); );
assert_eq!(diagnostics.total_responses(), DISKS); assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
@@ -7300,7 +6768,7 @@ mod tests {
} }
#[tokio::test] #[tokio::test]
async fn bounded_metadata_early_stop_defaults_keep_non_inline_data_get_full_fanout() { async fn bounded_metadata_early_stop_defaults_keep_data_get_full_fanout() {
const DISKS: usize = 4; const DISKS: usize = 4;
let bucket = "bounded-data-get-default-bucket"; let bucket = "bounded-data-get-default-bucket";
let object = "bounded-data-get-default-object"; let object = "bounded-data-get-default-object";
@@ -7314,42 +6782,16 @@ mod tests {
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>), ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>),
], ],
async { async {
let barrier = rename_fanout_barrier::arm(object, 2, rename_fanout_barrier::PHASE_READ_VERSION);
let calls = disk_call_counters::observe(object); let calls = disk_call_counters::observe(object);
let disks_for_read = disks.clone(); let (parts_metadata, errs, diagnostics) =
let mut read = tokio::spawn(async move { SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2)
SetDisks::read_all_fileinfo_observed(&disks_for_read, bucket, bucket, object, "", true, false, false, true, 2)
.await .await
});
tokio::time::timeout(BARRIER_PAUSE_GUARD, barrier.wait_until_paused())
.await
.expect("default bounded non-inline read should schedule the paused metadata task");
tokio::time::timeout(BARRIER_PAUSE_GUARD, async {
while calls.for_disk(disk_call_counters::KIND_READ_VERSION, 3) == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect(
"default bounded non-inline read should immediately force full fanout after the first non-inline response",
);
let pending = tokio::time::timeout(BARRIER_PAUSE_GUARD, &mut read).await;
assert!(
pending.is_err(),
"default non-inline data reads must not return before the paused metadata response"
);
barrier.release();
let (parts_metadata, errs, diagnostics) = read
.await
.expect("metadata read task should not panic")
.expect("default data-read metadata should resolve"); .expect("default data-read metadata should resolve");
assert_eq!( assert_eq!(
calls.total(disk_call_counters::KIND_READ_VERSION), calls.total(disk_call_counters::KIND_READ_VERSION),
DISKS as u64, DISKS as u64,
"default non-inline GET data-read metadata must keep full fanout without waiting for a quorum miss first" "default GET data-read metadata must keep full fanout for read-failure tolerance"
); );
assert_eq!(diagnostics.total_responses(), DISKS); assert_eq!(diagnostics.total_responses(), DISKS);
assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS); assert_eq!(parts_metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
-28
View File
@@ -42,20 +42,12 @@ impl<'a> SetDisksCtx<'a> {
} }
/// The borrowed core, for state not yet fronted by a typed accessor. /// The borrowed core, for state not yet fronted by a typed accessor.
#[allow(
dead_code,
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
)]
pub(crate) fn core(&self) -> &'a SetDisks { pub(crate) fn core(&self) -> &'a SetDisks {
self.core self.core
} }
// --- Immutable topology / config (fixed after construction) --- // --- Immutable topology / config (fixed after construction) ---
#[allow(
dead_code,
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
)]
pub(crate) fn set_index(&self) -> usize { pub(crate) fn set_index(&self) -> usize {
self.core.set_index self.core.set_index
} }
@@ -64,26 +56,14 @@ impl<'a> SetDisksCtx<'a> {
self.core.pool_index self.core.pool_index
} }
#[allow(
dead_code,
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
)]
pub(crate) fn set_drive_count(&self) -> usize { pub(crate) fn set_drive_count(&self) -> usize {
self.core.set_drive_count self.core.set_drive_count
} }
#[allow(
dead_code,
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
)]
pub(crate) fn default_parity_count(&self) -> usize { pub(crate) fn default_parity_count(&self) -> usize {
self.core.default_parity_count self.core.default_parity_count
} }
#[allow(
dead_code,
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
)]
pub(crate) fn set_endpoints(&self) -> &'a [Endpoint] { pub(crate) fn set_endpoints(&self) -> &'a [Endpoint] {
&self.core.set_endpoints &self.core.set_endpoints
} }
@@ -92,10 +72,6 @@ impl<'a> SetDisksCtx<'a> {
&self.core.format &self.core.format
} }
#[allow(
dead_code,
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
)]
pub(crate) fn locker_owner(&self) -> &'a str { pub(crate) fn locker_owner(&self) -> &'a str {
&self.core.locker_owner &self.core.locker_owner
} }
@@ -108,10 +84,6 @@ impl<'a> SetDisksCtx<'a> {
// --- Locker trio --- // --- Locker trio ---
#[allow(
dead_code,
reason = "SetDisks split seam (backlog#815) with no caller in this port (backlog#1823)"
)]
pub(crate) fn lockers(&self) -> &'a [Arc<dyn LockClient>] { pub(crate) fn lockers(&self) -> &'a [Arc<dyn LockClient>] {
&self.core.lockers &self.core.lockers
} }
+48 -177
View File
@@ -39,6 +39,7 @@
//! - `metadata.rs`, `replication.rs`, `shard_source.rs` — supporting helpers. //! - `metadata.rs`, `replication.rs`, `shard_source.rs` — supporting helpers.
// #730: SetDisks still hosts staged read/heal/write migration helpers. // #730: SetDisks still hosts staged read/heal/write migration helpers.
#![allow(dead_code)]
#![allow(unused_imports)] #![allow(unused_imports)]
#![allow(unused_variables)] #![allow(unused_variables)]
@@ -58,10 +59,7 @@ use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl};
use crate::cluster::rpc::heal_bucket_local_on_disks; use crate::cluster::rpc::heal_bucket_local_on_disks;
use crate::data_usage::record_compression_total_memory; use crate::data_usage::record_compression_total_memory;
use crate::diagnostics::get::{ use crate::diagnostics::get::{
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING,
GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE, GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE, GET_OBJECT_PATH_DIRECT_MEMORY,
GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX,
GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_INLINE_PREPARE, GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_STAGE_INLINE_PREPARE,
@@ -102,7 +100,9 @@ use crate::storage_api_contracts::{
}; };
use crate::store::utils::is_reserved_or_invalid_bucket; use crate::store::utils::is_reserved_or_invalid_bucket;
use crate::{ use crate::{
bucket::lifecycle::bucket_lifecycle_ops::{LifecycleOps, get_transitioned_object_reader_with_tier_manager, put_restore_opts}, bucket::lifecycle::bucket_lifecycle_ops::{
LifecycleOps, gen_transition_objname, get_transitioned_object_reader_with_tier_manager, put_restore_opts,
},
cache_value::metacache_set::{ListPathRawOptions, list_path_raw}, cache_value::metacache_set::{ListPathRawOptions, list_path_raw},
config::storageclass, config::storageclass,
disk::{ disk::{
@@ -174,14 +174,15 @@ use std::future::Future;
use std::hash::{BuildHasher, Hash, Hasher}; use std::hash::{BuildHasher, Hash, Hasher};
use std::mem::{self}; use std::mem::{self};
use std::pin::Pin; use std::pin::Pin;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, OnceLock};
use std::task::{Context, Poll}; use std::task::{Context, Poll};
use std::time::{Instant, SystemTime, UNIX_EPOCH}; use std::time::{Instant, SystemTime, UNIX_EPOCH};
use std::{ use std::{
collections::{HashMap, HashSet}, collections::{HashMap, HashSet},
io::{Cursor, Write}, io::{Cursor, Write},
path::Path, path::Path,
sync::Arc,
time::Duration, time::Duration,
}; };
use time::OffsetDateTime; use time::OffsetDateTime;
@@ -620,9 +621,7 @@ fn adaptive_duplex_buffer_size(object_size: i64) -> usize {
// Each flag has a corresponding `*_ROLLOUT_PCT` for percentage-based gradual rollout. // Each flag has a corresponding `*_ROLLOUT_PCT` for percentage-based gradual rollout.
// ============================================================================ // ============================================================================
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1); const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1);
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750); const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750);
const GET_OBJECT_METADATA_CACHE_TTL: Duration = Duration::from_secs(2); // Increased from 250ms to 2s const GET_OBJECT_METADATA_CACHE_TTL: Duration = Duration::from_secs(2); // Increased from 250ms to 2s
const DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: usize = 4096; // Increased from 1024 to 4096 const DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES: usize = 4096; // Increased from 1024 to 4096
@@ -690,36 +689,22 @@ const DEFAULT_RUSTFS_GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD: usize = 128 * 102
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ENABLE"; const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ENABLE";
// Enabled by default (backlog#872): the early-stop path only engages for // Enabled by default (backlog#872): the early-stop path only engages for
// requests `should_allow_metadata_early_stop` classifies as safe (latest-version // requests `should_allow_metadata_early_stop` classifies as safe (latest-version
// reads by default, without version_id / healing / free-version needs) and still // metadata-only reads by default, without version_id / healing / free-version
// requires a full read-quorum agreement before stopping. Data-read requests add // needs) and still requires a full read-quorum agreement before stopping. Set
// a separate inline-shard verifier before cancelling the remaining fanout. Set
// the env var to `false` to fall back to full-wait metadata fanout. // the env var to `false` to fall back to full-wait metadata fanout.
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: bool = true; const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE: bool = true;
#[allow(
dead_code,
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
)]
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT"; const ENV_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT";
#[allow(
dead_code,
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
)]
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: u32 = 100; const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_ROLLOUT_PCT: u32 = 100;
const ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE"; const ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE";
const DEFAULT_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: bool = false; const DEFAULT_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: bool = false;
const ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE"; const ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE";
const DEFAULT_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: bool = true; const DEFAULT_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: bool = false;
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT"; const ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT";
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = true; const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = false;
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS";
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS";
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET";
const ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX: &str = "RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX";
// --- Multipart Reader-Setup Prefetch Configuration (backlog#870) --- // --- Multipart Reader-Setup Prefetch Configuration (backlog#870) ---
@@ -921,16 +906,18 @@ mod prepared_get_object_metadata_tests {
.expect("test should find an object whose initial fanout covers both data shards") .expect("test should find an object whose initial fanout covers both data shards")
} }
#[allow(
dead_code,
reason = "test fixture no assertion in this module uses today; the live namesake lives in io_primitives tests (backlog#1823)"
)]
fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize { fn bounded_spare_disk_index(bucket: &str, object: &str) -> usize {
*bounded_metadata_fanout_order(bucket, object, 4, 2) *bounded_metadata_fanout_order(bucket, object, 4, 2)
.get(3) .get(3)
.expect("4-disk test geometry should leave one bounded spare disk") .expect("4-disk test geometry should leave one bounded spare disk")
} }
fn bounded_slow_initial_disk_index(bucket: &str, object: &str) -> usize {
*bounded_metadata_fanout_order(bucket, object, 4, 2)
.get(2)
.expect("4-disk test geometry should include a third initial metadata disk")
}
#[tokio::test] #[tokio::test]
async fn prepared_metadata_is_consumed_exactly_once() { async fn prepared_metadata_is_consumed_exactly_once() {
let snapshot = GetObjectFileInfo::owned(FileInfo::default(), Vec::new(), Vec::new()); let snapshot = GetObjectFileInfo::owned(FileInfo::default(), Vec::new(), Vec::new());
@@ -1049,7 +1036,7 @@ mod prepared_get_object_metadata_tests {
#[test] #[test]
#[serial_test::serial(body_cache_hook)] #[serial_test::serial(body_cache_hook)]
fn inline_data_read_early_stop_defaults_return_exact_body() { fn inline_data_read_early_stop_reader_returns_exact_body() {
let runtime = tokio::runtime::Builder::new_current_thread() let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all() .enable_all()
.build() .build()
@@ -1081,14 +1068,14 @@ mod prepared_get_object_metadata_tests {
temp_env::async_with_vars( temp_env::async_with_vars(
[ [
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", None::<&str>), ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None::<&str>), ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>), ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
], ],
async { async {
let slow_parity_disk = bounded_spare_disk_index(bucket, &object); let slow_initial_disk = bounded_slow_initial_disk_index(bucket, &object);
let barrier = let barrier =
rename_fanout_barrier::arm(&object, slow_parity_disk, rename_fanout_barrier::PHASE_READ_VERSION); rename_fanout_barrier::arm(&object, slow_initial_disk, rename_fanout_barrier::PHASE_READ_VERSION);
let calls = disk_call_counters::observe(&object); let calls = disk_call_counters::observe(&object);
let set_disks_for_read = Arc::clone(&set_disks); let set_disks_for_read = Arc::clone(&set_disks);
let opts_for_read = opts.clone(); let opts_for_read = opts.clone();
@@ -1101,10 +1088,10 @@ mod prepared_get_object_metadata_tests {
tokio::time::timeout(READ_VERSION_BARRIER_GUARD, barrier.wait_until_paused()) tokio::time::timeout(READ_VERSION_BARRIER_GUARD, barrier.wait_until_paused())
.await .await
.expect("default inline GET should pause a slow parity metadata read"); .expect("bounded inline GET should pause a slow initial metadata read");
let mut reader = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut open_reader) let mut reader = tokio::time::timeout(READ_VERSION_BARRIER_GUARD, &mut open_reader)
.await .await
.expect("default production inline GET should return before the paused parity metadata response") .expect("production inline GET should return before the paused metadata response")
.expect("inline GET reader task should not panic") .expect("inline GET reader task should not panic")
.expect("inline GET reader should open"); .expect("inline GET reader should open");
let object_size = reader.object_info.size; let object_size = reader.object_info.size;
@@ -1125,17 +1112,14 @@ mod prepared_get_object_metadata_tests {
assert_eq!(object_size, payload.len() as i64); assert_eq!(object_size, payload.len() as i64);
assert_eq!(restored, payload); assert_eq!(restored, payload);
assert_eq!( assert_eq!(calls_total, 4, "bounded production GET should schedule the initial quorum plus one spare");
calls_total, 4,
"default production inline GET should schedule the initial bounded quorum plus one hedge"
);
assert_eq!( assert_eq!(
recorder.histogram_values( recorder.histogram_values(
"rustfs_io_get_object_metadata_fanout_scheduled", "rustfs_io_get_object_metadata_fanout_scheduled",
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
), ),
vec![4.0], vec![4.0],
"default production GET should record all scheduled metadata tasks" "bounded production GET should record all scheduled metadata tasks"
); );
assert_eq!( assert_eq!(
recorder.histogram_values( recorder.histogram_values(
@@ -1143,7 +1127,7 @@ mod prepared_get_object_metadata_tests {
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
), ),
vec![3.0], vec![3.0],
"default production GET should record only observed metadata responses as completed" "bounded production GET should record only observed metadata responses as completed"
); );
assert_eq!( assert_eq!(
recorder.histogram_values( recorder.histogram_values(
@@ -1151,7 +1135,7 @@ mod prepared_get_object_metadata_tests {
&[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)] &[("path", GET_OBJECT_PATH_LEGACY_DUPLEX)]
), ),
vec![1.0], vec![1.0],
"default production GET should record the aborted slow parity metadata task" "bounded production GET should record the aborted slow metadata task"
); );
} }
@@ -1298,9 +1282,9 @@ mod prepared_get_object_metadata_tests {
temp_env::async_with_vars( temp_env::async_with_vars(
[ [
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", None::<&str>), ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", None::<&str>), ("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", None::<&str>), ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
], ],
async { async {
let calls = disk_call_counters::observe(&object); let calls = disk_call_counters::observe(&object);
@@ -1702,95 +1686,6 @@ fn is_get_metadata_early_stop_bounded_fanout_enabled() -> bool {
} }
} }
#[derive(Debug)]
struct GetMetadataSlowtailFaultConfig {
delay: Duration,
disks: Arc<[usize]>,
bucket: Option<String>,
object_prefix: Option<String>,
}
#[derive(Clone, Debug)]
struct GetMetadataSlowtailFaultRequest {
delay: Duration,
disks: Arc<[usize]>,
}
impl GetMetadataSlowtailFaultRequest {
fn delay_for_disk(&self, disk_index: usize) -> Option<Duration> {
self.disks.contains(&disk_index).then_some(self.delay)
}
}
fn parse_get_metadata_slowtail_fault_disks(raw: &str) -> Option<Vec<usize>> {
let mut disks = Vec::new();
for item in raw.split(',').map(str::trim).filter(|item| !item.is_empty()) {
let Ok(index) = item.parse::<usize>() else {
return None;
};
if !disks.contains(&index) {
disks.push(index);
}
}
(!disks.is_empty()).then_some(disks)
}
fn load_get_metadata_slowtail_fault_config() -> Option<GetMetadataSlowtailFaultConfig> {
let delay_ms = rustfs_utils::get_env_u64(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, 0);
if delay_ms == 0 {
return None;
}
let disks = parse_get_metadata_slowtail_fault_disks(&std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS).ok()?)?;
let bucket = std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET)
.ok()
.filter(|value| !value.is_empty());
let object_prefix = std::env::var(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX)
.ok()
.filter(|value| !value.is_empty());
Some(GetMetadataSlowtailFaultConfig {
delay: Duration::from_millis(delay_ms),
disks: Arc::from(disks.into_boxed_slice()),
bucket,
object_prefix,
})
}
fn get_metadata_slowtail_fault_request(bucket: &str, object: &str, read_data: bool) -> Option<GetMetadataSlowtailFaultRequest> {
if !read_data {
return None;
}
#[cfg(test)]
let config = load_get_metadata_slowtail_fault_config();
#[cfg(test)]
let config = config.as_ref()?;
#[cfg(not(test))]
let config = ({
static CACHED: OnceLock<Option<GetMetadataSlowtailFaultConfig>> = OnceLock::new();
CACHED.get_or_init(load_get_metadata_slowtail_fault_config).as_ref()
})?;
if let Some(expected_bucket) = &config.bucket
&& expected_bucket != bucket
{
return None;
}
if let Some(expected_prefix) = &config.object_prefix
&& !object.starts_with(expected_prefix)
{
return None;
}
Some(GetMetadataSlowtailFaultRequest {
delay: config.delay,
disks: config.disks.clone(),
})
}
#[cfg(test)]
fn get_metadata_slowtail_fault_delay(bucket: &str, object: &str, disk_index: usize, read_data: bool) -> Option<Duration> {
get_metadata_slowtail_fault_request(bucket, object, read_data)?.delay_for_disk(disk_index)
}
/// Check if multipart reads prefetch the next part's bitrot reader setup /// Check if multipart reads prefetch the next part's bitrot reader setup
/// while the current part decodes (backlog#870). /// while the current part decodes (backlog#870).
/// ///
@@ -1816,10 +1711,6 @@ fn is_multipart_reader_setup_prefetch_enabled() -> bool {
} }
} }
#[allow(
dead_code,
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
)]
fn get_metadata_early_stop_rollout_pct() -> u32 { fn get_metadata_early_stop_rollout_pct() -> u32 {
static CACHED: OnceLock<u32> = OnceLock::new(); static CACHED: OnceLock<u32> = OnceLock::new();
*CACHED.get_or_init(|| { *CACHED.get_or_init(|| {
@@ -1859,10 +1750,6 @@ fn should_use_codec_streaming(config: GetCodecStreamingConfig, bucket: &str, obj
} }
/// Should this specific request use metadata early-stop? /// Should this specific request use metadata early-stop?
#[allow(
dead_code,
reason = "percentage-rollout facet of the metadata early-stop switch; its predicate has no caller while the sibling enable flag is live (backlog#1823)"
)]
pub fn should_use_metadata_early_stop(bucket: &str, object: &str) -> bool { pub fn should_use_metadata_early_stop(bucket: &str, object: &str) -> bool {
let base = is_get_metadata_early_stop_enabled(); let base = is_get_metadata_early_stop_enabled();
let pct = get_metadata_early_stop_rollout_pct(); let pct = get_metadata_early_stop_rollout_pct();
@@ -2296,7 +2183,6 @@ fn classify_get_codec_streaming_object_class(
GetCodecStreamingObjectClass::PlainSinglePart GetCodecStreamingObjectClass::PlainSinglePart
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn is_get_small_object_direct_memory_eligible_with_threshold( fn is_get_small_object_direct_memory_eligible_with_threshold(
range: &Option<HTTPRangeSpec>, range: &Option<HTTPRangeSpec>,
object_info: &ObjectInfo, object_info: &ObjectInfo,
@@ -2902,7 +2788,6 @@ pub struct SetDisks {
/// Stable namespace shared by every object lock created for this set. /// Stable namespace shared by every object lock created for this set.
set_lock_namespace: Arc<str>, set_lock_namespace: Arc<str>,
pub format: FormatV3, pub format: FormatV3,
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>, disk_health_cache: Arc<RwLock<Vec<Option<DiskHealthEntry>>>>,
get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>, get_object_metadata_cache: moka::future::Cache<GetObjectMetadataCacheKey, Arc<GetObjectMetadataCacheEntry>>,
get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState, get_object_metadata_cache_hash_builder: std::collections::hash_map::RandomState,
@@ -3178,13 +3063,11 @@ struct GetObjectMetadataCacheEntry {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
struct DiskHealthEntry { struct DiskHealthEntry {
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
last_check: Instant, last_check: Instant,
online: bool, online: bool,
} }
impl DiskHealthEntry { impl DiskHealthEntry {
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn cached_value(&self) -> Option<bool> { fn cached_value(&self) -> Option<bool> {
if self.last_check.elapsed() <= DISK_HEALTH_CACHE_TTL { if self.last_check.elapsed() <= DISK_HEALTH_CACHE_TTL {
Some(self.online) Some(self.online)
@@ -3778,7 +3661,6 @@ fn multipart_put_large_batch_min_size_bytes() -> usize {
}) })
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn classify_small_write_path(is_inline_buffer: bool, object_size: i64, block_size: usize) -> SmallWritePath { fn classify_small_write_path(is_inline_buffer: bool, object_size: i64, block_size: usize) -> SmallWritePath {
if should_use_inline_small_fast_path(is_inline_buffer, object_size, block_size) { if should_use_inline_small_fast_path(is_inline_buffer, object_size, block_size) {
SmallWritePath::Inline SmallWritePath::Inline
@@ -3981,20 +3863,11 @@ fn inline_erasure_shard_file_offset(
} }
fn collect_inline_data_shard_fileinfos_by_index<'a>( fn collect_inline_data_shard_fileinfos_by_index<'a>(
parts_metadata: &'a [FileInfo],
fi: &FileInfo,
data_shards: usize,
disk_is_online: impl FnMut(usize) -> bool,
) -> Option<Vec<&'a FileInfo>> {
collect_inline_data_shard_fileinfos_by_index_or_reason(parts_metadata, fi, data_shards, disk_is_online).ok()
}
fn collect_inline_data_shard_fileinfos_by_index_or_reason<'a>(
parts_metadata: &'a [FileInfo], parts_metadata: &'a [FileInfo],
fi: &FileInfo, fi: &FileInfo,
data_shards: usize, data_shards: usize,
mut disk_is_online: impl FnMut(usize) -> bool, mut disk_is_online: impl FnMut(usize) -> bool,
) -> std::result::Result<Vec<&'a FileInfo>, &'static str> { ) -> Option<Vec<&'a FileInfo>> {
let distribution = &fi.erasure.distribution; let distribution = &fi.erasure.distribution;
let mut data_files = vec![None; data_shards]; let mut data_files = vec![None; data_shards];
@@ -4002,35 +3875,27 @@ fn collect_inline_data_shard_fileinfos_by_index_or_reason<'a>(
if !disk_is_online(disk_index) { if !disk_is_online(disk_index) {
continue; continue;
} }
let Some(&block_index) = distribution.get(disk_index) else { let block_index = *distribution.get(disk_index)?;
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
};
if block_index == 0 || block_index > data_shards { if block_index == 0 || block_index > data_shards {
continue; continue;
} }
if file_info.name.is_empty() {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD);
}
if file_info.erasure.index != block_index { if file_info.erasure.index != block_index {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH); continue;
} }
if !file_info.has_valid_erasure_geometry() { if !file_info.has_valid_erasure_geometry() {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY); continue;
} }
if !core::io_primitives::metadata_early_stop_candidate_matches(file_info, fi) { if !core::io_primitives::metadata_early_stop_candidate_matches(file_info, fi) {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH); continue;
} }
if file_info.data.as_ref().is_none_or(|data| data.is_empty()) { if file_info.data.as_ref().is_none_or(|data| data.is_empty()) {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD); continue;
} }
data_files[block_index - 1] = Some(file_info); data_files[block_index - 1] = Some(file_info);
} }
data_files data_files.into_iter().collect()
.into_iter()
.collect::<Option<Vec<_>>>()
.ok_or(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD)
} }
impl SetDisks { impl SetDisks {
@@ -4357,7 +4222,6 @@ fn check_object_lock_retention_update(bucket: &str, object: &str, obj_info: &Obj
/// ///
/// Fail closed: when bucket metadata cannot be resolved the check stays on, so /// Fail closed: when bucket metadata cannot be resolved the check stays on, so
/// object-lock protection is never skipped because of a metadata lookup miss. /// object-lock protection is never skipped because of a metadata lookup miss.
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(crate) fn object_lock_delete_check_required(bucket_meta: Option<&crate::bucket::metadata::BucketMetadata>) -> bool { pub(crate) fn object_lock_delete_check_required(bucket_meta: Option<&crate::bucket::metadata::BucketMetadata>) -> bool {
bucket_meta.is_none_or(|meta| meta.object_locking()) bucket_meta.is_none_or(|meta| meta.object_locking())
} }
@@ -4633,6 +4497,15 @@ impl Hash for ObjProps {
} }
} }
#[derive(Default, Clone, Debug)]
pub struct HealEntryResult {
pub bytes: usize,
pub success: bool,
pub skipped: bool,
pub entry_done: bool,
pub name: String,
}
fn is_object_dangling( fn is_object_dangling(
meta_arr: &[FileInfo], meta_arr: &[FileInfo],
errs: &[Option<DiskError>], errs: &[Option<DiskError>],
@@ -5409,7 +5282,6 @@ pub fn is_valid_storage_class(storage_class: &str) -> bool {
} }
/// Returns true if the storage class is a cold storage tier that requires special handling /// Returns true if the storage class is a cold storage tier that requires special handling
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub fn is_cold_storage_class(storage_class: &str) -> bool { pub fn is_cold_storage_class(storage_class: &str) -> bool {
matches!( matches!(
storage_class, storage_class,
@@ -5418,7 +5290,6 @@ pub fn is_cold_storage_class(storage_class: &str) -> bool {
} }
/// Returns true if the storage class is an infrequent access tier /// Returns true if the storage class is an infrequent access tier
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub fn is_infrequent_access_class(storage_class: &str) -> bool { pub fn is_infrequent_access_class(storage_class: &str) -> bool {
matches!( matches!(
storage_class, storage_class,
+2 -11
View File
@@ -453,9 +453,7 @@ impl SetDisks {
..Default::default() ..Default::default()
}; };
// Bound, not `_`: this guard must live to the end of the scope. A bare let write_lock_guard = if !opts.no_lock {
// `_` would drop it here and release the namespace write lock.
let _write_lock_guard = if !opts.no_lock {
let ns_lock = self.new_ns_lock(bucket, object).await?; let ns_lock = self.new_ns_lock(bucket, object).await?;
Some( Some(
ns_lock ns_lock
@@ -998,7 +996,7 @@ impl SetDisks {
readers.push(None); readers.push(None);
continue; continue;
} }
Err(_e) => { Err(e) => {
readers.push(None); readers.push(None);
continue; continue;
} }
@@ -1547,9 +1545,6 @@ impl SetDisks {
for candidate in candidates.iter_mut().filter(|candidate| candidate.local_payload) { for candidate in candidates.iter_mut().filter(|candidate| candidate.local_payload) {
for (disk_index, disk) in disks.iter().enumerate() { for (disk_index, disk) in disks.iter().enumerate() {
// Only the #[cfg(test)] fault-injection branch below reads this.
#[cfg(not(test))]
let _ = disk_index;
let Some(disk) = disk else { let Some(disk) = disk else {
return Ok(DanglingDeleteSafety::UnsafeToDelete); return Ok(DanglingDeleteSafety::UnsafeToDelete);
}; };
@@ -1721,10 +1716,6 @@ impl SetDisks {
Ok((result, None)) Ok((result, None))
} }
#[allow(
dead_code,
reason = "lock-taking wrapper over the live heal_object_dir_locked; only comments reference it (backlog#1823)"
)]
#[tracing::instrument(level = "trace", skip(self), fields(bucket = %bucket, object = %object))] #[tracing::instrument(level = "trace", skip(self), fields(bucket = %bucket, object = %object))]
pub(in crate::set_disk) async fn heal_object_dir( pub(in crate::set_disk) async fn heal_object_dir(
&self, &self,
@@ -66,7 +66,6 @@ impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
} }
impl SetDisks { impl SetDisks {
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String { pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
match err { match err {
LockResult::Timeout => { LockResult::Timeout => {
@@ -80,7 +79,6 @@ impl SetDisks {
} }
} }
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
pub(in crate::set_disk) fn format_lock_error_from_error( pub(in crate::set_disk) fn format_lock_error_from_error(
&self, &self,
bucket: &str, bucket: &str,
@@ -145,7 +143,6 @@ impl SetDisks {
disks disks
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(in crate::set_disk) async fn get_online_disks(&self) -> Vec<Option<DiskStore>> { pub(in crate::set_disk) async fn get_online_disks(&self) -> Vec<Option<DiskStore>> {
let snapshot = self.drive_membership_snapshot().await; let snapshot = self.drive_membership_snapshot().await;
let mut disks = snapshot.strict_online_candidates().into_iter().map(Some).collect::<Vec<_>>(); let mut disks = snapshot.strict_online_candidates().into_iter().map(Some).collect::<Vec<_>>();
@@ -156,10 +153,6 @@ impl SetDisks {
disks disks
} }
#[allow(
dead_code,
reason = "local-only sibling of the test-covered get_online_disks; no caller in this port (backlog#1823)"
)]
pub(in crate::set_disk) async fn get_online_local_disks(&self) -> Vec<Option<DiskStore>> { pub(in crate::set_disk) async fn get_online_local_disks(&self) -> Vec<Option<DiskStore>> {
let snapshot = self.drive_membership_snapshot().await; let snapshot = self.drive_membership_snapshot().await;
let mut disks = snapshot let mut disks = snapshot
@@ -439,10 +432,6 @@ impl SetDisks {
Ok((disk, fm)) Ok((disk, fm))
} }
#[allow(
dead_code,
reason = "MinIO-parity healing-disk accessor with no caller in this port (backlog#1823)"
)]
pub(in crate::set_disk) async fn get_online_disk_with_healing( pub(in crate::set_disk) async fn get_online_disk_with_healing(
&self, &self,
incl_healing: bool, incl_healing: bool,
@@ -451,10 +440,6 @@ impl SetDisks {
Ok((new_disks, healing > 0)) Ok((new_disks, healing > 0))
} }
#[allow(
dead_code,
reason = "reached only from get_online_disk_with_healing, itself uncalled in this port (backlog#1823)"
)]
pub(in crate::set_disk) async fn get_online_disk_with_healing_and_info( pub(in crate::set_disk) async fn get_online_disk_with_healing_and_info(
&self, &self,
incl_healing: bool, incl_healing: bool,
+5 -7
View File
@@ -415,7 +415,6 @@ fn reduce_quorum_part_numbers(object_parts: Vec<Vec<String>>, read_quorum: usize
/// never returned, but flips `is_truncated` to `true` and yields a /// never returned, but flips `is_truncated` to `true` and yields a
/// `next_upload_id_marker` pointing at the last returned upload so the caller can /// `next_upload_id_marker` pointing at the last returned upload so the caller can
/// resume paging. /// resume paging.
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn paginate_upload_page(remaining: &[MultipartInfo], max_uploads: usize) -> (Vec<MultipartInfo>, bool, Option<String>) { fn paginate_upload_page(remaining: &[MultipartInfo], max_uploads: usize) -> (Vec<MultipartInfo>, bool, Option<String>) {
let is_truncated = remaining.len() > max_uploads; let is_truncated = remaining.len() > max_uploads;
let page: Vec<MultipartInfo> = remaining.iter().take(max_uploads).cloned().collect(); let page: Vec<MultipartInfo> = remaining.iter().take(max_uploads).cloned().collect();
@@ -558,7 +557,6 @@ impl SetDisks {
} }
#[tracing::instrument(level = "debug", skip(self))] #[tracing::instrument(level = "debug", skip(self))]
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
pub(super) async fn check_upload_id_exists( pub(super) async fn check_upload_id_exists(
&self, &self,
bucket: &str, bucket: &str,
@@ -1400,7 +1398,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
let mut count = max_parts; let mut count = max_parts;
for part in object_parts.iter() { for (i, part) in object_parts.iter().enumerate() {
if let Some(err) = &part.error { if let Some(err) = &part.error {
warn!("list_object_parts part error: {:?}", &err); warn!("list_object_parts part error: {:?}", &err);
} }
@@ -2043,8 +2041,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
&& let Err(err) = checksum.add_part(&cs, ext_part.actual_size) && let Err(err) = checksum.add_part(&cs, ext_part.actual_size)
{ {
error!( error!(
"complete_multipart_upload checksum add_part failed part_id={}, bucket={}, object={}, err={}", "complete_multipart_upload checksum add_part failed part_id={}, bucket={}, object={}",
p.part_num, bucket, object, err p.part_num, bucket, object
); );
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default())); return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
} }
@@ -2089,8 +2087,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
} }
} else if let Err(err) = wtcs.matches(&checksum_combined, uploaded_parts.len() as i32) { } else if let Err(err) = wtcs.matches(&checksum_combined, uploaded_parts.len() as i32) {
error!( error!(
"complete_multipart_upload checksum matches failed want={}, got={}, err={}", "complete_multipart_upload checksum matches failed want={}, got={}",
wtcs.encoded, checksum.encoded, err wtcs.encoded, checksum.encoded
); );
return Err(Error::other(format!( return Err(Error::other(format!(
"complete_multipart_upload checksum matches failed want={}, got={}", "complete_multipart_upload checksum matches failed want={}, got={}",
+13 -333
View File
@@ -3507,10 +3507,6 @@ struct TransitionUploadedSaveProbeState {
} }
#[cfg(test)] #[cfg(test)]
#[allow(
dead_code,
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
)]
struct TransitionUploadedSaveProbe { struct TransitionUploadedSaveProbe {
state: Arc<TransitionUploadedSaveProbeState>, state: Arc<TransitionUploadedSaveProbeState>,
} }
@@ -3521,10 +3517,6 @@ static TRANSITION_UPLOADED_SAVE_PROBE: std::sync::OnceLock<std::sync::Mutex<Opti
#[cfg(test)] #[cfg(test)]
impl TransitionUploadedSaveProbe { impl TransitionUploadedSaveProbe {
#[allow(
dead_code,
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
)]
fn install(bucket: &str, object: &str) -> Self { fn install(bucket: &str, object: &str) -> Self {
let state = Arc::new(TransitionUploadedSaveProbeState { let state = Arc::new(TransitionUploadedSaveProbeState {
bucket: bucket.to_string(), bucket: bucket.to_string(),
@@ -3541,10 +3533,6 @@ impl TransitionUploadedSaveProbe {
Self { state } Self { state }
} }
#[allow(
dead_code,
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
)]
fn attempts(&self) -> usize { fn attempts(&self) -> usize {
self.state.attempts.load(std::sync::atomic::Ordering::Acquire) self.state.attempts.load(std::sync::atomic::Ordering::Acquire)
} }
@@ -3750,10 +3738,6 @@ struct TransitionCommitBarrierState {
} }
#[cfg(test)] #[cfg(test)]
#[allow(
dead_code,
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
)]
struct TransitionCommitBarrier { struct TransitionCommitBarrier {
state: Arc<TransitionCommitBarrierState>, state: Arc<TransitionCommitBarrierState>,
} }
@@ -3764,26 +3748,14 @@ static TRANSITION_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Ar
#[cfg(test)] #[cfg(test)]
impl TransitionCommitBarrier { impl TransitionCommitBarrier {
#[allow(
dead_code,
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
)]
fn install_before_lock_lost_check(bucket: &str, object: &str) -> Self { fn install_before_lock_lost_check(bucket: &str, object: &str) -> Self {
Self::install_at(bucket, object, TransitionCommitPause::BeforeLockLost) Self::install_at(bucket, object, TransitionCommitPause::BeforeLockLost)
} }
#[allow(
dead_code,
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
)]
fn install(bucket: &str, object: &str) -> Self { fn install(bucket: &str, object: &str) -> Self {
Self::install_at(bucket, object, TransitionCommitPause::BeforeLeaseValidation) Self::install_at(bucket, object, TransitionCommitPause::BeforeLeaseValidation)
} }
#[allow(
dead_code,
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
)]
fn install_after_lease_check(bucket: &str, object: &str) -> Self { fn install_after_lease_check(bucket: &str, object: &str) -> Self {
Self::install_at(bucket, object, TransitionCommitPause::AfterLeaseValidation) Self::install_at(bucket, object, TransitionCommitPause::AfterLeaseValidation)
} }
@@ -3806,20 +3778,12 @@ impl TransitionCommitBarrier {
Self { state } Self { state }
} }
#[allow(
dead_code,
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
)]
async fn wait_until_paused(&self) { async fn wait_until_paused(&self) {
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
.await .await
.expect("transition should reach the deterministic commit barrier"); .expect("transition should reach the deterministic commit barrier");
} }
#[allow(
dead_code,
reason = "installed by set_disk tests behind `--features test-util` (backlog#1823)"
)]
fn release(&self) { fn release(&self) {
self.state.release.notify_one(); self.state.release.notify_one();
} }
@@ -5656,9 +5620,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
// TODO: Lifecycle // TODO: Lifecycle
let mut version_found = true; let mut version_found = true;
// delete_object_version below derives its own majority quorum from the let (mut goi, write_quorum, gerr) = self.get_object_info_and_quorum(bucket, object, &opts).await;
// disk array, so the object-derived quorum here is unused.
let (mut goi, _write_quorum, gerr) = self.get_object_info_and_quorum(bucket, object, &opts).await;
if let Some(err) = &gerr if let Some(err) = &gerr
&& goi.name.is_empty() && goi.name.is_empty()
{ {
@@ -6412,7 +6374,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks); self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
for disk in disks.iter() { for disk in disks.iter() {
if disk.is_some() { if let Some(disk) = disk {
continue; continue;
} }
let _ = self let _ = self
@@ -7543,7 +7505,7 @@ mod get_object_downstream_close_accounting_tests {
use super::hermetic_set_disks_support::hermetic_set_disks; use super::hermetic_set_disks_support::hermetic_set_disks;
use super::*; use super::*;
use crate::diagnostics::get::{ use crate::diagnostics::get::{
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_OBJECT_PATH_INTERNAL_META, GET_STAGE_DECODE, GET_STAGE_EMIT, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_OBJECT_PATH_INTERNAL_META, GET_STAGE_DECODE, GET_STAGE_EMIT,
GetObjectFailureReason, GetObjectFailureReason,
}; };
use crate::disk::RUSTFS_META_BUCKET; use crate::disk::RUSTFS_META_BUCKET;
@@ -7675,8 +7637,8 @@ mod get_object_downstream_close_accounting_tests {
legacy_completed, legacy_completed,
internal_cancelled, internal_cancelled,
legacy_cancelled, legacy_cancelled,
internal_not_found_miss, internal_unsafe_miss,
legacy_not_found_miss, legacy_unsafe_miss,
internal_saved, internal_saved,
legacy_saved, legacy_saved,
) = metrics::with_local_recorder(&recorder, || { ) = metrics::with_local_recorder(&recorder, || {
@@ -7752,7 +7714,7 @@ mod get_object_downstream_close_accounting_tests {
&[ &[
("path", GET_OBJECT_PATH_INTERNAL_META), ("path", GET_OBJECT_PATH_INTERNAL_META),
("decision", "miss"), ("decision", "miss"),
("reason", GET_METADATA_EARLY_STOP_REASON_NOT_FOUND), ("reason", GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST),
], ],
), ),
recorder.counter_value( recorder.counter_value(
@@ -7760,7 +7722,7 @@ mod get_object_downstream_close_accounting_tests {
&[ &[
("path", GET_OBJECT_PATH_LEGACY_DUPLEX), ("path", GET_OBJECT_PATH_LEGACY_DUPLEX),
("decision", "miss"), ("decision", "miss"),
("reason", GET_METADATA_EARLY_STOP_REASON_NOT_FOUND), ("reason", GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST),
], ],
), ),
recorder.histogram_values( recorder.histogram_values(
@@ -7811,21 +7773,21 @@ mod get_object_downstream_close_accounting_tests {
"internal metadata lifecycle cancelled count must not leak into legacy_duplex" "internal metadata lifecycle cancelled count must not leak into legacy_duplex"
); );
assert_eq!( assert_eq!(
internal_not_found_miss, 1, internal_unsafe_miss, 1,
"internal metadata not-found early-stop miss must retain its path label" "internal metadata unsafe early-stop miss must retain its path label"
); );
assert_eq!( assert_eq!(
legacy_not_found_miss, 0, legacy_unsafe_miss, 0,
"internal metadata not-found early-stop miss must not leak into legacy_duplex" "internal metadata unsafe early-stop miss must not leak into legacy_duplex"
); );
assert_eq!( assert_eq!(
internal_saved, internal_saved,
vec![0.0], vec![0.0],
"internal metadata not-found miss must record zero saved responses on internal_meta" "internal metadata unsafe miss must record zero saved responses on internal_meta"
); );
assert!( assert!(
legacy_saved.is_empty(), legacy_saved.is_empty(),
"internal metadata not-found miss saved responses must not leak into legacy_duplex" "internal metadata unsafe miss saved responses must not leak into legacy_duplex"
); );
} }
} }
@@ -10208,288 +10170,6 @@ mod transition_upload_integrity_tests {
assert!(backend.contains(remote_object).await, "committed remote object should remain available"); assert!(backend.contains(remote_object).await, "committed remote object should remain available");
} }
/// Compresses `plaintext` with the codec the PUT path uses, so the stored
/// bytes round-trip through the read path's decompressor.
async fn compress_for_storage(plaintext: &[u8]) -> Vec<u8> {
let mut reader = crate::io_support::rio::compression_reader(
Cursor::new(plaintext.to_vec()),
rustfs_utils::CompressionAlgorithm::default(),
false,
);
let mut compressed = Vec::new();
reader.read_to_end(&mut compressed).await.expect("plaintext should compress");
assert!(compressed.len() < plaintext.len(), "test payload must actually compress");
compressed
}
/// Writes a genuinely compressed object: stored data is `compressed`, and the
/// metadata marks it compressed with the plaintext length as its actual size,
/// exactly as the app-layer compress path records it.
async fn write_compressed_source(
set_disks: &Arc<SetDisks>,
disk_stores: &[DiskStore],
bucket: &str,
object: &str,
plaintext: &[u8],
compressed: &[u8],
) -> ObjectInfo {
for disk in disk_stores {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
let mut user_defined = HashMap::new();
rustfs_utils::http::insert_str(
&mut user_defined,
rustfs_utils::http::SUFFIX_COMPRESSION,
crate::io_support::rio::compression_metadata_value(rustfs_utils::CompressionAlgorithm::default()),
);
rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, plaintext.len().to_string());
let stream = crate::io_support::rio::HashReader::from_stream(
Cursor::new(compressed.to_vec()),
compressed.len() as i64,
plaintext.len() as i64,
None,
None,
false,
)
.expect("hash reader over compressed bytes");
let mut reader = PutObjReader::new(stream);
set_disks
.put_object(
bucket,
object,
&mut reader,
&ObjectOptions {
no_lock: true,
user_defined,
..Default::default()
},
)
.await
.expect("compressed object should be written")
}
async fn read_transitioned(
set_disks: &Arc<SetDisks>,
bucket: &str,
object: &str,
range: Option<HTTPRangeSpec>,
opts: &ObjectOptions,
) -> (Vec<u8>, i64) {
let mut reader = set_disks
.get_object_reader(bucket, object, range, HeaderMap::new(), opts)
.await
.expect("transitioned object reader should open");
let published_size = reader.object_info.size;
let mut body = Vec::new();
reader
.stream
.read_to_end(&mut body)
.await
.expect("transitioned body should drain");
(body, published_size)
}
/// Transition uploads the object's STORED bytes, so a tiered read has to
/// apply the same transform an erasure read would. #6107 routed this path
/// through `ReadPlan` to stop serving an encrypted object's ciphertext;
/// compression rides the same plan, and nothing pinned it (backlog#1851).
/// Without the transform this GET returns the compressed bytes under the
/// compressed size — silent corruption for every client of a compressed
/// object that ILM has moved to a warm tier.
#[tokio::test]
#[serial_test::serial]
async fn transitioned_compressed_object_get_returns_plaintext() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "transitioned-compressed-get-bucket";
let object = "object.txt";
let plaintext = b"transitioned compressed objects must decompress on read ".repeat(20_000);
let compressed = compress_for_storage(&plaintext).await;
let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await;
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
let (local_body, local_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await;
assert_eq!(local_body, plaintext, "control: the pre-transition read must decompress");
assert_eq!(
local_size,
plaintext.len() as i64,
"control: the pre-transition read publishes the plaintext size"
);
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
set_disks
.transition_object(bucket, object, &transition_options(&original, tier_name))
.await
.expect("transition should commit");
let put_versions = backend.put_versions().await;
assert_eq!(put_versions.len(), 1, "transition should upload one remote candidate");
let remote_bytes = backend
.bytes(&put_versions[0].0)
.await
.expect("remote candidate should be stored");
assert_eq!(
remote_bytes, compressed,
"transition uploads the stored representation; the read side is what has to decode it"
);
let (body, published_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await;
assert_eq!(body, plaintext, "a tiered read must return the object's content, not its stored bytes");
assert_eq!(
published_size,
plaintext.len() as i64,
"a tiered read must publish the plaintext size, not the compressed one"
);
}
/// A ranged tiered read is expressed in plaintext coordinates, so the plan
/// has to translate it into the remote copy's compressed extent and skip
/// into the decompressed stream — the same translation the erasure path does.
#[tokio::test]
#[serial_test::serial]
async fn transitioned_compressed_object_range_get_returns_plaintext_slice() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "transitioned-compressed-range-bucket";
let object = "object.txt";
let plaintext = b"ranged reads of transitioned compressed objects must land in plaintext ".repeat(20_000);
let compressed = compress_for_storage(&plaintext).await;
let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await;
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
set_disks
.transition_object(bucket, object, &transition_options(&original, tier_name))
.await
.expect("transition should commit");
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
// Deliberately past the compressed size, so a range still measured in
// stored coordinates could not produce this slice.
let start = compressed.len() as i64 + 4096;
let end = start + 511;
let range = HTTPRangeSpec {
is_suffix_length: false,
start,
end,
};
let (body, published_size) = read_transitioned(&set_disks, bucket, object, Some(range), &opts).await;
let expected = &plaintext[start as usize..=end as usize];
assert_eq!(body, expected, "a ranged tiered read must return that plaintext slice");
assert_eq!(published_size, expected.len() as i64, "a ranged tiered read publishes the slice length");
}
/// The restore copy-back re-writes the object under its original metadata,
/// which still says "compressed". It therefore has to keep receiving the
/// STORED bytes: `restore_request_active` holds it on the plan's `Plain`
/// branch, and decompressing there would write plaintext under compressed
/// metadata.
#[tokio::test]
#[serial_test::serial]
async fn restore_read_of_transitioned_compressed_object_keeps_stored_bytes() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "transitioned-compressed-restore-bucket";
let object = "object.txt";
let plaintext = b"restore copy-back must keep the stored representation intact ".repeat(20_000);
let compressed = compress_for_storage(&plaintext).await;
let original = write_compressed_source(&set_disks, &disk_stores, bucket, object, &plaintext, &compressed).await;
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
set_disks
.transition_object(bucket, object, &transition_options(&original, tier_name))
.await
.expect("transition should commit");
let oi = set_disks
.get_object_info(
bucket,
object,
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
.expect("transitioned metadata should resolve");
let restore_opts = ObjectOptions {
no_lock: true,
part_number: Some(1),
transition: TransitionOptions {
restore_request: s3s::dto::RestoreRequest {
days: Some(1),
..Default::default()
},
..Default::default()
},
..Default::default()
};
let mut reader = get_transitioned_object_reader_with_tier_manager(
bucket,
object,
&None,
&HeaderMap::new(),
&oi,
&restore_opts,
&set_disks.ctx.tier_config_mgr(),
set_disks.ctx.object_encryption_resolver(),
)
.await
.expect("restore read of the tiered copy should open");
let published_size = reader.object_info.size;
let mut body = Vec::new();
reader.stream.read_to_end(&mut body).await.expect("restore body should drain");
assert_eq!(body, compressed, "a restore read must copy the stored bytes back verbatim");
assert_eq!(
published_size,
compressed.len() as i64,
"a restore read must keep publishing the stored size"
);
}
/// Plain objects must keep streaming the remote bytes through untouched:
/// their plan is `Plain`, so the tiered read stays byte-identical.
#[tokio::test]
#[serial_test::serial]
async fn transitioned_plain_object_get_is_unchanged() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "transitioned-plain-get-bucket";
let object = "object.bin";
let payload = b"plain transitioned objects must keep reading back byte-identical ".repeat(1024);
let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
set_disks
.transition_object(bucket, object, &transition_options(&original, tier_name))
.await
.expect("transition should commit");
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
let (body, published_size) = read_transitioned(&set_disks, bucket, object, None, &opts).await;
assert_eq!(body, payload);
assert_eq!(published_size, payload.len() as i64);
let range = HTTPRangeSpec {
is_suffix_length: false,
start: 100,
end: 611,
};
let (ranged_body, ranged_size) = read_transitioned(&set_disks, bucket, object, Some(range), &opts).await;
assert_eq!(ranged_body, &payload[100..=611]);
assert_eq!(ranged_size, payload.len() as i64, "a plain ranged read keeps publishing the object size");
}
async fn corrupt_beyond_read_quorum( async fn corrupt_beyond_read_quorum(
temp_dirs: &[tempfile::TempDir], temp_dirs: &[tempfile::TempDir],
bucket: &str, bucket: &str,
+3 -35
View File
@@ -116,7 +116,6 @@ impl SetDisks {
.then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE) .then_some(GET_METADATA_CACHE_REASON_DIST_ERASURE)
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option<Arc<GetObjectMetadataCacheEntry>> { async fn cached_get_object_fileinfo(&self, bucket: &str, object: &str) -> Option<Arc<GetObjectMetadataCacheEntry>> {
match self.lookup_cached_get_object_fileinfo(bucket, object).await { match self.lookup_cached_get_object_fileinfo(bucket, object).await {
MetadataCacheLookup::Hit(entry) => Some(entry), MetadataCacheLookup::Hit(entry) => Some(entry),
@@ -1827,7 +1826,6 @@ fn get_object_metadata_cache_request_bypass_reason(bucket: &str, opts: &ObjectOp
.then_some(GET_METADATA_CACHE_REASON_META_BUCKET) .then_some(GET_METADATA_CACHE_REASON_META_BUCKET)
} }
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOptions, read_data: bool) -> bool { fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOptions, read_data: bool) -> bool {
get_object_metadata_cache_request_bypass_reason(bucket, opts, read_data).is_none() get_object_metadata_cache_request_bypass_reason(bucket, opts, read_data).is_none()
} }
@@ -3888,15 +3886,13 @@ mod tests {
assert!(metadata_early_stop_permitted(true, true, false, "", false, false)); assert!(metadata_early_stop_permitted(true, true, false, "", false, false));
// observe=false (non-observed fanout) also disables early-stop. // observe=false (non-observed fanout) also disables early-stop.
assert!(!metadata_early_stop_permitted(true, false, false, "", false, false)); assert!(!metadata_early_stop_permitted(true, false, false, "", false, false));
// Whole/latest data-read metadata is now allowed by default; assert!(!metadata_early_stop_permitted(true, true, true, "", false, false));
// the inline verifier still decides whether it can stop early.
assert!(metadata_early_stop_permitted(true, true, true, "", false, false));
}, },
); );
} }
#[test] #[test]
fn metadata_early_stop_allows_safe_data_reads_by_default() { fn metadata_early_stop_keeps_data_reads_opt_in_by_default() {
temp_env::with_vars( temp_env::with_vars(
[ [
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")), (ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
@@ -3904,7 +3900,7 @@ mod tests {
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, None), (ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, None),
], ],
|| { || {
assert!(should_allow_metadata_early_stop(true, "", false, false)); assert!(!should_allow_metadata_early_stop(true, "", false, false));
assert!(!should_allow_metadata_early_stop(true, "version-id", false, false)); assert!(!should_allow_metadata_early_stop(true, "version-id", false, false));
assert!(should_allow_metadata_early_stop(false, "", false, false)); assert!(should_allow_metadata_early_stop(false, "", false, false));
assert!(!should_allow_metadata_early_stop(false, "version-id", false, false)); assert!(!should_allow_metadata_early_stop(false, "version-id", false, false));
@@ -3936,34 +3932,6 @@ mod tests {
); );
} }
#[test]
fn metadata_early_stop_bounded_fanout_defaults_to_enabled() {
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, None),
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, None),
],
|| {
assert!(is_get_metadata_data_read_early_stop_enabled());
assert!(is_get_metadata_early_stop_bounded_fanout_enabled());
},
);
temp_env::with_vars([(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("false"))], || {
assert!(!is_get_metadata_early_stop_bounded_fanout_enabled());
});
temp_env::with_vars(
[
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("false")),
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true")),
],
|| {
assert!(!is_get_metadata_data_read_early_stop_enabled());
assert!(is_get_metadata_early_stop_bounded_fanout_enabled());
},
);
}
#[test] #[test]
fn metadata_early_stop_rejects_healing_and_free_version_requests() { fn metadata_early_stop_rejects_healing_and_free_version_requests() {
temp_env::with_vars( temp_env::with_vars(
+4 -1
View File
@@ -135,7 +135,10 @@ impl FileMeta {
let i = buf.len() as u64; let i = buf.len() as u64;
// check version, buf = buf[8..] // check version, buf = buf[8..]
let (buf, _, _) = Self::check_xl2_v1(buf)?; let (buf, _, _) = Self::check_xl2_v1(buf).map_err(|e| {
error!("failed to check XL2 v1 format: {}", e);
e
})?;
if buf.len() < 5 { if buf.len() < 5 {
error!( error!(
+2 -7
View File
@@ -82,8 +82,8 @@ impl Error {
/// Whether a heal operation can be retried without changing its inputs. /// Whether a heal operation can be retried without changing its inputs.
pub(crate) fn is_recoverable_heal(&self) -> bool { pub(crate) fn is_recoverable_heal(&self) -> bool {
match self { match self {
Error::TaskCancelled | Error::TaskTimeout => false, Error::TaskCancelled => false,
Error::TransientSkip { .. } => true, Error::TaskTimeout | Error::TransientSkip { .. } => true,
Error::Storage(err) => { Error::Storage(err) => {
err.is_quorum_error() err.is_quorum_error()
|| matches!( || matches!(
@@ -165,9 +165,4 @@ mod tests {
assert!(Error::Storage(EcstoreError::DiskNotFound).is_recoverable_heal()); assert!(Error::Storage(EcstoreError::DiskNotFound).is_recoverable_heal());
assert!(Error::Storage(EcstoreError::VolumeNotFound).is_recoverable_heal()); assert!(Error::Storage(EcstoreError::VolumeNotFound).is_recoverable_heal());
} }
#[test]
fn task_timeout_is_terminal() {
assert!(!Error::TaskTimeout.is_recoverable_heal());
}
} }
+20 -30
View File
@@ -673,12 +673,6 @@ fn retry_request_for_result(task: &HealTask, result: &Result<()>) -> Option<(Hea
Some((request, delay, error)) Some((request, delay, error))
} }
async fn retry_request_for_result_with_budget(task: &HealTask, result: &Result<()>) -> Option<(HealRequest, Duration, String)> {
let (_, delay, error) = retry_request_for_result(task, result)?;
let request = task.retry_request_with_remaining_timeout().await.ok()?;
Some((request, delay, error))
}
fn recoverable_heal_retry_delay(retry_attempt: u32) -> Duration { fn recoverable_heal_retry_delay(retry_attempt: u32) -> Duration {
let retry_attempt = retry_attempt.clamp(1, 5); let retry_attempt = retry_attempt.clamp(1, 5);
let delay = Duration::from_secs(2_u64.saturating_pow(retry_attempt)); let delay = Duration::from_secs(2_u64.saturating_pow(retry_attempt));
@@ -696,7 +690,7 @@ pub struct HealConfig {
pub max_concurrent_heals: usize, pub max_concurrent_heals: usize,
/// Maximum concurrent heal tasks allowed for a single erasure set /// Maximum concurrent heal tasks allowed for a single erasure set
pub max_concurrent_per_set: usize, pub max_concurrent_per_set: usize,
/// Aggregate task execution timeout across recoverable retries /// Task timeout
pub task_timeout: Duration, pub task_timeout: Duration,
/// Queue size /// Queue size
pub queue_size: usize, pub queue_size: usize,
@@ -3112,7 +3106,7 @@ impl HealManager {
"Heal scheduler task started" "Heal scheduler task started"
); );
let result = task.execute().await; let result = task.execute().await;
let retry_request = retry_request_for_result_with_budget(task.as_ref(), &result).await; let retry_request = retry_request_for_result(task.as_ref(), &result);
match &result { match &result {
Ok(_) => { Ok(_) => {
debug!( debug!(
@@ -4545,25 +4539,6 @@ mod tests {
assert!(retry_error.contains("Lock acquisition timeout")); assert!(retry_error.contains("Lock acquisition timeout"));
} }
#[tokio::test]
async fn retry_request_for_result_preserves_remaining_timeout_budget() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let mut request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
request.options.timeout = Some(Duration::from_secs(60));
let task = HealTask::from_request(request, storage);
let result = task.execute().await;
let (retry_request, _, _) = retry_request_for_result_with_budget(&task, &result)
.await
.expect("read quorum failure should retain the unused timeout budget");
let remaining = retry_request
.options
.timeout
.expect("configured timeout should remain present");
assert!(remaining < Duration::from_secs(60));
assert!(remaining > Duration::from_secs(59));
}
#[test] #[test]
fn test_retry_request_for_incomplete_heal_rename() { fn test_retry_request_for_incomplete_heal_rename() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage); let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
@@ -6079,7 +6054,7 @@ mod tests {
process_manager_queue_once(&manager).await; process_manager_queue_once(&manager).await;
let defaulted_status = tokio::time::timeout(Duration::from_secs(1), async { let defaulted_status = tokio::time::timeout(Duration::from_secs(1), async {
loop { loop {
if let Ok(status @ HealTaskStatus::Timeout) = manager.get_task_status(&defaulted_id).await { if let Ok(status @ HealTaskStatus::Retrying { .. }) = manager.get_task_status(&defaulted_id).await {
break status; break status;
} }
tokio::task::yield_now().await; tokio::task::yield_now().await;
@@ -6087,8 +6062,23 @@ mod tests {
}) })
.await .await
.expect("configured timeout should finish the task"); .expect("configured timeout should finish the task");
assert_eq!(defaulted_status, HealTaskStatus::Timeout); assert!(matches!(defaulted_status, HealTaskStatus::Retrying { .. }));
assert!(manager.retrying_heals.lock().await.get(&defaulted_id).is_none()); assert_eq!(
manager
.retrying_heals
.lock()
.await
.get(&defaulted_id)
.expect("timed out task should retain its retry request")
.request
.options
.timeout,
Some(Duration::ZERO)
);
manager
.cancel_task(&defaulted_id)
.await
.expect("retrying timeout task should be cancelled");
let mut explicit = bucket_request("explicit-timeout", HealPriority::Normal, HealRequestSource::Admin); let mut explicit = bucket_request("explicit-timeout", HealPriority::Normal, HealRequestSource::Admin);
explicit.options.timeout = Some(Duration::from_secs(60)); explicit.options.timeout = Some(Duration::from_secs(60));
+1 -39
View File
@@ -196,7 +196,7 @@ pub struct HealOptions {
/// Whether to skip namespace locking /// Whether to skip namespace locking
#[serde(default)] #[serde(default)]
pub no_lock: bool, pub no_lock: bool,
/// Aggregate execution timeout across recoverable manager retries /// Timeout
pub timeout: Option<Duration>, pub timeout: Option<Duration>,
/// pool index /// pool index
pub pool_index: Option<usize>, pub pool_index: Option<usize>,
@@ -442,14 +442,6 @@ impl HealTask {
} }
} }
pub(crate) async fn retry_request_with_remaining_timeout(&self) -> Result<HealRequest> {
let mut request = self.retry_request();
if self.options.timeout.is_some() {
request.options.timeout = self.remaining_timeout().await?;
}
Ok(request)
}
pub(crate) fn from_replacement_recovery_request( pub(crate) fn from_replacement_recovery_request(
request: HealRequest, request: HealRequest,
storage: Arc<dyn HealStorageAPI>, storage: Arc<dyn HealStorageAPI>,
@@ -2665,36 +2657,6 @@ mod tests {
use super::super::storage_api::status::BucketInfo; use super::super::storage_api::status::BucketInfo;
#[tokio::test]
async fn retry_request_carries_remaining_timeout_budget() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage::default());
let mut request = HealRequest::bucket("bucket".to_string());
request.options.timeout = Some(Duration::from_secs(100));
let task = HealTask::from_request(request, storage.clone());
*task.task_start_instant.write().await = Some(Instant::now() - Duration::from_secs(40));
let retry = task
.retry_request_with_remaining_timeout()
.await
.expect("first retry should retain the unused timeout budget");
let first_remaining = retry.options.timeout.expect("configured timeout should remain present");
assert!(first_remaining <= Duration::from_secs(60));
assert!(first_remaining > Duration::from_secs(59));
let retry_task = HealTask::from_request(retry, storage);
*retry_task.task_start_instant.write().await = Some(Instant::now() - Duration::from_secs(20));
let second_retry = retry_task
.retry_request_with_remaining_timeout()
.await
.expect("second retry should retain only the unused aggregate budget");
let second_remaining = second_retry
.options
.timeout
.expect("configured timeout should remain present");
assert!(second_remaining <= Duration::from_secs(40));
assert!(second_remaining > Duration::from_secs(39));
}
#[test] #[test]
fn format_result_requires_every_requested_target_to_be_ok() { fn format_result_requires_every_requested_target_to_be_ok() {
let result = HealResultItem { let result = HealResultItem {
+30 -33
View File
@@ -487,21 +487,22 @@ pub fn record_get_object_completion(total_duration_secs: f64, response_size_byte
/// Record the streaming strategy chosen for a GetObject response body. /// Record the streaming strategy chosen for a GetObject response body.
#[inline(always)] #[inline(always)]
pub fn record_get_object_stream_strategy(strategy: &'static str, buffer_size_bytes: usize, response_size_bytes: i64) { pub fn record_get_object_stream_strategy(strategy: &str, buffer_size_bytes: usize, response_size_bytes: i64) {
if !get_stage_metrics_enabled() { if !get_stage_metrics_enabled() {
return; return;
} }
counter!("rustfs_io_get_object_stream_strategy_total", "strategy" => strategy).increment(1); counter!("rustfs_io_get_object_stream_strategy_total", "strategy" => strategy.to_string()).increment(1);
histogram!("rustfs_io_get_object_stream_buffer_size_bytes", "strategy" => strategy).record(usize_to_f64(buffer_size_bytes)); histogram!("rustfs_io_get_object_stream_buffer_size_bytes", "strategy" => strategy.to_string())
histogram!("rustfs_io_get_object_stream_response_size_bytes", "strategy" => strategy) .record(usize_to_f64(buffer_size_bytes));
histogram!("rustfs_io_get_object_stream_response_size_bytes", "strategy" => strategy.to_string())
.record(i64_non_negative_to_f64(response_size_bytes)); .record(i64_non_negative_to_f64(response_size_bytes));
} }
/// Record the response-body handoff shape from a GetObject reader into the S3 streaming body. /// Record the response-body handoff shape from a GetObject reader into the S3 streaming body.
#[inline(always)] #[inline(always)]
pub fn record_get_object_response_handoff( pub fn record_get_object_response_handoff(
strategy: &'static str, strategy: &str,
buffer_source: &'static str, buffer_source: &str,
buffer_size_bytes: usize, buffer_size_bytes: usize,
response_size_bytes: i64, response_size_bytes: i64,
duration_secs: f64, duration_secs: f64,
@@ -511,26 +512,26 @@ pub fn record_get_object_response_handoff(
} }
counter!( counter!(
"rustfs_io_get_object_response_handoff_total", "rustfs_io_get_object_response_handoff_total",
"strategy" => strategy, "strategy" => strategy.to_string(),
"buffer_source" => buffer_source "buffer_source" => buffer_source.to_string()
) )
.increment(1); .increment(1);
histogram!( histogram!(
"rustfs_io_get_object_response_handoff_buffer_size_bytes", "rustfs_io_get_object_response_handoff_buffer_size_bytes",
"strategy" => strategy, "strategy" => strategy.to_string(),
"buffer_source" => buffer_source "buffer_source" => buffer_source.to_string()
) )
.record(usize_to_f64(buffer_size_bytes)); .record(usize_to_f64(buffer_size_bytes));
histogram!( histogram!(
"rustfs_io_get_object_response_handoff_response_size_bytes", "rustfs_io_get_object_response_handoff_response_size_bytes",
"strategy" => strategy, "strategy" => strategy.to_string(),
"buffer_source" => buffer_source "buffer_source" => buffer_source.to_string()
) )
.record(i64_non_negative_to_f64(response_size_bytes)); .record(i64_non_negative_to_f64(response_size_bytes));
histogram!( histogram!(
"rustfs_io_get_object_response_handoff_duration_seconds", "rustfs_io_get_object_response_handoff_duration_seconds",
"strategy" => strategy, "strategy" => strategy.to_string(),
"buffer_source" => buffer_source "buffer_source" => buffer_source.to_string()
) )
.record(duration_secs); .record(duration_secs);
record_get_object_response_handoff_duration("s3_handler", duration_secs); record_get_object_response_handoff_duration("s3_handler", duration_secs);
@@ -538,18 +539,14 @@ pub fn record_get_object_response_handoff(
/// Record ReaderStream capacity chosen for GetObject handoff. /// Record ReaderStream capacity chosen for GetObject handoff.
#[inline(always)] #[inline(always)]
pub fn record_get_object_reader_stream_buffer_size( pub fn record_get_object_reader_stream_buffer_size(strategy: &str, buffer_source: &str, buffer_size_bytes: usize) {
strategy: &'static str,
buffer_source: &'static str,
buffer_size_bytes: usize,
) {
if !get_stage_metrics_enabled() { if !get_stage_metrics_enabled() {
return; return;
} }
histogram!( histogram!(
"rustfs_io_get_object_reader_stream_buffer_size_bytes", "rustfs_io_get_object_reader_stream_buffer_size_bytes",
"strategy" => strategy, "strategy" => strategy.to_string(),
"buffer_source" => buffer_source "buffer_source" => buffer_source.to_string()
) )
.record(usize_to_f64(buffer_size_bytes)); .record(usize_to_f64(buffer_size_bytes));
} }
@@ -557,8 +554,8 @@ pub fn record_get_object_reader_stream_buffer_size(
/// Record ReaderStream poll outcomes for GetObject handoff attribution. /// Record ReaderStream poll outcomes for GetObject handoff attribution.
#[inline(always)] #[inline(always)]
pub fn record_get_object_reader_stream_poll( pub fn record_get_object_reader_stream_poll(
strategy: &'static str, strategy: &str,
buffer_source: &'static str, buffer_source: &str,
outcome: &'static str, outcome: &'static str,
remaining_before: usize, remaining_before: usize,
bytes: usize, bytes: usize,
@@ -570,36 +567,36 @@ pub fn record_get_object_reader_stream_poll(
let bytes = u64::try_from(bytes).unwrap_or(u64::MAX); let bytes = u64::try_from(bytes).unwrap_or(u64::MAX);
counter!( counter!(
"rustfs_io_get_object_reader_stream_poll_total", "rustfs_io_get_object_reader_stream_poll_total",
"strategy" => strategy, "strategy" => strategy.to_string(),
"buffer_source" => buffer_source, "buffer_source" => buffer_source.to_string(),
"outcome" => outcome "outcome" => outcome
) )
.increment(1); .increment(1);
counter!( counter!(
"rustfs_io_get_object_reader_stream_poll_bytes_total", "rustfs_io_get_object_reader_stream_poll_bytes_total",
"strategy" => strategy, "strategy" => strategy.to_string(),
"buffer_source" => buffer_source, "buffer_source" => buffer_source.to_string(),
"outcome" => outcome "outcome" => outcome
) )
.increment(bytes); .increment(bytes);
histogram!( histogram!(
"rustfs_io_get_object_reader_stream_poll_remaining_bytes", "rustfs_io_get_object_reader_stream_poll_remaining_bytes",
"strategy" => strategy, "strategy" => strategy.to_string(),
"buffer_source" => buffer_source, "buffer_source" => buffer_source.to_string(),
"outcome" => outcome "outcome" => outcome
) )
.record(usize_to_f64(remaining_before)); .record(usize_to_f64(remaining_before));
histogram!( histogram!(
"rustfs_io_get_object_reader_stream_poll_bytes", "rustfs_io_get_object_reader_stream_poll_bytes",
"strategy" => strategy, "strategy" => strategy.to_string(),
"buffer_source" => buffer_source, "buffer_source" => buffer_source.to_string(),
"outcome" => outcome "outcome" => outcome
) )
.record(usize_to_f64(bytes as usize)); .record(usize_to_f64(bytes as usize));
histogram!( histogram!(
"rustfs_io_get_object_reader_stream_poll_duration_seconds", "rustfs_io_get_object_reader_stream_poll_duration_seconds",
"strategy" => strategy, "strategy" => strategy.to_string(),
"buffer_source" => buffer_source, "buffer_source" => buffer_source.to_string(),
"outcome" => outcome "outcome" => outcome
) )
.record(duration_secs); .record(duration_secs);
+1 -4
View File
@@ -31,10 +31,7 @@ pub struct KeystoneClient {
admin_password: Option<String>, admin_password: Option<String>,
admin_project: Option<String>, admin_project: Option<String>,
admin_domain: String, admin_domain: String,
#[allow( #[allow(dead_code)]
dead_code,
reason = "TLS verification flag parsed from config; the reqwest client is built before it is consulted, so nothing reads it back (backlog#1823)"
)]
verify_ssl: bool, verify_ssl: bool,
/// Request timeout applied to the underlying HTTP client. /// Request timeout applied to the underlying HTTP client.
timeout: std::time::Duration, timeout: std::time::Duration,
+1 -4
View File
@@ -20,10 +20,7 @@ use tracing::{debug, info};
/// Maps Keystone identities to RustFS concepts /// Maps Keystone identities to RustFS concepts
pub struct KeystoneIdentityMapper { pub struct KeystoneIdentityMapper {
#[allow( #[allow(dead_code)]
dead_code,
reason = "keeps the Keystone client alive for the mapper's lifetime; the mapping paths do not call through it yet (backlog#1823)"
)]
client: Arc<KeystoneClient>, client: Arc<KeystoneClient>,
role_policy_map: HashMap<String, String>, role_policy_map: HashMap<String, String>,
enable_tenant_prefix: bool, enable_tenant_prefix: bool,
-58
View File
@@ -293,15 +293,6 @@ enum StrictVaultAuthMethod {
#[serde(default)] #[serde(default)]
refresh_safety_window_secs: Option<u64>, refresh_safety_window_secs: Option<u64>,
}, },
Kubernetes {
role: String,
#[serde(default)]
mount: Option<String>,
#[serde(default)]
jwt_path: Option<std::path::PathBuf>,
#[serde(default)]
refresh_safety_window_secs: Option<u64>,
},
TokenFile { TokenFile {
path: std::path::PathBuf, path: std::path::PathBuf,
#[serde(default)] #[serde(default)]
@@ -328,17 +319,6 @@ impl From<StrictVaultAuthMethod> for VaultAuthMethod {
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()), mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_APPROLE_MOUNT.to_string()),
refresh_safety_window_secs, refresh_safety_window_secs,
}, },
StrictVaultAuthMethod::Kubernetes {
role,
mount,
jwt_path,
refresh_safety_window_secs,
} => Self::Kubernetes {
role,
mount: mount.unwrap_or_else(|| crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()),
jwt_path: jwt_path.unwrap_or_else(|| std::path::PathBuf::from(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH)),
refresh_safety_window_secs,
},
StrictVaultAuthMethod::TokenFile { StrictVaultAuthMethod::TokenFile {
path, path,
poll_interval_secs, poll_interval_secs,
@@ -519,7 +499,6 @@ impl From<&KmsConfig> for KmsConfigSummary {
auth_method_type: match &vault_config.auth_method { auth_method_type: match &vault_config.auth_method {
VaultAuthMethod::Token { .. } => "token".to_string(), VaultAuthMethod::Token { .. } => "token".to_string(),
VaultAuthMethod::AppRole { .. } => "approle".to_string(), VaultAuthMethod::AppRole { .. } => "approle".to_string(),
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(), VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
}, },
has_stored_credentials: true, has_stored_credentials: true,
@@ -534,7 +513,6 @@ impl From<&KmsConfig> for KmsConfigSummary {
auth_method_type: match &vault_config.auth_method { auth_method_type: match &vault_config.auth_method {
VaultAuthMethod::Token { .. } => "token".to_string(), VaultAuthMethod::Token { .. } => "token".to_string(),
VaultAuthMethod::AppRole { .. } => "approle".to_string(), VaultAuthMethod::AppRole { .. } => "approle".to_string(),
VaultAuthMethod::Kubernetes { .. } => "kubernetes".to_string(),
VaultAuthMethod::TokenFile { .. } => "token_file".to_string(), VaultAuthMethod::TokenFile { .. } => "token_file".to_string(),
}, },
has_stored_credentials: true, has_stored_credentials: true,
@@ -923,42 +901,6 @@ mod tests {
assert!(request.to_kms_config().validate().is_ok()); assert!(request.to_kms_config().validate().is_ok());
} }
/// The admin API reaches Kubernetes auth with the role alone; the mount and
/// the projected token path fall back to the cluster defaults, so a Tenant
/// manifest carries no credential and no cluster-specific paths.
#[test]
fn test_deserialize_vault_configure_request_accepts_kubernetes_auth() {
let raw = serde_json::json!({
"backend_type": "vault-transit",
"address": "https://vault.example.com:8200",
"mount_path": "rustfs",
"auth_method": { "Kubernetes": { "role": "rustfs" } }
});
let request: ConfigureKmsRequest = serde_json::from_value(raw).expect("kubernetes auth should deserialize");
let config = request.to_kms_config();
config.validate().expect("kubernetes auth must validate");
let vault = config.vault_transit_config().expect("vault transit backend config");
let VaultAuthMethod::Kubernetes {
role, mount, jwt_path, ..
} = &vault.auth_method
else {
panic!("expected Kubernetes auth, got {:?}", vault.auth_method);
};
assert_eq!(role, "rustfs");
assert_eq!(mount, crate::config::DEFAULT_VAULT_KUBERNETES_MOUNT);
assert_eq!(jwt_path, std::path::Path::new(crate::config::DEFAULT_VAULT_KUBERNETES_JWT_PATH));
let unknown_field = serde_json::json!({
"backend_type": "vault-transit",
"address": "https://vault.example.com:8200",
"auth_method": { "Kubernetes": { "role": "rustfs", "service_account": "rustfs" } }
});
serde_json::from_value::<ConfigureKmsRequest>(unknown_field)
.expect_err("an unknown auth field must be rejected rather than silently dropped");
}
#[test] #[test]
fn test_deserialize_aws_configure_request_accepts_type_aliases() { fn test_deserialize_aws_configure_request_accepts_type_aliases() {
for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] { for backend_type in ["AWS", "AwsKms", "aws", "aws-kms", "aws_kms"] {
-1
View File
@@ -550,7 +550,6 @@ impl VaultKmsClient {
address: config.address.clone(), address: config.address.clone(),
namespace: config.namespace.clone(), namespace: config.namespace.clone(),
attempt_timeout: kms_config.effective_timeout(), attempt_timeout: kms_config.effective_timeout(),
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
}; };
let source = token_source_for(&config.auth_method, &settings)?; let source = token_source_for(&config.auth_method, &settings)?;
let policy = VaultCredentialPolicy::from_kms_config( let policy = VaultCredentialPolicy::from_kms_config(
+5 -271
View File
@@ -326,97 +326,6 @@ impl fmt::Debug for AppRoleLogin {
} }
} }
/// Token source for [`VaultAuthMethod::Kubernetes`]: exchanges the pod's
/// projected ServiceAccount token for a lease-bound Vault token.
///
/// The JWT is re-read on every login because the kubelet rotates a projected
/// token well inside the pod's lifetime; caching it would strand the source on
/// an expired assertion once the current Vault token can no longer be renewed.
///
/// Unlike [`TokenFileSource`], the file mode is not checked: the kubelet owns
/// the projected token and mounts it world-readable by default, so rejecting
/// group/other bits would refuse every standard pod rather than catch a
/// deployment error.
pub(crate) struct KubernetesLogin {
/// Unauthenticated client used only for the login exchange.
login_client: VaultClient,
mount: String,
role: String,
jwt_path: PathBuf,
}
impl KubernetesLogin {
pub(crate) fn new(settings: &VaultConnectionSettings, mount: String, role: String, jwt_path: PathBuf) -> Result<Self> {
Ok(Self {
login_client: settings.build_login_client()?,
mount,
role,
jwt_path,
})
}
/// Read the ServiceAccount token for one login attempt.
///
/// Mirrors [`AppRoleLogin::resolve_secret_id`]: a read failure is fatal for
/// the attempt but the refresh loop keeps retrying, so a token the kubelet
/// has not projected yet heals the source without a restart.
async fn resolve_jwt(&self) -> AttemptResult<SecretString> {
let mut raw = tokio::fs::read_to_string(&self.jwt_path)
.await
.map_err(|error| AttemptError {
class: ErrorClass::Fatal,
error: KmsError::configuration_error(format!(
"Failed to read Kubernetes ServiceAccount token {}: {error}",
self.jwt_path.display()
)),
})?;
let trimmed = raw.trim();
if trimmed.is_empty() {
raw.zeroize();
return Err(AttemptError {
class: ErrorClass::Fatal,
error: KmsError::configuration_error(format!(
"Kubernetes ServiceAccount token {} is empty",
self.jwt_path.display()
)),
});
}
let jwt = SecretString::new(trimmed.to_string());
raw.zeroize();
Ok(jwt)
}
}
#[async_trait]
impl TokenSource for KubernetesLogin {
async fn acquire(&self) -> AttemptResult<TokenLease> {
let jwt = self.resolve_jwt().await?;
let auth = vaultrs::auth::kubernetes::login(&self.login_client, &self.mount, &self.role, jwt.expose())
.await
.map_err(|error| attempt_error("Kubernetes login", error))?;
Ok(TokenLease::from_auth(auth))
}
async fn renew(&self, client: &VaultClient) -> AttemptResult<TokenLease> {
let auth = vaultrs::token::renew_self(client, None)
.await
.map_err(|error| attempt_error("token renewal", error))?;
Ok(TokenLease::from_auth(auth))
}
}
impl fmt::Debug for KubernetesLogin {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// The login client embeds Vault client settings and must stay out of
// Debug output; the role name is not a secret, and the JWT is never held.
f.debug_struct("KubernetesLogin")
.field("mount", &self.mount)
.field("role", &self.role)
.field("jwt_path", &self.jwt_path)
.finish_non_exhaustive()
}
}
/// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed /// Token source for [`VaultAuthMethod::TokenFile`]: reads an agent-managed
/// token file (for example a Vault Agent auto-auth sink). /// token file (for example a Vault Agent auto-auth sink).
/// ///
@@ -555,9 +464,6 @@ pub(crate) fn token_source_for(
secret_id.clone(), secret_id.clone(),
secret_id_file.clone(), secret_id_file.clone(),
)?)), )?)),
VaultAuthMethod::Kubernetes {
role, mount, jwt_path, ..
} => Ok(Box::new(KubernetesLogin::new(settings, mount.clone(), role.clone(), jwt_path.clone())?)),
VaultAuthMethod::TokenFile { VaultAuthMethod::TokenFile {
path, path,
poll_interval_secs, poll_interval_secs,
@@ -580,9 +486,6 @@ pub(crate) struct VaultConnectionSettings {
pub(crate) namespace: Option<String>, pub(crate) namespace: Option<String>,
/// Per-attempt HTTP timeout applied to the underlying reqwest client. /// Per-attempt HTTP timeout applied to the underlying reqwest client.
pub(crate) attempt_timeout: Duration, pub(crate) attempt_timeout: Duration,
/// Whether to accept an unverified Vault server certificate. Gated on
/// `allow_insecure_dev_defaults` by `KmsConfig::validate`.
pub(crate) skip_tls_verify: bool,
} }
impl VaultConnectionSettings { impl VaultConnectionSettings {
@@ -596,11 +499,6 @@ impl VaultConnectionSettings {
// operation-level retry policy. // operation-level retry policy.
settings_builder.timeout(Some(self.attempt_timeout)); settings_builder.timeout(Some(self.attempt_timeout));
settings_builder.token(token); settings_builder.token(token);
// Always set explicitly: left unset, vaultrs derives this from its own
// VAULT_SKIP_VERIFY variable, so a stray value in the environment would
// disable certificate verification behind the KMS configuration and its
// insecure-defaults gate.
settings_builder.verify(!self.skip_tls_verify);
if let Some(namespace) = &self.namespace { if let Some(namespace) = &self.namespace {
settings_builder.namespace(Some(namespace.clone())); settings_builder.namespace(Some(namespace.clone()));
@@ -653,10 +551,6 @@ impl VaultCredentialPolicy {
refresh_safety_window_secs: Some(secs), refresh_safety_window_secs: Some(secs),
.. ..
} }
| VaultAuthMethod::Kubernetes {
refresh_safety_window_secs: Some(secs),
..
}
| VaultAuthMethod::TokenFile { | VaultAuthMethod::TokenFile {
refresh_safety_window_secs: Some(secs), refresh_safety_window_secs: Some(secs),
.. ..
@@ -690,25 +584,15 @@ pub(crate) struct VaultClientHandle {
impl VaultClientHandle { impl VaultClientHandle {
/// Absolute expiry of this generation's token. /// Absolute expiry of this generation's token.
///
/// `lease.ttl` is built from the `lease_duration` the Vault server sent, so
/// a value too large to add to `issued_at` would panic on the bare `+`. A
/// TTL that cannot be represented is indistinguishable from no expiry, so it
/// collapses to `None` — the same answer already given for the zero-lease
/// tokens Vault issues, which keeps the token in use and still fully
/// validated by Vault on every call.
fn expires_at(&self) -> Option<Instant> { fn expires_at(&self) -> Option<Instant> {
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl)) self.lease.map(|lease| self.issued_at + lease.ttl)
} }
/// When the renewal task should refresh this generation: half the TTL, /// When the renewal task should refresh this generation: half the TTL,
/// leaving the second half as budget for retries before the fail-closed /// leaving the second half as budget for retries before the fail-closed
/// window is reached. /// window is reached.
///
/// Unrepresentable TTLs collapse to `None` as in [`Self::expires_at`],
/// leaving a token that never expires with nothing to renew.
fn renew_at(&self) -> Option<Instant> { fn renew_at(&self) -> Option<Instant> {
self.lease.and_then(|lease| self.issued_at.checked_add(lease.ttl / 2)) self.lease.map(|lease| self.issued_at + lease.ttl / 2)
} }
} }
@@ -778,7 +662,7 @@ impl VaultCredentialProvider {
let handle = self.current.load_full(); let handle = self.current.load_full();
if let Some(expires_at) = handle.expires_at() { if let Some(expires_at) = handle.expires_at() {
let now = Instant::now(); let now = Instant::now();
if self.inside_safety_window(now, expires_at) { if now + self.policy.safety_window >= expires_at {
return Err(KmsError::credentials_unavailable(format!( return Err(KmsError::credentials_unavailable(format!(
"Vault token (generation {}) is within {:?} of expiry and has not been refreshed; refusing to use it", "Vault token (generation {}) is within {:?} of expiry and has not been refreshed; refusing to use it",
handle.generation, self.policy.safety_window handle.generation, self.policy.safety_window
@@ -788,18 +672,6 @@ impl VaultCredentialProvider {
Ok(handle) Ok(handle)
} }
/// Whether the token expiring at `expires_at` is close enough to refuse.
///
/// `safety_window` reaches here from persisted configuration, so it is not
/// guaranteed to have passed this version's validation: a window too large
/// to add to the current instant would panic on the bare `+`. Such a window
/// means every token is always inside it, so saturating to "refuse" is both
/// the fail-closed answer and the one the arithmetic was reaching for.
fn inside_safety_window(&self, now: Instant, expires_at: Instant) -> bool {
now.checked_add(self.policy.safety_window)
.is_none_or(|deadline| deadline >= expires_at)
}
/// Publish the credential gauges for the generation currently installed. /// Publish the credential gauges for the generation currently installed.
/// ///
/// The fail-closed gauge re-evaluates the very gate /// The fail-closed gauge re-evaluates the very gate
@@ -811,7 +683,7 @@ impl VaultCredentialProvider {
let fail_closed = match handle.expires_at() { let fail_closed = match handle.expires_at() {
Some(expires_at) => { Some(expires_at) => {
metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64()); metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
self.inside_safety_window(now, expires_at) now + self.policy.safety_window >= expires_at
} }
// A generation without an expiry has no remaining TTL to report // A generation without an expiry has no remaining TTL to report
// and can never lapse, so it can never fail closed either. // and can never lapse, so it can never fail closed either.
@@ -988,7 +860,7 @@ impl Drop for CredentialTaskHandle {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::config::{DEFAULT_VAULT_KUBERNETES_MOUNT, REDACTED_SECRET}; use crate::config::REDACTED_SECRET;
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
const TEST_TOKEN: &str = "vault-token-debug-leak-canary"; const TEST_TOKEN: &str = "vault-token-debug-leak-canary";
@@ -999,7 +871,6 @@ mod tests {
address: "http://127.0.0.1:8200".to_string(), address: "http://127.0.0.1:8200".to_string(),
namespace: Some("team-namespace".to_string()), namespace: Some("team-namespace".to_string()),
attempt_timeout: Duration::from_secs(30), attempt_timeout: Duration::from_secs(30),
skip_tls_verify: false,
} }
} }
@@ -1186,143 +1057,6 @@ mod tests {
assert!(format!("{source:?}").contains("AppRoleLogin")); assert!(format!("{source:?}").contains("AppRoleLogin"));
} }
#[tokio::test]
async fn test_kubernetes_auth_method_maps_to_login_source() {
let settings = test_settings();
let source = token_source_for(&VaultAuthMethod::kubernetes("rustfs".to_string()), &settings)
.expect("kubernetes auth must map to a login source");
assert!(format!("{source:?}").contains("KubernetesLogin"));
}
/// `refresh_safety_window_secs` is operator-supplied and reaches the request
/// path from persisted configuration, so the fail-closed comparison must
/// survive a window too large to add to the current instant. Before the
/// checked arithmetic this panicked with "overflow when adding duration to
/// instant" on the first request after a lease-bearing login.
#[tokio::test]
async fn test_current_refuses_rather_than_panics_on_an_unrepresentable_safety_window() {
let (provider, _state) = scripted_provider(
Duration::from_secs(60),
true,
test_policy(Duration::from_secs(u64::MAX), Duration::from_secs(5)),
)
.await;
let error = provider
.current()
.expect_err("a window wider than any lease must refuse the token");
assert!(
matches!(error, KmsError::CredentialsUnavailable { .. }),
"expected CredentialsUnavailable, got {error:?}"
);
}
/// `lease_duration` is a bare u64 straight off the Vault response and forms
/// the other side of the same comparison, so an absurd one must not panic
/// either. It is indistinguishable from a non-expiring token, which is how
/// the zero-lease case already behaves.
#[tokio::test]
async fn test_an_unrepresentable_lease_is_treated_as_non_expiring() {
let (provider, _state) = scripted_provider(
Duration::from_secs(u64::MAX),
true,
test_policy(Duration::from_secs(30), Duration::from_secs(5)),
)
.await;
provider
.current()
.expect("a token whose expiry cannot be represented must stay usable");
}
/// The configured flag has to reach the HTTP client, not just the config
/// struct: every generation (authenticated and login) builds its own client,
/// and a Vault with a self-signed certificate fails the handshake unless
/// each one carries the setting.
#[test]
fn test_skip_tls_verify_reaches_every_vault_client_generation() {
for skip_tls_verify in [false, true] {
let settings = VaultConnectionSettings {
address: "https://vault.example.com:8200".to_string(),
namespace: None,
attempt_timeout: Duration::from_secs(30),
skip_tls_verify,
};
let authenticated = settings.build_client(TEST_TOKEN).expect("authenticated client must build");
assert_eq!(authenticated.settings.verify, !skip_tls_verify);
let login = settings.build_login_client().expect("login client must build");
assert_eq!(login.settings.verify, !skip_tls_verify);
}
}
/// vaultrs derives `verify` from its own VAULT_SKIP_VERIFY variable when the
/// builder leaves it unset, which would disable certificate verification
/// without passing the KMS insecure-defaults gate.
#[test]
fn test_vaultrs_skip_verify_env_cannot_override_the_configured_setting() {
temp_env::with_var("VAULT_SKIP_VERIFY", Some("true"), || {
let client = test_settings().build_client(TEST_TOKEN).expect("client must build");
assert!(
client.settings.verify,
"a stray VAULT_SKIP_VERIFY must not disable verification behind the KMS configuration"
);
});
}
/// The projected token is read fresh per login attempt and trimmed, so a
/// kubelet rotation is picked up without a restart and a trailing newline
/// does not corrupt the assertion sent to Vault.
#[tokio::test]
async fn test_kubernetes_login_rereads_and_trims_the_service_account_token() {
let dir = tempfile::tempdir().expect("temp dir");
let path = dir.path().join("token");
tokio::fs::write(&path, " first-jwt\n").await.expect("write token");
let login = KubernetesLogin::new(
&test_settings(),
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
"rustfs".to_string(),
path.clone(),
)
.expect("login source must build");
assert_eq!(login.resolve_jwt().await.expect("first read").expose(), "first-jwt");
tokio::fs::write(&path, "rotated-jwt").await.expect("rotate token");
assert_eq!(
login.resolve_jwt().await.expect("second read").expose(),
"rotated-jwt",
"a rotated projected token must be picked up without a restart"
);
}
/// The ServiceAccount token is re-read per attempt, so an unreadable or
/// empty one fails that attempt without reaching Vault; the refresh loop
/// keeps retrying, which is what lets a late projection heal the source.
#[tokio::test]
async fn test_kubernetes_login_rejects_an_unusable_service_account_token() {
let dir = tempfile::tempdir().expect("temp dir");
let missing = dir.path().join("absent-token");
let empty = dir.path().join("empty-token");
tokio::fs::write(&empty, " \n").await.expect("write empty token");
for (path, expected) in [(missing, "Failed to read"), (empty, "is empty")] {
let login =
KubernetesLogin::new(&test_settings(), DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(), "rustfs".to_string(), path)
.expect("login source must build");
let error = login
.acquire()
.await
.expect_err("an unusable ServiceAccount token must fail the attempt");
assert!(matches!(error.class, ErrorClass::Fatal));
assert!(error.error.to_string().contains(expected), "got {}", error.error);
}
}
#[tokio::test(start_paused = true)] #[tokio::test(start_paused = true)]
async fn test_renewal_task_renews_at_half_ttl() { async fn test_renewal_task_renews_at_half_ttl() {
let (provider, state) = scripted_provider( let (provider, state) = scripted_provider(
-1
View File
@@ -415,7 +415,6 @@ impl VaultTransitKmsClient {
address: config.address.clone(), address: config.address.clone(),
namespace: config.namespace.clone(), namespace: config.namespace.clone(),
attempt_timeout: kms_config.effective_timeout(), attempt_timeout: kms_config.effective_timeout(),
skip_tls_verify: config.tls.as_ref().is_some_and(|tls| tls.skip_verify),
}; };
let source = token_source_for(&config.auth_method, &settings)?; let source = token_source_for(&config.auth_method, &settings)?;
let policy = VaultCredentialPolicy::from_kms_config( let policy = VaultCredentialPolicy::from_kms_config(
-4
View File
@@ -450,10 +450,6 @@ impl VaultRestoreClient {
address: target.address.clone(), address: target.address.clone(),
namespace: target.namespace.clone(), namespace: target.namespace.clone(),
attempt_timeout: kms_config.effective_timeout(), attempt_timeout: kms_config.effective_timeout(),
// A restore target carries no TLS settings, so certificates are
// always verified: recovery is the last path that should accept an
// unauthenticated Vault.
skip_tls_verify: false,
}; };
let source = token_source_for(&target.auth_method, &settings)?; let source = token_source_for(&target.auth_method, &settings)?;
let policy = VaultCredentialPolicy::from_kms_config( let policy = VaultCredentialPolicy::from_kms_config(
+51 -292
View File
@@ -25,10 +25,6 @@ use url::Url;
pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS"; pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS";
pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION"; pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION";
pub const ENV_KMS_VAULT_ADDRESS: &str = "RUSTFS_KMS_VAULT_ADDRESS";
pub const ENV_KMS_VAULT_TOKEN: &str = "RUSTFS_KMS_VAULT_TOKEN";
pub const ENV_KMS_VAULT_NAMESPACE: &str = "RUSTFS_KMS_VAULT_NAMESPACE";
pub const ENV_KMS_VAULT_MOUNT_PATH: &str = "RUSTFS_KMS_VAULT_MOUNT_PATH";
pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY"; pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY";
pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT"; pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT";
pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX"; pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
@@ -39,9 +35,6 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE"; pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT"; pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE"; pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
pub const ENV_KMS_VAULT_KUBERNETES_ROLE: &str = "RUSTFS_KMS_VAULT_KUBERNETES_ROLE";
pub const ENV_KMS_VAULT_KUBERNETES_MOUNT: &str = "RUSTFS_KMS_VAULT_KUBERNETES_MOUNT";
pub const ENV_KMS_VAULT_KUBERNETES_JWT_PATH: &str = "RUSTFS_KMS_VAULT_KUBERNETES_JWT_PATH";
pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION"; pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL"; pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
/// Age in whole seconds beyond which a key is reported as due for rotation; /// Age in whole seconds beyond which a key is reported as due for rotation;
@@ -52,9 +45,6 @@ pub const ENV_KMS_ROTATION_MAX_WRAPS: &str = "RUSTFS_KMS_ROTATION_MAX_WRAPS";
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret"; pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata"; pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle"; pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
pub const DEFAULT_VAULT_KUBERNETES_MOUNT: &str = "kubernetes";
/// Where the kubelet projects a pod's ServiceAccount token by default.
pub const DEFAULT_VAULT_KUBERNETES_JWT_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token";
/// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior. /// Upper bound applied to `KmsConfig::timeout` when deriving backend behavior.
/// ///
@@ -94,14 +84,6 @@ fn default_vault_approle_mount() -> String {
DEFAULT_VAULT_APPROLE_MOUNT.to_string() DEFAULT_VAULT_APPROLE_MOUNT.to_string()
} }
fn default_vault_kubernetes_mount() -> String {
DEFAULT_VAULT_KUBERNETES_MOUNT.to_string()
}
fn default_vault_kubernetes_jwt_path() -> PathBuf {
PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH)
}
pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[ pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[
RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"), RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"),
RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"), RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"),
@@ -508,23 +490,6 @@ pub enum VaultAuthMethod {
#[serde(default)] #[serde(default)]
refresh_safety_window_secs: Option<u64>, refresh_safety_window_secs: Option<u64>,
}, },
/// Kubernetes authentication: the pod's ServiceAccount token is exchanged
/// for a lease-bound Vault token that is renewed in the background.
Kubernetes {
/// Vault role bound to this ServiceAccount.
role: String,
/// Kubernetes auth engine mount path.
#[serde(default = "default_vault_kubernetes_mount")]
mount: String,
/// Projected ServiceAccount token to present. Re-read on every login so
/// a token the kubelet rotates is picked up without a restart.
#[serde(default = "default_vault_kubernetes_jwt_path")]
jwt_path: PathBuf,
/// Fail-closed margin in seconds, as on `AppRole`. Defaults to the
/// per-attempt timeout.
#[serde(default)]
refresh_safety_window_secs: Option<u64>,
},
/// Agent-managed token file (for example a Vault Agent auto-auth sink): /// Agent-managed token file (for example a Vault Agent auto-auth sink):
/// the token is read from `path` and re-read periodically so a token /// the token is read from `path` and re-read periodically so a token
/// rotated by the agent is picked up without a restart. /// rotated by the agent is picked up without a restart.
@@ -555,16 +520,6 @@ impl VaultAuthMethod {
} }
} }
/// Kubernetes authentication with the default mount and projected token path.
pub fn kubernetes(role: String) -> Self {
Self::Kubernetes {
role,
mount: default_vault_kubernetes_mount(),
jwt_path: default_vault_kubernetes_jwt_path(),
refresh_safety_window_secs: None,
}
}
/// Agent-managed token file with the default poll interval. /// Agent-managed token file with the default poll interval.
pub fn token_file(path: PathBuf) -> Self { pub fn token_file(path: PathBuf) -> Self {
Self::TokenFile { Self::TokenFile {
@@ -593,20 +548,6 @@ impl fmt::Debug for VaultAuthMethod {
.field("mount", mount) .field("mount", mount)
.field("refresh_safety_window_secs", refresh_safety_window_secs) .field("refresh_safety_window_secs", refresh_safety_window_secs)
.finish(), .finish(),
// No redaction: the role and mount name a Vault binding, and the
// ServiceAccount token itself is never held on this type.
Self::Kubernetes {
role,
mount,
jwt_path,
refresh_safety_window_secs,
} => f
.debug_struct("Kubernetes")
.field("role", role)
.field("mount", mount)
.field("jwt_path", jwt_path)
.field("refresh_safety_window_secs", refresh_safety_window_secs)
.finish(),
Self::TokenFile { Self::TokenFile {
path, path,
poll_interval_secs, poll_interval_secs,
@@ -1087,12 +1028,50 @@ impl KmsConfig {
}); });
} }
KmsBackend::VaultKv2 => { KmsBackend::VaultKv2 => {
config.backend_config = let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
BackendConfig::VaultKv2(Box::new(vault_kv2_config_from_env(VaultCliOverrides::default())?)); let auth_method = vault_auth_method_from_env()?;
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
let mount_path = match get_env_opt_str("RUSTFS_KMS_VAULT_MOUNT_PATH") {
Some(path) => {
tracing::warn!(
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
);
path
}
None => default_vault_kv2_mount_path(),
};
config.backend_config = BackendConfig::VaultKv2(Box::new(VaultConfig {
address,
auth_method,
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
mount_path,
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
tls: vault_tls_config(skip_tls_verify),
}));
} }
KmsBackend::VaultTransit => { KmsBackend::VaultTransit => {
config.backend_config = let address = get_env_str("RUSTFS_KMS_VAULT_ADDRESS", "http://localhost:8200");
BackendConfig::VaultTransit(Box::new(vault_transit_config_from_env(VaultCliOverrides::default())?)); let auth_method = vault_auth_method_from_env()?;
let skip_tls_verify = get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false);
config.backend_config = BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
address,
auth_method,
namespace: get_env_opt_str("RUSTFS_KMS_VAULT_NAMESPACE"),
mount_path: get_env_str("RUSTFS_KMS_VAULT_MOUNT_PATH", "transit"),
metadata_kv_mount: get_env_str(
ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT,
DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT,
),
metadata_key_prefix: get_env_str(
ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX,
DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX,
),
tls: vault_tls_config(skip_tls_verify),
}));
} }
KmsBackend::Static => { KmsBackend::Static => {
// Read from file first, then fall back to direct env var // Read from file first, then fall back to direct env var
@@ -1223,78 +1202,6 @@ fn is_under_temp_dir(path: &Path) -> bool {
path.starts_with(std::env::temp_dir()) path.starts_with(std::env::temp_dir())
} }
/// Command-line values that take precedence over the matching environment
/// variables when assembling a Vault backend configuration.
///
/// Every field has a `RUSTFS_KMS_VAULT_*` equivalent that the CLI layer already
/// reads, so these are only set when the operator passed an explicit flag.
///
/// Deliberately not `Debug`: `token` holds the raw Vault token, and the
/// redacting `Debug` impls elsewhere in this module exist because a derived one
/// would print it. Denying the derive makes a future `{overrides:?}` a compile
/// error instead of a leak.
#[derive(Default, Clone, Copy)]
pub struct VaultCliOverrides<'a> {
pub address: Option<&'a str>,
pub token: Option<&'a str>,
pub mount_path: Option<&'a str>,
}
/// Assemble the Vault KV2 backend configuration from the environment.
///
/// Shared by [`KmsConfig::from_env`] and the server's command-line startup path
/// so both resolve the same auth method, namespace, TLS and mount settings.
pub fn vault_kv2_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultConfig> {
let mount_path = match overrides
.mount_path
.map(str::to_string)
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_MOUNT_PATH))
{
Some(path) => {
tracing::warn!(
"RUSTFS_KMS_VAULT_MOUNT_PATH is deprecated for the Vault KV2 backend: it never calls the Transit engine and the value is stored but unused"
);
path
}
None => default_vault_kv2_mount_path(),
};
Ok(VaultConfig {
address: vault_address_from_env(overrides.address),
auth_method: vault_auth_method_from_env(overrides.token)?,
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
mount_path,
kv_mount: get_env_str("RUSTFS_KMS_VAULT_KV_MOUNT", "secret"),
key_path_prefix: get_env_str("RUSTFS_KMS_VAULT_KEY_PREFIX", "rustfs/kms/keys"),
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
})
}
/// Assemble the Vault Transit backend configuration from the environment.
///
/// Companion to [`vault_kv2_config_from_env`]; see there for why both entry
/// points share it.
pub fn vault_transit_config_from_env(overrides: VaultCliOverrides<'_>) -> Result<VaultTransitConfig> {
Ok(VaultTransitConfig {
address: vault_address_from_env(overrides.address),
auth_method: vault_auth_method_from_env(overrides.token)?,
namespace: get_env_opt_str(ENV_KMS_VAULT_NAMESPACE),
mount_path: overrides
.mount_path
.map(str::to_string)
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_MOUNT_PATH, "transit")),
metadata_kv_mount: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT),
metadata_key_prefix: get_env_str(ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX),
tls: vault_tls_config(get_env_bool(ENV_KMS_VAULT_SKIP_TLS_VERIFY, false)),
})
}
fn vault_address_from_env(override_value: Option<&str>) -> String {
override_value
.map(str::to_string)
.unwrap_or_else(|| get_env_str(ENV_KMS_VAULT_ADDRESS, "http://localhost:8200"))
}
/// Resolve the Vault auth method from environment variables. /// Resolve the Vault auth method from environment variables.
/// ///
/// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication; /// Setting `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID` selects AppRole authentication;
@@ -1302,59 +1209,27 @@ fn vault_address_from_env(override_value: Option<&str>) -> String {
/// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE` /// (re-read on every login, mirroring the `RUSTFS_KMS_STATIC_SECRET_KEY_FILE`
/// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the /// precedent) or inline from `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID`, with the
/// file taking precedence. Without a role id the legacy token flow applies. /// file taking precedence. Without a role id the legacy token flow applies.
/// fn vault_auth_method_from_env() -> Result<VaultAuthMethod> {
/// `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` selects Kubernetes authentication, which
/// presents the pod's projected ServiceAccount token.
///
/// `token_override` carries a token supplied on the command line; it stands in
/// for `RUSTFS_KMS_VAULT_TOKEN` everywhere below, including the conflict checks,
/// so a flag and the variable it mirrors select the same method.
fn vault_auth_method_from_env(token_override: Option<&str>) -> Result<VaultAuthMethod> {
let token = token_override
.map(str::to_string)
.or_else(|| get_env_opt_str(ENV_KMS_VAULT_TOKEN));
let role_id = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID);
let kubernetes_role = get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_ROLE);
if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) { if let Some(token_file) = get_env_opt_str(ENV_KMS_VAULT_TOKEN_FILE) {
// A token file names one authoritative credential source; combining it // A token file names one authoritative credential source; combining it
// with another one would leave the effective identity ambiguous, so // with another one would leave the effective identity ambiguous, so
// that is a configuration error rather than a precedence rule. // that is a configuration error rather than a precedence rule.
for (name, configured) in [ if get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID).is_some() {
(ENV_KMS_VAULT_APPROLE_ROLE_ID, role_id.is_some()),
(ENV_KMS_VAULT_KUBERNETES_ROLE, kubernetes_role.is_some()),
(ENV_KMS_VAULT_TOKEN, token.is_some()),
] {
if configured {
return Err(KmsError::configuration_error(format!( return Err(KmsError::configuration_error(format!(
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {name}; configure exactly one Vault auth method" "{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
))); )));
} }
if get_env_opt_str("RUSTFS_KMS_VAULT_TOKEN").is_some() {
return Err(KmsError::configuration_error(format!(
"{ENV_KMS_VAULT_TOKEN_FILE} cannot be combined with RUSTFS_KMS_VAULT_TOKEN; configure exactly one Vault auth method"
)));
} }
return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file))); return Ok(VaultAuthMethod::token_file(PathBuf::from(token_file)));
} }
if let Some(role) = kubernetes_role { let Some(role_id) = get_env_opt_str(ENV_KMS_VAULT_APPROLE_ROLE_ID) else {
// Unlike a leftover static token, a second login method is never a
// stale remnant: both were configured deliberately and neither can be
// ranked over the other.
if role_id.is_some() {
return Err(KmsError::configuration_error(format!(
"{ENV_KMS_VAULT_KUBERNETES_ROLE} cannot be combined with {ENV_KMS_VAULT_APPROLE_ROLE_ID}; configure exactly one Vault auth method"
)));
}
return Ok(VaultAuthMethod::Kubernetes {
role,
mount: get_env_str(ENV_KMS_VAULT_KUBERNETES_MOUNT, DEFAULT_VAULT_KUBERNETES_MOUNT),
jwt_path: get_env_opt_str(ENV_KMS_VAULT_KUBERNETES_JWT_PATH)
.map_or_else(default_vault_kubernetes_jwt_path, PathBuf::from),
refresh_safety_window_secs: None,
});
}
let Some(role_id) = role_id else {
return Ok(VaultAuthMethod::Token { return Ok(VaultAuthMethod::Token {
token: token.unwrap_or_else(|| "dev-token".to_string()), token: get_env_str("RUSTFS_KMS_VAULT_TOKEN", "dev-token"),
}); });
}; };
@@ -1398,22 +1273,6 @@ fn validate_vault_auth_method(backend_name: &str, auth_method: &VaultAuthMethod)
} }
Ok(()) Ok(())
} }
VaultAuthMethod::Kubernetes {
role, mount, jwt_path, ..
} => {
if role.is_empty() {
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes role cannot be empty")));
}
if mount.is_empty() {
return Err(KmsError::configuration_error(format!("{backend_name} Kubernetes mount cannot be empty")));
}
if jwt_path.as_os_str().is_empty() {
return Err(KmsError::configuration_error(format!(
"{backend_name} Kubernetes ServiceAccount token path cannot be empty"
)));
}
Ok(())
}
VaultAuthMethod::TokenFile { VaultAuthMethod::TokenFile {
path, path,
poll_interval_secs, poll_interval_secs,
@@ -2117,106 +1976,6 @@ mod tests {
.expect("well-formed token file auth must validate"); .expect("well-formed token file auth must validate");
} }
/// A Kubernetes role alone configures the method: the credential is the
/// pod's projected ServiceAccount token, so nothing secret is in the
/// environment and the mount and token path fall back to the cluster
/// defaults.
#[test]
fn test_from_env_selects_kubernetes() {
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
(ENV_KMS_VAULT_ADDRESS, Some("https://vault.example.com")),
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
(ENV_KMS_VAULT_KUBERNETES_MOUNT, None),
(ENV_KMS_VAULT_KUBERNETES_JWT_PATH, None),
(ENV_KMS_VAULT_TOKEN, None),
(ENV_KMS_VAULT_TOKEN_FILE, None),
(ENV_KMS_VAULT_APPROLE_ROLE_ID, None),
],
|| {
let config = KmsConfig::from_env().expect("kms config should load from env");
let vault = config.vault_transit_config().expect("vault transit backend config");
let VaultAuthMethod::Kubernetes {
role,
mount,
jwt_path,
refresh_safety_window_secs,
} = &vault.auth_method
else {
panic!(
"a kubernetes role in the environment must select Kubernetes auth, got {:?}",
vault.auth_method
);
};
assert_eq!(role, "rustfs");
assert_eq!(mount, DEFAULT_VAULT_KUBERNETES_MOUNT);
assert_eq!(jwt_path, Path::new(DEFAULT_VAULT_KUBERNETES_JWT_PATH));
assert_eq!(refresh_safety_window_secs, &None);
},
);
}
#[test]
fn test_from_env_kubernetes_is_mutually_exclusive_with_other_auth() {
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("vault-transit")),
(ENV_KMS_VAULT_KUBERNETES_ROLE, Some("rustfs")),
(ENV_KMS_VAULT_APPROLE_ROLE_ID, Some("env-role-id")),
(ENV_KMS_VAULT_TOKEN, None),
(ENV_KMS_VAULT_TOKEN_FILE, None),
],
|| {
let error = KmsConfig::from_env().expect_err("kubernetes combined with approle must be rejected");
assert!(error.to_string().contains(ENV_KMS_VAULT_KUBERNETES_ROLE));
assert!(error.to_string().contains(ENV_KMS_VAULT_APPROLE_ROLE_ID));
},
);
}
#[test]
fn test_validate_rejects_bad_kubernetes_settings() {
let vault_config = |auth_method: VaultAuthMethod| KmsConfig {
backend: KmsBackend::VaultTransit,
backend_config: BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
address: "https://vault.example.com:8200".to_string(),
auth_method,
..Default::default()
})),
..Default::default()
};
let error = vault_config(VaultAuthMethod::kubernetes(String::new()))
.validate()
.expect_err("an empty kubernetes role must be rejected");
assert!(error.to_string().contains("role"), "got {error}");
let error = vault_config(VaultAuthMethod::Kubernetes {
role: "rustfs".to_string(),
mount: String::new(),
jwt_path: PathBuf::from(DEFAULT_VAULT_KUBERNETES_JWT_PATH),
refresh_safety_window_secs: None,
})
.validate()
.expect_err("an empty kubernetes mount must be rejected");
assert!(error.to_string().contains("mount"), "got {error}");
let error = vault_config(VaultAuthMethod::Kubernetes {
role: "rustfs".to_string(),
mount: DEFAULT_VAULT_KUBERNETES_MOUNT.to_string(),
jwt_path: PathBuf::new(),
refresh_safety_window_secs: None,
})
.validate()
.expect_err("an empty ServiceAccount token path must be rejected");
assert!(error.to_string().contains("token path"), "got {error}");
vault_config(VaultAuthMethod::kubernetes("rustfs".to_string()))
.validate()
.expect("well-formed kubernetes auth must validate");
}
/// Every KV2 read, write and listing is routed through `kv_mount`, so an /// Every KV2 read, write and listing is routed through `kv_mount`, so an
/// empty one names a path no Vault engine answers. The Transit backend /// empty one names a path no Vault engine answers. The Transit backend
/// already rejects its own empty mounts; this closes the same gap on the /// already rejects its own empty mounts; this closes the same gap on the
+2
View File
@@ -18,6 +18,8 @@
//! data encryption keys using master keys. It abstracts the encryption //! data encryption keys using master keys. It abstracts the encryption
//! operations so that different backends can share the same encryption logic. //! operations so that different backends can share the same encryption logic.
#![allow(dead_code)] // Trait methods may be used by implementations
use crate::error::{KmsError, Result}; use crate::error::{KmsError, Result};
use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummary}; use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummary};
use async_trait::async_trait; use async_trait::async_trait;
+2
View File
@@ -225,6 +225,8 @@ async fn nothing_readable_leaves_the_bundle_unwrapped() {
"artifact {} carries the raw on-disk record", "artifact {} carries the raw on-disk record",
artifact.path artifact.path
); );
// A cheap structural check too: an encrypted payload is not JSON.
assert_ne!(payload.first(), Some(&b'{'), "artifact {} looks like plaintext JSON", artifact.path);
} }
// The manifest itself is not encrypted, so assert directly that it carries // The manifest itself is not encrypted, so assert directly that it carries
-5
View File
@@ -37,11 +37,7 @@ hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
[dependencies] [dependencies]
hotpath.workspace = true hotpath.workspace = true
humantime.workspace = true humantime.workspace = true
http.workspace = true
hyper = { workspace = true, features = ["http2", "http1", "server"] } hyper = { workspace = true, features = ["http2", "http1", "server"] }
reqwest = { workspace = true, features = ["json"] }
rustfs-signer.workspace = true
s3s.workspace = true
jiff = { workspace = true, features = ["serde"] } jiff = { workspace = true, features = ["serde"] }
serde = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true, features = ["raw_value"] } serde_json = { workspace = true, features = ["raw_value"] }
@@ -53,4 +49,3 @@ doctest = false
[dev-dependencies] [dev-dependencies]
rmp-serde.workspace = true rmp-serde.workspace = true
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net"] }
-851
View File
@@ -1,851 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Admin API HTTP client for heal and scanner management (rustfs/backlog#1869).
//!
//! [`AdminClient`] speaks the `/rustfs/admin/v3` surface with S3 SigV4
//! request signing (the same scheme the server's admin router authenticates),
//! so `mc`-style tooling and automation can drive heal start/query/cancel and
//! read background-heal / scanner status without hand-rolling HTTP.
//!
//! Wire structs in this module mirror the server-side shapes
//! (`rustfs/src/admin/handlers/heal.rs`, `handlers/scanner.rs`,
//! `rustfs-common/src/heal_channel.rs`), following the madmin-go model where
//! the SDK owns its own copies and round-trip tests pin the encoding. Deeply
//! nested status payloads that the server composes from runtime types are
//! carried through as `serde_json::Value` and flattened maps rather than
//! duplicated field-for-field, so the client cannot silently drift on fields
//! it never interprets.
use crate::heal_commands::HealResultItem;
use http::Method;
use serde::{Deserialize, Serialize, de};
use std::time::Duration;
/// Default admin API path prefix on a RustFS endpoint.
pub const DEFAULT_ADMIN_API_PREFIX: &str = "/rustfs/admin";
/// Default SigV4 region when the server has no explicit region configured.
pub const DEFAULT_REGION: &str = "us-east-1";
/// Scan mode for a heal request, mirroring the server's numeric-or-name wire
/// encoding (`0` unknown/default, `1` normal, `2` deep).
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum HealScanMode {
/// Server default; behaves as [`HealScanMode::Normal`].
#[default]
Unknown,
/// Metadata-level checks only.
Normal,
/// Full bitrot verification while healing.
Deep,
}
impl HealScanMode {
fn wire_number(self) -> u8 {
match self {
Self::Unknown => 0,
Self::Normal => 1,
Self::Deep => 2,
}
}
fn from_wire_number(value: u8) -> Option<Self> {
match value {
0 => Some(Self::Unknown),
1 => Some(Self::Normal),
2 => Some(Self::Deep),
_ => None,
}
}
fn from_wire_name(value: &str) -> Option<Self> {
match value {
"unknown" => Some(Self::Unknown),
"normal" => Some(Self::Normal),
"deep" => Some(Self::Deep),
_ => None,
}
}
}
impl Serialize for HealScanMode {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
serializer.serialize_u8(self.wire_number())
}
}
impl<'de> Deserialize<'de> for HealScanMode {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
struct HealScanModeVisitor;
impl de::Visitor<'_> for HealScanModeVisitor {
type Value = HealScanMode;
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str("a heal scan mode number or name")
}
fn visit_u64<E: de::Error>(self, value: u64) -> Result<Self::Value, E> {
u8::try_from(value)
.ok()
.and_then(HealScanMode::from_wire_number)
.ok_or_else(|| E::custom(format!("unknown heal scan mode number: {value}")))
}
fn visit_str<E: de::Error>(self, value: &str) -> Result<Self::Value, E> {
HealScanMode::from_wire_name(value).ok_or_else(|| E::custom(format!("unknown heal scan mode name: {value}")))
}
}
deserializer.deserialize_any(HealScanModeVisitor)
}
}
/// Heal options for an admin heal request (mirror of the server body type).
/// Fields default on decode: a client should tolerate a server response whose
/// settings object omits fields it never set.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct HealOpts {
#[serde(default)]
pub recursive: bool,
#[serde(rename = "dryRun", default)]
pub dry_run: bool,
#[serde(default)]
pub remove: bool,
#[serde(default)]
pub recreate: bool,
#[serde(rename = "scanMode", default)]
pub scan_mode: HealScanMode,
#[serde(rename = "updateParity", default)]
pub update_parity: bool,
#[serde(rename = "nolock", default)]
pub no_lock: bool,
#[serde(rename = "pool", default)]
pub pool: Option<usize>,
#[serde(rename = "set", default)]
pub set: Option<usize>,
}
/// Successful heal start / path-scoped cancel response.
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct HealStartSuccess {
pub client_token: String,
pub client_address: String,
#[serde(default)]
pub start_time: String,
}
/// Heal task status response (query, cancel-with-token, start-then-poll).
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct HealTaskStatus {
/// `running` | `finished` | `stopped` | `notFound`.
pub summary: String,
/// Failure detail for stopped tasks; empty otherwise.
#[serde(rename = "detail", default)]
pub failure_detail: String,
#[serde(default)]
pub start_time: String,
#[serde(default)]
pub settings: HealOpts,
#[serde(default)]
pub items: Vec<HealResultItem>,
#[serde(default)]
pub truncated: bool,
/// Live progress snapshot; the exact shape is owned by the heal runtime.
#[serde(default)]
pub progress: Option<serde_json::Value>,
}
/// `POST /v3/background-heal/status` response. Known top-level fields are
/// typed; the flattened heal info and operations matrix pass through verbatim.
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct BackgroundHealStatus {
/// `disabled` | `uninitialized` | `idle` | `active` | `degraded`.
pub state: String,
#[serde(default)]
pub heal_queue_length: u64,
#[serde(default)]
pub heal_active_tasks: u64,
#[serde(default)]
pub cluster_status_complete: bool,
#[serde(default)]
pub progress: Option<serde_json::Value>,
/// Remaining wire fields (flattened `BackgroundHealInfo` plus the
/// priority-by-source operations matrix), carried verbatim.
#[serde(flatten)]
pub extra: serde_json::Map<String, serde_json::Value>,
}
/// `GET /v3/scanner/status` response, typed at the fields operators branch
/// on; everything else passes through verbatim.
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ScannerStatus {
pub enabled: bool,
/// `fresh` | `stale` | `unknown`; absent when the scanner never completed
/// a cycle.
#[serde(default)]
pub freshness: Option<ScannerFreshness>,
#[serde(flatten)]
pub extra: serde_json::Map<String, serde_json::Value>,
}
/// Freshness block of the scanner status response.
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ScannerFreshness {
/// `fresh` | `stale` | `unknown`.
pub state: String,
}
impl ScannerStatus {
/// Convenience accessor for the freshness state string.
pub fn freshness(&self) -> &str {
self.freshness
.as_ref()
.map(|freshness| freshness.state.as_str())
.unwrap_or("unknown")
}
}
/// Everything that can go wrong in an admin client call.
#[derive(Debug)]
pub enum AdminClientError {
/// The endpoint URL could not be parsed.
InvalidEndpoint(String),
/// Request build/send failed (DNS, connect, timeout, body read).
Transport(reqwest::Error),
/// The server answered a non-2xx status.
HttpStatus { status: u16, body: String },
/// The response body did not decode into the expected shape.
Decode { message: String },
}
impl std::fmt::Display for AdminClientError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::InvalidEndpoint(message) => write!(f, "invalid admin endpoint: {message}"),
Self::Transport(err) => write!(f, "admin request transport failure: {err}"),
Self::HttpStatus { status, body } => write!(f, "admin request failed with HTTP {status}: {body}"),
Self::Decode { message } => write!(f, "admin response decode failure: {message}"),
}
}
}
impl std::error::Error for AdminClientError {}
impl From<reqwest::Error> for AdminClientError {
fn from(err: reqwest::Error) -> Self {
Self::Transport(err)
}
}
/// A signed client for a RustFS admin API.
#[derive(Debug, Clone)]
pub struct AdminClient {
endpoint: reqwest::Url,
access_key: String,
secret_key: String,
session_token: String,
region: String,
api_prefix: String,
http: reqwest::Client,
}
impl AdminClient {
/// Build a client for `endpoint` (e.g. `http://127.0.0.1:9000`) using root
/// or admin credentials. Requests are SigV4-signed with the same scheme
/// the server's admin router authenticates.
pub fn new(endpoint: &str, access_key: &str, secret_key: &str) -> Result<Self, AdminClientError> {
let url = reqwest::Url::parse(endpoint).map_err(|err| AdminClientError::InvalidEndpoint(err.to_string()))?;
if url.host_str().is_none() {
return Err(AdminClientError::InvalidEndpoint("endpoint has no host".to_string()));
}
let http = reqwest::Client::builder()
.connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(30))
.build()
.map_err(AdminClientError::Transport)?;
Ok(Self {
endpoint: url,
access_key: access_key.to_string(),
secret_key: secret_key.to_string(),
session_token: String::new(),
region: DEFAULT_REGION.to_string(),
api_prefix: DEFAULT_ADMIN_API_PREFIX.to_string(),
http,
})
}
/// Attach an STS session token (signed as `x-amz-security-token`).
pub fn with_session_token(mut self, session_token: impl Into<String>) -> Self {
self.session_token = session_token.into();
self
}
/// Override the SigV4 region (defaults to `us-east-1`, matching a
/// region-less RustFS deployment).
pub fn with_region(mut self, region: impl Into<String>) -> Self {
self.region = region.into();
self
}
/// Override the admin API path prefix (defaults to `/rustfs/admin`).
pub fn with_api_prefix(mut self, prefix: impl Into<String>) -> Self {
self.api_prefix = prefix.into();
self
}
/// Start a heal. `bucket` empty and `prefix` empty heals the whole
/// deployment (requires `recursive` or a `pool`/`set` pair in `opts`,
/// enforced server-side); a bucket alone heals the bucket (the server
/// forces `recursive` for bucket heals).
pub async fn heal_start(
&self,
bucket: Option<&str>,
prefix: Option<&str>,
opts: &HealOpts,
force_start: bool,
) -> Result<HealStartSuccess, AdminClientError> {
let body = serde_json::to_vec(opts).map_err(|err| AdminClientError::Decode {
message: err.to_string(),
})?;
let mut query = Vec::new();
if force_start {
query.push(("forceStart", "true".to_string()));
}
self.post_json(&heal_path(bucket, prefix), &query, body).await
}
/// Query the status of the heal identified by `client_token` (the token
/// returned by [`Self::heal_start`]) at the path it was started on.
pub async fn heal_status(
&self,
bucket: Option<&str>,
prefix: Option<&str>,
client_token: &str,
) -> Result<HealTaskStatus, AdminClientError> {
self.post_json(&heal_path(bucket, prefix), &[("clientToken", client_token.to_string())], Vec::new())
.await
}
/// Stop a heal: with a `client_token` only that task is cancelled and its
/// final status returned; without one, every heal task at the path is
/// cancelled (the server answers with a start-success-shaped receipt).
pub async fn heal_stop(
&self,
bucket: Option<&str>,
prefix: Option<&str>,
client_token: Option<&str>,
) -> Result<HealStopOutcome, AdminClientError> {
let mut query = vec![("forceStop", "true".to_string())];
if let Some(token) = client_token {
query.push(("clientToken", token.to_string()));
}
match client_token {
Some(_) => {
let status: HealTaskStatus = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?;
Ok(HealStopOutcome::Stopped(status))
}
None => {
let success: HealStartSuccess = self.post_json(&heal_path(bucket, prefix), &query, Vec::new()).await?;
Ok(HealStopOutcome::PathStopped(success))
}
}
}
/// Cluster-aggregated background heal status.
pub async fn background_heal_status(&self) -> Result<BackgroundHealStatus, AdminClientError> {
self.get_json("/v3/background-heal/status").await
}
/// Data scanner status (enabled state, freshness, runtime config).
pub async fn scanner_status(&self) -> Result<ScannerStatus, AdminClientError> {
self.get_json("/v3/scanner/status").await
}
/// ILM expiry worker status. The payload is owned by the expiry
/// subsystem and still evolving; returned verbatim.
pub async fn ilm_expiry_status(&self) -> Result<serde_json::Value, AdminClientError> {
self.get_json("/v3/ilm/expiry/status").await
}
/// Durable replacement-recovery status (admin v4). The payload is owned
/// by the heal runtime; returned verbatim.
pub async fn replacement_recovery_status(&self) -> Result<serde_json::Value, AdminClientError> {
self.get_json("/v4/heal/replacement-recovery").await
}
/// Signed GET returning a decoded JSON body; escape hatch for endpoints
/// this client does not wrap yet.
pub async fn get_json<T: for<'de> Deserialize<'de>>(&self, path: &str) -> Result<T, AdminClientError> {
let url = self.url_for(path, &[])?;
let request = self.sign_and_build(Method::GET, url, Vec::new(), None).await?;
self.execute(request).await
}
/// Signed POST returning a decoded JSON body.
async fn post_json<T: for<'de> Deserialize<'de>>(
&self,
path: &str,
query: &[(&str, String)],
body: Vec<u8>,
) -> Result<T, AdminClientError> {
let content_type = if body.is_empty() { None } else { Some("application/json") };
let url = self.url_for(path, query)?;
let request = self.sign_and_build(Method::POST, url, body, content_type).await?;
self.execute(request).await
}
fn url_for(&self, path: &str, query: &[(&str, String)]) -> Result<reqwest::Url, AdminClientError> {
let mut url = self
.endpoint
.join(&format!("{}{}", self.api_prefix.trim_end_matches('/'), path))
.map_err(|err| AdminClientError::InvalidEndpoint(err.to_string()))?;
if !query.is_empty() {
let mut pairs = url.query_pairs_mut();
for (key, value) in query {
pairs.append_pair(key, value);
}
}
Ok(url)
}
/// Build a SigV4-signed request via the same signer the server trusts,
/// then hand the signed headers to the HTTP client. The signature covers
/// method, path, query, and an unsigned-payload marker — the same shape
/// RustFS itself sends for peer admin calls.
async fn sign_and_build(
&self,
method: Method,
url: reqwest::Url,
body: Vec<u8>,
content_type: Option<&str>,
) -> Result<reqwest::Request, AdminClientError> {
let authority = match (url.host_str(), url.port_or_known_default()) {
(Some(host), Some(port)) => format!("{host}:{port}"),
_ => return Err(AdminClientError::InvalidEndpoint("endpoint has no authority".to_string())),
};
let mut builder = http::Request::builder()
.method(method.clone())
.uri(url.as_str())
.header(http::header::HOST, &authority)
.header("x-amz-content-sha256", rustfs_signer::constants::UNSIGNED_PAYLOAD);
if let Some(content_type) = content_type {
builder = builder.header(http::header::CONTENT_TYPE, content_type);
}
let unsigned = builder
.body(s3s::Body::empty())
.map_err(|err| AdminClientError::InvalidEndpoint(format!("build request failed: {err}")))?;
let signed = rustfs_signer::sign_v4(
unsigned,
body.len() as i64,
&self.access_key,
&self.secret_key,
&self.session_token,
&self.region,
);
let mut request = self
.http
.request(method, url)
.body(body)
.build()
.map_err(AdminClientError::Transport)?;
let headers = request.headers_mut();
for (name, value) in signed.headers().iter() {
// HOST is owned by the HTTP client; the signed value above was
// built from the same URL authority, so they always agree.
if name == http::header::HOST {
continue;
}
headers.insert(name, value.clone());
}
Ok(request)
}
async fn execute<T: for<'de> Deserialize<'de>>(&self, request: reqwest::Request) -> Result<T, AdminClientError> {
let response = self.http.execute(request).await?;
let status = response.status();
let bytes = response.bytes().await?;
if !status.is_success() {
return Err(AdminClientError::HttpStatus {
status: status.as_u16(),
body: String::from_utf8_lossy(&bytes).into_owned(),
});
}
serde_json::from_slice(&bytes).map_err(|err| AdminClientError::Decode {
message: err.to_string(),
})
}
}
/// Response of [`AdminClient::heal_stop`]: cancelling a single tokened task
/// answers with that task's status, cancelling a whole path answers with a
/// start-success-shaped receipt.
#[derive(Debug, Clone)]
pub enum HealStopOutcome {
Stopped(HealTaskStatus),
PathStopped(HealStartSuccess),
}
fn heal_path(bucket: Option<&str>, prefix: Option<&str>) -> String {
match (bucket, prefix) {
(Some(bucket), Some(prefix)) if !bucket.is_empty() && !prefix.is_empty() => {
format!("/v3/heal/{}/{}", percent_encode_path_segment(bucket), percent_encode_path_segment(prefix))
}
(Some(bucket), Some(_)) | (Some(bucket), None) if !bucket.is_empty() => {
format!("/v3/heal/{}", percent_encode_path_segment(bucket))
}
_ => "/v3/heal/".to_string(),
}
}
/// Encode a single path segment (slashes are content, not separators, inside
/// bucket/prefix path params).
fn percent_encode_path_segment(segment: &str) -> String {
let mut out = String::with_capacity(segment.len());
for byte in segment.bytes() {
match byte {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => out.push(byte as char),
_ => out.push_str(&format!("%{byte:02X}")),
}
}
out
}
#[cfg(test)]
mod tests {
use super::{
AdminClient, AdminClientError, BackgroundHealStatus, HealOpts, HealScanMode, HealStartSuccess, HealTaskStatus,
ScannerStatus, heal_path, percent_encode_path_segment,
};
use serde_json::json;
use std::sync::{Arc, Mutex};
#[test]
fn heal_paths_cover_root_bucket_and_prefix() {
assert_eq!(heal_path(None, None), "/v3/heal/");
assert_eq!(heal_path(Some(""), Some("")), "/v3/heal/");
assert_eq!(heal_path(Some("bucket"), None), "/v3/heal/bucket");
assert_eq!(heal_path(Some("bucket"), Some("pre/fix")), "/v3/heal/bucket/pre%2Ffix");
}
#[test]
fn path_segments_percent_encode_reserved_characters() {
assert_eq!(percent_encode_path_segment("a b"), "a%20b");
assert_eq!(percent_encode_path_segment("a/b"), "a%2Fb");
assert_eq!(percent_encode_path_segment("ü"), "%C3%BC");
}
#[test]
fn heal_opts_round_trip_through_the_server_wire_shape() {
let opts = HealOpts {
recursive: true,
dry_run: false,
remove: true,
recreate: false,
scan_mode: HealScanMode::Deep,
update_parity: true,
no_lock: false,
pool: Some(1),
set: Some(2),
};
let wire = serde_json::to_value(&opts).unwrap();
assert_eq!(wire["scanMode"], json!(2), "the server body decodes scanMode as a number");
let back: HealOpts = serde_json::from_value(wire).unwrap();
assert_eq!(back.scan_mode, HealScanMode::Deep);
assert_eq!(back.pool, Some(1));
}
#[test]
fn heal_scan_mode_accepts_both_wire_encodings() {
assert_eq!(serde_json::from_value::<HealScanMode>(json!(1)).unwrap(), HealScanMode::Normal);
assert_eq!(serde_json::from_value::<HealScanMode>(json!("deep")).unwrap(), HealScanMode::Deep);
assert!(serde_json::from_value::<HealScanMode>(json!(9)).is_err());
assert!(serde_json::from_value::<HealScanMode>(json!("sideways")).is_err());
}
#[test]
fn heal_task_status_decodes_the_server_response_shape() {
let raw = json!({
"summary": "finished",
"detail": "",
"startTime": "2026-08-17T00:00:00Z",
"settings": {"recursive": false, "scanMode": 1},
"items": [{
"resultId": 1, "type": "object", "bucket": "b", "object": "o", "versionId": "", "detail": "",
"parityBlocks": 2, "dataBlocks": 2, "diskCount": 4, "setCount": 1,
"before": {"drives": []}, "after": {"drives": []}, "objectSize": 128
}],
"truncated": false
});
let status: HealTaskStatus = serde_json::from_value(raw).unwrap();
assert_eq!(status.summary, "finished");
assert_eq!(status.items.len(), 1);
assert_eq!(status.settings.scan_mode, HealScanMode::Normal);
assert!(status.progress.is_none());
}
#[test]
fn background_heal_status_types_known_fields_and_passes_the_rest_through() {
let raw = json!({
"state": "active",
"bitrotStartTime": "t",
"healQueueLength": 3,
"healActiveTasks": 1,
"healOperations": {"queueLength": 3},
"clusterStatusComplete": true
});
let status: BackgroundHealStatus = serde_json::from_value(raw).unwrap();
assert_eq!(status.state, "active");
assert_eq!(status.heal_queue_length, 3);
assert!(status.cluster_status_complete);
assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through");
}
#[test]
fn scanner_status_defaults_freshness_to_unknown() {
let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}});
let status: ScannerStatus = serde_json::from_value(raw).unwrap();
assert_eq!(status.freshness(), "stale");
let bare: ScannerStatus = serde_json::from_value(json!({"enabled": false})).unwrap();
assert_eq!(bare.freshness(), "unknown");
}
#[test]
fn invalid_endpoint_is_rejected_without_io() {
let err = AdminClient::new("not a url", "ak", "sk").unwrap_err();
assert!(matches!(err, AdminClientError::InvalidEndpoint(_)));
}
#[tokio::test]
async fn signed_requests_carry_sigv4_authorization_and_correct_target() {
let server = TestServer::spawn(r#"{"clientToken":"token-1","clientAddress":"127.0.0.1:9","startTime":"t"}"#, 200).await;
let client = AdminClient::new(&format!("http://{}", server.addr), "minioadmin", "minioadmin")
.expect("client builds against the test server");
let start: HealStartSuccess = client
.heal_start(
Some("bucket"),
None,
&HealOpts {
recursive: true,
..Default::default()
},
false,
)
.await
.expect("signed heal start decodes");
assert_eq!(start.client_token, "token-1");
let request = server.recorded();
assert_eq!(request.method, "POST");
assert_eq!(request.path, "/rustfs/admin/v3/heal/bucket");
assert!(!request.query.contains("forceStart"), "absent flags must not be sent");
let auth = request.header("authorization").expect("request must be signed");
assert!(auth.starts_with("AWS4-HMAC-SHA256"), "SigV4 scheme, got: {auth}");
assert!(auth.contains("Credential=minioadmin/"), "credentials must be in the Authorization header");
assert_eq!(
request.header("x-amz-content-sha256").as_deref(),
Some("UNSIGNED-PAYLOAD"),
"the client signs the same payload marker RustFS peer calls use"
);
assert_eq!(request.header("content-type").as_deref(), Some("application/json"));
assert!(request.body.contains("\"recursive\":true"));
}
#[tokio::test]
async fn query_sends_client_token_on_the_same_path() {
let body = r#"{"summary":"running","detail":"","settings":{"recursive":false},"items":[],"truncated":false}"#;
let server = TestServer::spawn(body, 200).await;
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
let status = client
.heal_status(Some("bucket"), None, "token-1")
.await
.expect("status decodes");
assert_eq!(status.summary, "running");
let request = server.recorded();
assert_eq!(request.path, "/rustfs/admin/v3/heal/bucket");
assert!(request.query.contains("clientToken=token-1"));
assert!(!request.query.contains("forceStop"));
}
#[tokio::test]
async fn stop_without_token_takes_the_path_cancel_branch() {
let server = TestServer::spawn(r#"{"clientToken":"path","clientAddress":"c","startTime":"t"}"#, 200).await;
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
let outcome = client.heal_stop(Some("bucket"), None, None).await.expect("path stop decodes");
assert!(matches!(outcome, super::HealStopOutcome::PathStopped(_)));
let request = server.recorded();
assert!(request.query.contains("forceStop=true"));
assert!(!request.query.contains("clientToken"));
}
#[tokio::test]
async fn http_error_status_maps_to_a_typed_error_with_body() {
let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await;
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
let err = client.scanner_status().await.unwrap_err();
match err {
AdminClientError::HttpStatus { status, body } => {
assert_eq!(status, 403);
assert!(body.contains("AccessDenied"));
}
other => panic!("expected HttpStatus, got {other:?}"),
}
}
#[tokio::test]
async fn malformed_success_body_maps_to_a_decode_error() {
let server = TestServer::spawn("not json", 200).await;
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
assert!(matches!(client.scanner_status().await.unwrap_err(), AdminClientError::Decode { .. }));
}
/// One recorded request, parsed off the wire with the minimum needed for
/// assertions: method, path, query, headers, body.
#[derive(Debug, Clone)]
struct RecordedRequest {
method: String,
path: String,
query: String,
headers: Vec<(String, String)>,
body: String,
}
impl RecordedRequest {
fn header(&self, name: &str) -> Option<String> {
self.headers
.iter()
.find(|(key, _)| key.eq_ignore_ascii_case(name))
.map(|(_, value)| value.clone())
}
}
/// Minimal HTTP/1.1 server: one canned response per connection, every
/// request recorded behind an `Arc<Mutex>`. Deliberately dependency-free —
/// the assertions only need the raw request bytes.
struct TestServer {
addr: std::net::SocketAddr,
requests: Arc<Mutex<Vec<RecordedRequest>>>,
}
impl TestServer {
async fn spawn(response_body: &'static str, status: u16) -> Self {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind ephemeral port");
let addr = listener.local_addr().expect("local addr");
let requests: Arc<Mutex<Vec<RecordedRequest>>> = Arc::new(Mutex::new(Vec::new()));
let recorded = requests.clone();
tokio::spawn(async move {
let reason = if status == 200 { "OK" } else { "Forbidden" };
let response = format!(
"HTTP/1.1 {status} {reason}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{response_body}",
response_body.len()
);
// Each request is a fresh connection (connection: close); a
// bounded loop serves every call a test makes while letting
// the task exit instead of lingering for the whole process.
for _ in 0..16 {
let Ok((mut stream, _)) = listener.accept().await else {
break;
};
let mut buffer = Vec::with_capacity(2048);
let mut chunk = [0u8; 2048];
// Read headers plus content-length body, or stop on close.
loop {
if let Some(end) = find_header_end(&buffer) {
let content_length = extract_content_length(&buffer[..end]);
if buffer.len() >= end + content_length {
break;
}
}
let n = match stream.read(&mut chunk).await {
Ok(0) | Err(_) => break,
Ok(n) => n,
};
buffer.extend_from_slice(&chunk[..n]);
if buffer.len() > 64 * 1024 {
break;
}
}
if let Some(request) = parse_request(&buffer) {
recorded.lock().expect("recorded lock").push(request);
}
let _ = stream.write_all(response.as_bytes()).await;
let _ = stream.shutdown().await;
}
});
Self { addr, requests }
}
fn recorded(&self) -> RecordedRequest {
self.requests
.lock()
.expect("recorded lock")
.last()
.cloned()
.expect("the client call must have produced one recorded request")
}
}
fn find_header_end(buffer: &[u8]) -> Option<usize> {
buffer.windows(4).position(|window| window == b"\r\n\r\n").map(|pos| pos + 4)
}
fn extract_content_length(headers: &[u8]) -> usize {
let text = String::from_utf8_lossy(headers).to_ascii_lowercase();
text.lines()
.find_map(|line| line.strip_prefix("content-length:"))
.and_then(|value| value.trim().parse().ok())
.unwrap_or(0)
}
fn parse_request(raw: &[u8]) -> Option<RecordedRequest> {
let end = find_header_end(raw)?;
let head = String::from_utf8_lossy(&raw[..end]);
let body = String::from_utf8_lossy(&raw[end..]).into_owned();
let mut lines = head.lines();
let request_line = lines.next()?;
let mut parts = request_line.split_whitespace();
let method = parts.next()?.to_string();
let target = parts.next()?.to_string();
let (path, query) = match target.split_once('?') {
Some((path, query)) => (path.to_string(), query.to_string()),
None => (target, String::new()),
};
let headers = lines
.filter_map(|line| line.split_once(':'))
.map(|(name, value)| (name.trim().to_string(), value.trim().to_string()))
.collect();
Some(RecordedRequest {
method,
path,
query,
headers,
body,
})
}
}
-2
View File
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
pub mod client;
pub mod group; pub mod group;
pub mod heal_commands; pub mod heal_commands;
pub mod health; pub mod health;
@@ -26,7 +25,6 @@ pub mod trace;
pub mod user; pub mod user;
pub mod utils; pub mod utils;
pub use client::*;
pub use group::*; pub use group::*;
pub use info_commands::*; pub use info_commands::*;
pub use policy::*; pub use policy::*;
+3 -3
View File
@@ -258,7 +258,7 @@ pub struct SRLDAPUser {
pub api_version: Option<String>, pub api_version: Option<String>,
} }
#[derive(Debug, Clone, Serialize, Deserialize, Default)] #[derive(Debug, Serialize, Deserialize, Default)]
pub struct SRIAMUser { pub struct SRIAMUser {
#[serde(rename = "accessKey", default)] #[serde(rename = "accessKey", default)]
pub access_key: String, pub access_key: String,
@@ -270,7 +270,7 @@ pub struct SRIAMUser {
pub api_version: Option<String>, pub api_version: Option<String>,
} }
#[derive(Debug, Clone, Serialize, Deserialize, Default)] #[derive(Debug, Serialize, Deserialize, Default)]
pub struct SRGroupInfo { pub struct SRGroupInfo {
#[serde(rename = "updateReq", default)] #[serde(rename = "updateReq", default)]
pub update_req: GroupAddRemove, pub update_req: GroupAddRemove,
@@ -346,7 +346,7 @@ pub struct SRCredInfo {
pub api_version: Option<String>, pub api_version: Option<String>,
} }
#[derive(Debug, Clone, Serialize, Deserialize, Default)] #[derive(Debug, Serialize, Deserialize, Default)]
pub struct SRIAMItem { pub struct SRIAMItem {
#[serde(default)] #[serde(default)]
pub r#type: String, pub r#type: String,
+1 -4
View File
@@ -40,10 +40,7 @@ impl RuleEvents for RuleView {
#[derive(Debug)] #[derive(Debug)]
struct CompiledRules { struct CompiledRules {
// Keep RulesMap (can be used later if you want to make more complex judgments during the snapshot reading phase) // Keep RulesMap (can be used later if you want to make more complex judgments during the snapshot reading phase)
#[allow( #[allow(dead_code)]
dead_code,
reason = "speculative retention: the comment above keeps it for richer snapshot-time judgements that no code performs yet (backlog#1823)"
)]
rules_map: RulesMap, rules_map: RulesMap,
// for RulesContainer::iter_rules // for RulesContainer::iter_rules
rule_views: Vec<RuleView>, rule_views: Vec<RuleView>,
+3
View File
@@ -187,6 +187,7 @@ impl RulesMap {
/// # Parameters /// # Parameters
/// * `event_name` - The EventName from which to remove the rule. /// * `event_name` - The EventName from which to remove the rule.
/// * `pattern` - The pattern of the rule to be removed. /// * `pattern` - The pattern of the rule to be removed.
#[allow(dead_code)]
pub fn remove_rule(&mut self, event_name: &EventName, pattern: &str) { pub fn remove_rule(&mut self, event_name: &EventName, pattern: &str) {
let mut remove_event = false; let mut remove_event = false;
@@ -208,6 +209,7 @@ impl RulesMap {
/// ///
/// # Parameters /// # Parameters
/// * `event_names` - A slice of EventNames to be removed. /// * `event_names` - A slice of EventNames to be removed.
#[allow(dead_code)]
pub fn remove_rules(&mut self, event_names: &[EventName]) { pub fn remove_rules(&mut self, event_names: &[EventName]) {
for event_name in event_names { for event_name in event_names {
self.map.remove(event_name); self.map.remove(event_name);
@@ -221,6 +223,7 @@ impl RulesMap {
/// * `event_name` - The EventName to update. /// * `event_name` - The EventName to update.
/// * `pattern` - The pattern of the rule to be updated. /// * `pattern` - The pattern of the rule to be updated.
/// * `target_id` - The TargetID to be added. /// * `target_id` - The TargetID to be added.
#[allow(dead_code)]
pub fn update_rule(&mut self, event_name: EventName, pattern: String, target_id: TargetID) { pub fn update_rule(&mut self, event_name: EventName, pattern: String, target_id: TargetID) {
self.map.entry(event_name).or_default().add(pattern, target_id); self.map.entry(event_name).or_default().add(pattern, target_id);
self.total_events_mask |= event_name.mask(); // Update only the relevant bitmask self.total_events_mask |= event_name.mask(); // Update only the relevant bitmask
+6
View File
@@ -18,6 +18,12 @@ use rustfs_targets::arn::TargetID;
/// TargetIDSet - A collection representation of TargetID. /// TargetIDSet - A collection representation of TargetID.
pub type TargetIdSet = HashSet<TargetID>; pub type TargetIdSet = HashSet<TargetID>;
/// Provides a Go-like method for TargetIdSet (can be implemented as trait if needed)
#[allow(dead_code)]
pub(crate) fn new_target_id_set(target_ids: Vec<TargetID>) -> TargetIdSet {
target_ids.into_iter().collect()
}
// HashSet has built-in clone, union, difference and other operations. // HashSet has built-in clone, union, difference and other operations.
// But the Go version of the method returns a new Set, and the HashSet method is usually iterator or modify itself. // But the Go version of the method returns a new Set, and the HashSet method is usually iterator or modify itself.
// If you need to exactly match Go's API style, you can add wrapper functions. // If you need to exactly match Go's API style, you can add wrapper functions.
+1
View File
@@ -17,6 +17,7 @@ use std::time::Duration;
/// Environment variable key for the global default metrics interval (seconds). /// Environment variable key for the global default metrics interval (seconds).
pub const ENV_DEFAULT_METRICS_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL_SEC"; pub const ENV_DEFAULT_METRICS_INTERVAL: &str = "RUSTFS_METRICS_DEFAULT_INTERVAL_SEC";
/// Default interval for metrics collection if not specified otherwise. /// Default interval for metrics collection if not specified otherwise.
#[allow(dead_code)]
pub const DEFAULT_METRICS_INTERVAL: Duration = Duration::from_secs(60); pub const DEFAULT_METRICS_INTERVAL: Duration = Duration::from_secs(60);
/// Environment variable key for cluster metrics interval (seconds). /// Environment variable key for cluster metrics interval (seconds).
+3
View File
@@ -145,18 +145,21 @@ impl PrometheusMetric {
} }
#[inline] #[inline]
#[allow(dead_code)]
pub fn with_label(mut self, key: &'static str, value: impl Into<Cow<'static, str>>) -> Self { pub fn with_label(mut self, key: &'static str, value: impl Into<Cow<'static, str>>) -> Self {
self.labels.push((key, value.into())); self.labels.push((key, value.into()));
self self
} }
#[inline] #[inline]
#[allow(dead_code)]
pub fn with_label_owned(mut self, key: &'static str, value: String) -> Self { pub fn with_label_owned(mut self, key: &'static str, value: String) -> Self {
self.labels.push((key, Cow::Owned(value))); self.labels.push((key, Cow::Owned(value)));
self self
} }
#[inline] #[inline]
#[allow(dead_code)]
pub fn with_labels(mut self, labels: Vec<(&'static str, Cow<'static, str>)>) -> Self { pub fn with_labels(mut self, labels: Vec<(&'static str, Cow<'static, str>)>) -> Self {
self.labels = labels; self.labels = labels;
self self
@@ -16,6 +16,7 @@ use crate::{MetricName, MetricNamespace, MetricSubsystem, MetricType};
use std::collections::HashSet; use std::collections::HashSet;
/// MetricDescriptor - Metric descriptors /// MetricDescriptor - Metric descriptors
#[allow(dead_code)]
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct MetricDescriptor { pub struct MetricDescriptor {
pub name: MetricName, pub name: MetricName,
@@ -51,6 +52,7 @@ impl MetricDescriptor {
} }
/// Get the full metric name in Prometheus style: <namespace>_<subsystem>_<name> /// Get the full metric name in Prometheus style: <namespace>_<subsystem>_<name>
#[allow(dead_code)]
pub fn get_full_metric_name(&self) -> String { pub fn get_full_metric_name(&self) -> String {
let namespace = self.namespace.as_str(); let namespace = self.namespace.as_str();
let formatted_subsystem = self.subsystem.as_str(); let formatted_subsystem = self.subsystem.as_str();
@@ -59,6 +61,7 @@ impl MetricDescriptor {
} }
/// check whether the label is in the label set /// check whether the label is in the label set
#[allow(dead_code)]
pub fn has_label(&mut self, label: &str) -> bool { pub fn has_label(&mut self, label: &str) -> bool {
self.get_label_set().contains(label) self.get_label_set().contains(label)
} }
@@ -13,6 +13,7 @@
// limitations under the License. // limitations under the License.
/// The metric name is the individual name of the metric /// The metric name is the individual name of the metric
#[allow(dead_code)]
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub enum MetricName { pub enum MetricName {
// The generic metric name // The generic metric name
@@ -442,6 +443,7 @@ pub enum MetricName {
} }
impl MetricName { impl MetricName {
#[allow(dead_code)]
pub fn as_str(&self) -> String { pub fn as_str(&self) -> String {
match self { match self {
Self::AuthTotal => "auth_total".to_string(), Self::AuthTotal => "auth_total".to_string(),
@@ -13,6 +13,7 @@
// limitations under the License. // limitations under the License.
/// MetricType - Indicates the type of indicator /// MetricType - Indicates the type of indicator
#[allow(dead_code)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MetricType { pub enum MetricType {
Counter, Counter,
@@ -22,6 +23,7 @@ pub enum MetricType {
impl MetricType { impl MetricType {
/// convert the metric type to a string representation /// convert the metric type to a string representation
#[allow(dead_code)]
pub fn as_str(&self) -> &'static str { pub fn as_str(&self) -> &'static str {
match self { match self {
Self::Counter => "counter", Self::Counter => "counter",
@@ -32,6 +34,7 @@ impl MetricType {
/// Convert the metric type to the Prometheus value type /// Convert the metric type to the Prometheus value type
/// In a Rust implementation, this might return the corresponding Prometheus Rust client type /// In a Rust implementation, this might return the corresponding Prometheus Rust client type
#[allow(dead_code)]
pub fn as_prom(&self) -> &'static str { pub fn as_prom(&self) -> &'static str {
match self { match self {
Self::Counter => "counter.", Self::Counter => "counter.",
@@ -56,6 +56,7 @@ pub fn new_gauge_md(
} }
/// create a new histogram indicator descriptor /// create a new histogram indicator descriptor
#[allow(dead_code)]
pub fn new_histogram_md( pub fn new_histogram_md(
name: impl Into<MetricName>, name: impl Into<MetricName>,
help: impl Into<String>, help: impl Into<String>,
@@ -19,6 +19,7 @@ pub enum MetricNamespace {
} }
impl MetricNamespace { impl MetricNamespace {
#[allow(dead_code)]
pub fn as_str(&self) -> &'static str { pub fn as_str(&self) -> &'static str {
match self { match self {
Self::RustFS => "rustfs", Self::RustFS => "rustfs",
@@ -14,6 +14,7 @@
/// Format the path to the metric name format /// Format the path to the metric name format
/// Replace '/' and '-' with '_' /// Replace '/' and '-' with '_'
#[allow(dead_code)]
pub fn format_path_to_metric_name(path: &str) -> String { pub fn format_path_to_metric_name(path: &str) -> String {
path.trim_start_matches('/').replace(['/', '-'], "_") path.trim_start_matches('/').replace(['/', '-'], "_")
} }

Some files were not shown because too many files have changed in this diff Show More