mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-23 04:39:04 +00:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5a5a759176 | |||
| 290662bc3a | |||
| 858cf20d9a | |||
| 66e63d20dd | |||
| 81ef2882df |
@@ -39,10 +39,11 @@ jobs:
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
- name: Checkout main branch
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: main
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -88,10 +89,11 @@ jobs:
|
||||
# either casing.
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
- name: Checkout main branch
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: main
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -176,10 +178,11 @@ jobs:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
- name: Checkout main branch
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: main
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
|
||||
@@ -307,38 +307,6 @@ pub struct DiskUsageStatus {
|
||||
pub snapshot_exists: bool,
|
||||
}
|
||||
|
||||
/// A bounded reconciliation record for an object whose logical size could not
|
||||
/// be trusted at the scanner boundary. The scanner persists these records in
|
||||
/// its cache; keeping the model here avoids a second, incompatible accounting
|
||||
/// representation in storage-facing crates.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct SizeReconciliationEntry {
|
||||
/// Stable object/version identity key (not a metrics label).
|
||||
pub key: String,
|
||||
pub bucket: String,
|
||||
pub object: String,
|
||||
#[serde(default)]
|
||||
pub version_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub generation: Option<String>,
|
||||
/// Structured reason label; raw metadata values must never be stored here.
|
||||
pub reason: String,
|
||||
#[serde(default)]
|
||||
pub physical_size: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub first_seen: u64,
|
||||
#[serde(default)]
|
||||
pub attempts: u32,
|
||||
}
|
||||
|
||||
/// Object scope refreshed by one scanner pass. Existing debts in this scope
|
||||
/// are removed before the pass's unresolved records are inserted.
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq)]
|
||||
pub struct SizeReconciliationScope {
|
||||
pub bucket: String,
|
||||
pub object: String,
|
||||
}
|
||||
|
||||
/// Size summary for a single object or group of objects
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct SizeSummary {
|
||||
@@ -368,16 +336,6 @@ pub struct SizeSummary {
|
||||
pub repl_target_stats: HashMap<String, ReplTargetSizeSummary>,
|
||||
/// Per-tier accounting, keyed by storage class or remote tier name
|
||||
pub tier_stats: HashMap<String, TierStats>,
|
||||
/// Size-resolution debts observed while scanning this summary.
|
||||
pub size_reconciliation: Vec<SizeReconciliationEntry>,
|
||||
/// True when the per-object summary exceeded its bounded debt buffer.
|
||||
/// Callers must retain prior ledger entries rather than treating the
|
||||
/// partial list as a complete refresh.
|
||||
pub size_reconciliation_truncated: bool,
|
||||
/// Object scopes refreshed by this summary. They let the durable ledger
|
||||
/// remove versions that resolved without allocating one key per healthy
|
||||
/// version on the hot path.
|
||||
pub reconciliation_scopes: Vec<SizeReconciliationScope>,
|
||||
}
|
||||
|
||||
/// Replication target size summary
|
||||
@@ -875,8 +833,7 @@ impl DataUsageEntry {
|
||||
///
|
||||
/// The canonical wire format is written by the hand-written map-encoded
|
||||
/// `Serialize` on the scanner-side `DataUsageCacheInfo`
|
||||
/// (`crates/scanner/src/data_usage_define.rs`), which carries the original 16
|
||||
/// fields plus an optional reconciliation field.
|
||||
/// (`crates/scanner/src/data_usage_define.rs`), which carries 16 fields.
|
||||
/// This type decodes only the shared subset and is deliberately not
|
||||
/// `Serialize`: a derived (array) encoding of this 6-field subset would
|
||||
/// corrupt the cache for scanner readers, so no write path may exist here.
|
||||
@@ -1820,51 +1777,6 @@ impl SizeSummary {
|
||||
entry.pending_count = entry.pending_count.saturating_add(stats.pending_count);
|
||||
entry.failed_count = entry.failed_count.saturating_add(stats.failed_count);
|
||||
}
|
||||
|
||||
for entry in &other.size_reconciliation {
|
||||
self.record_size_reconciliation(entry.clone());
|
||||
}
|
||||
self.size_reconciliation_truncated |= other.size_reconciliation_truncated;
|
||||
for scope in &other.reconciliation_scopes {
|
||||
self.record_reconciliation_scope(&scope.bucket, &scope.object);
|
||||
}
|
||||
}
|
||||
|
||||
/// Add one reconciliation debt, coalescing repeated observations in the
|
||||
/// same object summary. The scanner cache applies its own larger bound.
|
||||
pub fn record_size_reconciliation(&mut self, entry: SizeReconciliationEntry) {
|
||||
const MAX_SUMMARY_RECONCILIATION_ENTRIES: usize = 1024;
|
||||
if let Some(existing) = self.size_reconciliation.iter_mut().find(|value| value.key == entry.key) {
|
||||
existing.reason = entry.reason;
|
||||
existing.physical_size = entry.physical_size;
|
||||
existing.generation = entry.generation;
|
||||
existing.version_id = entry.version_id;
|
||||
return;
|
||||
}
|
||||
if self.size_reconciliation.len() < MAX_SUMMARY_RECONCILIATION_ENTRIES {
|
||||
self.size_reconciliation.push(entry);
|
||||
} else {
|
||||
self.size_reconciliation_truncated = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark one object scope as refreshed. Duplicate scopes are suppressed so
|
||||
/// merging summaries remains bounded and deterministic.
|
||||
pub fn record_reconciliation_scope(&mut self, bucket: &str, object: &str) {
|
||||
if !self
|
||||
.reconciliation_scopes
|
||||
.iter()
|
||||
.any(|scope| scope.bucket == bucket && scope.object == object)
|
||||
{
|
||||
if self.reconciliation_scopes.len() >= 1024 {
|
||||
self.size_reconciliation_truncated = true;
|
||||
return;
|
||||
}
|
||||
self.reconciliation_scopes.push(SizeReconciliationScope {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -61,6 +61,11 @@ mod get_codec_streaming_compat_test;
|
||||
#[cfg(test)]
|
||||
mod version_id_regression_test;
|
||||
|
||||
// Receiver-side replication LWW (rustfs/backlog#1953): stale inbound
|
||||
// replication metadata must not overwrite a newer local category state.
|
||||
#[cfg(test)]
|
||||
mod replication_lww_receiver_test;
|
||||
|
||||
// Data usage regression tests
|
||||
#[cfg(test)]
|
||||
mod data_usage_test;
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
#![cfg(test)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Receiver-side replication LWW over the wire (rustfs/backlog#1953, audit
|
||||
//! A4/P1-6).
|
||||
//!
|
||||
//! In an active-active topology both sites' metadata states arrive at the
|
||||
//! peer as authorized replication PUTs carrying per-category source
|
||||
//! timestamps (`x-rustfs-source-replication-tagging-timestamp` header
|
||||
//! family). Before the fix the receiver applied them unconditionally, so a
|
||||
//! stale delivery overwrote a newer local state and the two sites diverged
|
||||
//! permanently while both reported COMPLETED. This test drives one live
|
||||
//! `rustfs` server with simulated inbound replication PUTs for the same
|
||||
//! object version and asserts the newer tagging state wins regardless of
|
||||
//! delivery order, while a stale delivery still succeeds at the object level
|
||||
//! (a failure would loop through MRF re-delivering the stale value).
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
|
||||
const HDR_SOURCE_VERSION_ID: &str = "x-rustfs-source-version-id";
|
||||
const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
|
||||
const HDR_SOURCE_TAGGING_TIMESTAMP: &str = "x-rustfs-source-replication-tagging-timestamp";
|
||||
|
||||
const SOURCE_MTIME: &str = "2026-01-01T00:00:00Z";
|
||||
const T_STALE: &str = "2026-01-01T00:00:01Z";
|
||||
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
|
||||
const T_NEWER: &str = "2026-03-01T00:00:00Z";
|
||||
|
||||
/// Simulated inbound authorized replication PUT: same object version, tags and
|
||||
/// the source-authored tagging timestamp carried in transport headers.
|
||||
async fn inbound_replication_put(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
version_id: &str,
|
||||
tags: &str,
|
||||
tagging_timestamp: &str,
|
||||
) -> TestResult {
|
||||
let version_id = version_id.to_string();
|
||||
let tagging_timestamp = tagging_timestamp.to_string();
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"lww-e2e-body"))
|
||||
.tagging(tags)
|
||||
.customize()
|
||||
.mutate_request(move |req| {
|
||||
req.headers_mut().insert(HDR_SOURCE_REPLICATION_REQUEST, "true");
|
||||
req.headers_mut().insert(HDR_SOURCE_VERSION_ID, version_id.clone());
|
||||
req.headers_mut().insert(HDR_SOURCE_MTIME, SOURCE_MTIME);
|
||||
req.headers_mut()
|
||||
.insert(HDR_SOURCE_TAGGING_TIMESTAMP, tagging_timestamp.clone());
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn tag_value(client: &Client, bucket: &str, key: &str, version_id: &str, tag_key: &str) -> Option<String> {
|
||||
let tagging = client
|
||||
.get_object_tagging()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id(version_id)
|
||||
.send()
|
||||
.await
|
||||
.expect("object tagging should be readable");
|
||||
tagging
|
||||
.tag_set()
|
||||
.iter()
|
||||
.find(|tag| tag.key() == tag_key)
|
||||
.map(|tag| tag.value().to_string())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn receiver_lww_keeps_newer_tags_across_delivery_orders() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let client = env.create_s3_client();
|
||||
|
||||
let bucket = "replication-lww-receiver";
|
||||
let key = "object";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// First delivery establishes version V with tags stamped T_LOCAL.
|
||||
let version_id = uuid::Uuid::new_v4().to_string();
|
||||
inbound_replication_put(&client, bucket, key, &version_id, "site=local", T_LOCAL).await?;
|
||||
assert_eq!(
|
||||
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
|
||||
Some("local"),
|
||||
"the first delivery must establish the tagged version"
|
||||
);
|
||||
|
||||
// A stale delivery (older source timestamp) must succeed at the object
|
||||
// level but must NOT overwrite the newer tags.
|
||||
inbound_replication_put(&client, bucket, key, &version_id, "site=stale", T_STALE).await?;
|
||||
assert_eq!(
|
||||
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
|
||||
Some("local"),
|
||||
"a stale inbound delivery must not overwrite newer tags (rustfs/backlog#1953)"
|
||||
);
|
||||
|
||||
// A newer delivery still converges the version onto the newest state.
|
||||
inbound_replication_put(&client, bucket, key, &version_id, "site=newer", T_NEWER).await?;
|
||||
assert_eq!(
|
||||
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
|
||||
Some("newer"),
|
||||
"a newer inbound delivery must overwrite older tags"
|
||||
);
|
||||
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.version_id(&version_id)
|
||||
.send()
|
||||
.await?;
|
||||
env.delete_test_bucket(bucket).await.ok();
|
||||
Ok(())
|
||||
}
|
||||
@@ -3868,6 +3868,7 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
|
||||
actual_size,
|
||||
object_info.etag.clone().unwrap_or_default(),
|
||||
object_info.mod_time,
|
||||
&put_opts.internal,
|
||||
),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -472,6 +472,7 @@ pub(crate) fn replication_complete_multipart_options(
|
||||
actual_size: String,
|
||||
source_etag: String,
|
||||
source_mtime: Option<OffsetDateTime>,
|
||||
source_internal: &AdvancedPutOptions,
|
||||
) -> PutObjectOptions {
|
||||
let mut user_metadata = HashMap::new();
|
||||
insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size);
|
||||
@@ -484,6 +485,14 @@ pub(crate) fn replication_complete_multipart_options(
|
||||
// mtime must degrade to epoch so header() suppresses the header
|
||||
// instead of asserting the replication time as the object's mtime.
|
||||
source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH),
|
||||
// Carry the per-category LWW timestamps on the complete request as
|
||||
// well: the receiver's CompleteMultipartUpload options builder
|
||||
// parses the same headers, so the multipart transport gets the
|
||||
// same receiver-side LWW as the single-PUT transport
|
||||
// (rustfs/backlog#1953). Epoch values keep the headers suppressed.
|
||||
tagging_timestamp: source_internal.tagging_timestamp,
|
||||
retention_timestamp: source_internal.retention_timestamp,
|
||||
legalhold_timestamp: source_internal.legalhold_timestamp,
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
replication_request: true,
|
||||
..Default::default()
|
||||
@@ -663,20 +672,39 @@ mod tests {
|
||||
#[test]
|
||||
fn replication_complete_multipart_options_sets_actual_size() {
|
||||
let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp");
|
||||
let source_internal = AdvancedPutOptions {
|
||||
tagging_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_100).expect("valid test timestamp"),
|
||||
retention_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_200).expect("valid test timestamp"),
|
||||
legalhold_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_300).expect("valid test timestamp"),
|
||||
..Default::default()
|
||||
};
|
||||
let options = replication_complete_multipart_options(
|
||||
"1024".to_string(),
|
||||
"0123456789abcdef0123456789abcdef-3".to_string(),
|
||||
Some(source_mtime),
|
||||
&source_internal,
|
||||
);
|
||||
assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3");
|
||||
assert_eq!(options.internal.source_mtime, source_mtime);
|
||||
|
||||
// The complete request must carry the same per-category LWW timestamps
|
||||
// as the initiate request; the receiver reads them from the complete
|
||||
// headers (rustfs/backlog#1953).
|
||||
assert_eq!(options.internal.tagging_timestamp, source_internal.tagging_timestamp);
|
||||
assert_eq!(options.internal.retention_timestamp, source_internal.retention_timestamp);
|
||||
assert_eq!(options.internal.legalhold_timestamp, source_internal.legalhold_timestamp);
|
||||
|
||||
// Absent source mtime must degrade to epoch (header suppressed), not
|
||||
// the AdvancedPutOptions default of now_utc() — that default would
|
||||
// stamp the replication time as the replica's mtime and break the
|
||||
// multipart HEAD convergence.
|
||||
let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None);
|
||||
// multipart HEAD convergence. Unset category timestamps stay epoch so
|
||||
// header() keeps suppressing them.
|
||||
let options_no_mtime =
|
||||
replication_complete_multipart_options("1024".to_string(), String::new(), None, &AdvancedPutOptions::default());
|
||||
assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0);
|
||||
assert_eq!(options_no_mtime.internal.tagging_timestamp.unix_timestamp(), 0);
|
||||
assert_eq!(options_no_mtime.internal.retention_timestamp.unix_timestamp(), 0);
|
||||
assert_eq!(options_no_mtime.internal.legalhold_timestamp.unix_timestamp(), 0);
|
||||
|
||||
assert_eq!(
|
||||
get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(),
|
||||
|
||||
@@ -784,24 +784,6 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
|
||||
///
|
||||
/// # Returns
|
||||
/// A Result containing the BitrotWriterWrapper or an error
|
||||
/// Size hint handed to `DiskAPI::create_file` for a bitrot-wrapped shard.
|
||||
///
|
||||
/// A known length is grown by one checksum per shard so the on-disk file size
|
||||
/// matches what the bitrot writer emits. A negative length is the
|
||||
/// unknown-size sentinel (`HashReader::SIZE_PRESERVE_LAYER`, used by SSE and
|
||||
/// compression) and must be preserved: `RemoteDisk::create_file` forwards it
|
||||
/// in the `put_file_stream` query, and the receiver only treats `size > 0` as
|
||||
/// a fixed body length when locating the authenticated trailer. Clamping it
|
||||
/// to `0` would claim an empty body and misframe the stream. `0` stays `0`
|
||||
/// because a genuinely empty object still means an empty body.
|
||||
fn bitrot_create_file_size(length: i64, shard_size: usize, checksum_algo: &HashAlgorithm) -> i64 {
|
||||
if length <= 0 {
|
||||
return length;
|
||||
}
|
||||
let length = length as usize;
|
||||
(length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
|
||||
}
|
||||
|
||||
pub async fn create_bitrot_writer(
|
||||
is_inline_buffer: bool,
|
||||
disk: Option<&DiskStore>,
|
||||
@@ -814,7 +796,12 @@ pub async fn create_bitrot_writer(
|
||||
let writer = if is_inline_buffer {
|
||||
CustomWriter::new_inline_buffer()
|
||||
} else if let Some(disk) = disk {
|
||||
let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
|
||||
let length = if length > 0 {
|
||||
let length = length as usize;
|
||||
(length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
let file = disk.create_file("", volume, path, length).await?;
|
||||
#[cfg(feature = "hotpath")]
|
||||
@@ -833,25 +820,6 @@ mod tests {
|
||||
use rustfs_rio::ChunkReader;
|
||||
use std::collections::VecDeque;
|
||||
|
||||
#[test]
|
||||
fn bitrot_create_file_size_grows_known_length_by_checksums() {
|
||||
// 10 bytes over 4-byte shards = 3 shards, each followed by a 32-byte hash.
|
||||
assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::HighwayHash256), 10 + 3 * 32);
|
||||
assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::None), 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bitrot_create_file_size_keeps_empty_and_unknown_distinct() {
|
||||
assert_eq!(bitrot_create_file_size(0, 4, &HashAlgorithm::HighwayHash256), 0);
|
||||
// SSE/compression streams advertise SIZE_PRESERVE_LAYER (-1); the remote
|
||||
// put_file_stream receiver relies on a non-positive size to parse the auth
|
||||
// trailer from the stream tail, so the sentinel must survive untouched.
|
||||
assert_eq!(
|
||||
bitrot_create_file_size(rustfs_rio::HashReader::SIZE_PRESERVE_LAYER, 4, &HashAlgorithm::HighwayHash256),
|
||||
rustfs_rio::HashReader::SIZE_PRESERVE_LAYER
|
||||
);
|
||||
}
|
||||
|
||||
struct TestChunkReader {
|
||||
chunks: VecDeque<Bytes>,
|
||||
}
|
||||
|
||||
@@ -2318,6 +2318,57 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
fi.set_data_moved();
|
||||
}
|
||||
|
||||
// Receiver-side LWW (rustfs/backlog#1953): the multipart replication
|
||||
// transport carries the category values at CreateMultipartUpload (in
|
||||
// the staged upload metadata) and the source category timestamps on
|
||||
// the complete request. Read the destination version under the held
|
||||
// object write lock and keep any category this site modified more
|
||||
// recently. Read failures (version absent on first replication, quorum
|
||||
// errors) keep today's overwrite semantics: failing the complete would
|
||||
// loop through MRF, re-delivering the stale value forever.
|
||||
if crate::set_disk::ops::object::replication_lww_applicable(opts)
|
||||
&& let Some(version_id) = fi.version_id
|
||||
{
|
||||
match self
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
version_id: Some(version_id.to_string()),
|
||||
no_lock: true,
|
||||
metadata_cache_safe: false,
|
||||
versioned: opts.versioned,
|
||||
version_suspended: opts.version_suspended,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(existing) => {
|
||||
let stored = crate::set_disk::ops::object::stored_replication_category_metadata(&existing);
|
||||
crate::set_disk::ops::object::merge_replication_metadata_lww(&mut fi.metadata, &stored, opts);
|
||||
}
|
||||
// Version absent: first replication of this version, nothing
|
||||
// local to compare — the normal path, not a degraded one.
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
|
||||
Err(err) => {
|
||||
// Degraded path: without the stored state the inbound
|
||||
// metadata is applied unchanged — exactly the overwrite
|
||||
// LWW exists to prevent — so this must be operator-visible.
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
bucket,
|
||||
object,
|
||||
version_id = %version_id,
|
||||
error = %err,
|
||||
state = "replication_lww_read_unavailable",
|
||||
"SetDisk multipart replication LWW read skipped; inbound metadata applied without comparison"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for meta in parts_metadatas.iter_mut() {
|
||||
if meta.has_valid_erasure_geometry() {
|
||||
meta.size = fi.size;
|
||||
@@ -7055,6 +7106,97 @@ mod tests {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Receiver-side LWW on the multipart replication transport
|
||||
/// (rustfs/backlog#1953): a metadata-only replication of a multipart
|
||||
/// source object rides CreateMultipartUpload (category values in the
|
||||
/// upload metadata) + CompleteMultipartUpload (category timestamps in
|
||||
/// the complete options). A stale inbound tagging timestamp must not
|
||||
/// overwrite a newer locally-tagged destination version.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn complete_multipart_upload_stale_replication_tags_keep_local() {
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||
use rustfs_utils::http::{SUFFIX_TAGGING_TIMESTAMP, get_str};
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
const T_OLD: &str = "2026-01-01T00:00:00Z";
|
||||
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
|
||||
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-replication-lww-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
|
||||
// Local destination version with newer tags.
|
||||
let version_id = Uuid::new_v4();
|
||||
let mut local_metadata = HashMap::new();
|
||||
local_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
|
||||
rustfs_utils::http::insert_str(&mut local_metadata, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
|
||||
let mut local_reader = PutObjReader::from_vec(b"local body".to_vec());
|
||||
set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut local_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version_id.to_string()),
|
||||
user_defined: local_metadata,
|
||||
// Explicit-version PUTs require the bucket Object Lock snapshot.
|
||||
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("local versioned put should commit");
|
||||
|
||||
// Inbound replication upload carrying older tags for the same version.
|
||||
let mut inbound_metadata = HashMap::new();
|
||||
inbound_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
|
||||
rustfs_utils::http::insert_str(&mut inbound_metadata, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
|
||||
let create_opts = ObjectOptions {
|
||||
versioned: true,
|
||||
user_defined: inbound_metadata,
|
||||
..Default::default()
|
||||
};
|
||||
let (upload_id, parts) =
|
||||
stage_upload_with_create_opts(&set_disks, bucket, object, &payload(0x5a), &create_opts).await;
|
||||
rewrite_staged_upload_version_id(&set_disks, bucket, object, &upload_id, Some(version_id)).await;
|
||||
|
||||
let complete_opts = ObjectOptions {
|
||||
versioned: true,
|
||||
replication_request: true,
|
||||
replication_tagging_timestamp: Some(OffsetDateTime::parse(T_OLD, &Rfc3339).expect("test timestamp should parse")),
|
||||
..Default::default()
|
||||
};
|
||||
set_disks
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts)
|
||||
.await
|
||||
.expect("replication multipart completion should succeed even when a category keeps local values");
|
||||
|
||||
let info = set_disks
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version_id.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("completed version should be readable");
|
||||
assert_eq!(
|
||||
info.user_tags.as_str(),
|
||||
"site=local",
|
||||
"older inbound multipart tags must not overwrite newer local tags"
|
||||
);
|
||||
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn complete_multipart_upload_assigns_completion_version_id() {
|
||||
|
||||
@@ -1881,6 +1881,110 @@ fn delete_file_info_with_replication_transport_metadata(fi: &FileInfo) -> FileIn
|
||||
transported
|
||||
}
|
||||
|
||||
/// True when an authorized replication write carries at least one per-category
|
||||
/// source timestamp, i.e. receiver-side LWW has something to judge.
|
||||
pub(in crate::set_disk) fn replication_lww_applicable(opts: &ObjectOptions) -> bool {
|
||||
opts.replication_request
|
||||
&& (opts.replication_tagging_timestamp.is_some()
|
||||
|| opts.replication_retention_timestamp.is_some()
|
||||
|| opts.replication_legalhold_timestamp.is_some())
|
||||
}
|
||||
|
||||
/// The stored per-category state of a destination version, as compared by
|
||||
/// [`merge_replication_metadata_lww`]. `ObjectInfo::from_file_info`
|
||||
/// externalizes tags into `user_tags` (stripping the metadata key), so the
|
||||
/// tag value is folded back into map form here.
|
||||
pub(in crate::set_disk) fn stored_replication_category_metadata(existing: &ObjectInfo) -> HashMap<String, String> {
|
||||
let mut stored = (*existing.user_defined).clone();
|
||||
if !existing.user_tags.is_empty() {
|
||||
stored.insert(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(), (*existing.user_tags).clone());
|
||||
}
|
||||
stored
|
||||
}
|
||||
|
||||
/// Receiver-side last-writer-wins for authorized replication writes
|
||||
/// (rustfs/backlog#1953, audit A4/P1-6). Metadata-only replication reuses the
|
||||
/// whole-object transports, so in active-active topologies an inbound write
|
||||
/// carries the source's tags / retention / legal hold verbatim and would
|
||||
/// otherwise overwrite a category the destination modified more recently —
|
||||
/// both sites end up permanently diverged while reporting COMPLETED.
|
||||
///
|
||||
/// Judged per category, only when the inbound request carries that category's
|
||||
/// source timestamp (`ObjectOptions::replication_*_timestamp`):
|
||||
/// - stored timestamp newer than inbound: the local category values and
|
||||
/// timestamp are kept; the rest of the write proceeds per the inbound
|
||||
/// metadata and the object-level result stays successful (failing the write
|
||||
/// instead would loop through MRF, re-delivering the stale value forever);
|
||||
/// - otherwise the inbound category wins and its internal timestamp key is
|
||||
/// pinned to the source-authored time — the PUT path re-stamps the
|
||||
/// object-lock timestamps with the receiver's clock
|
||||
/// (`parse_object_lock_retention` / `parse_object_lock_legal_hold` insert
|
||||
/// `now()` via `eval_metadata`), which would make the replica's clock the
|
||||
/// LWW authority and wedge later convergence;
|
||||
/// - no stored timestamp (pre-P1-6 data) or no inbound timestamp: the current
|
||||
/// overwrite behavior is preserved.
|
||||
///
|
||||
/// Returns whether `inbound` was modified. Callers must hold the object write
|
||||
/// lock so the stored values compared here are the ones being replaced.
|
||||
pub(in crate::set_disk) fn merge_replication_metadata_lww(
|
||||
inbound: &mut HashMap<String, String>,
|
||||
existing: &HashMap<String, String>,
|
||||
opts: &ObjectOptions,
|
||||
) -> bool {
|
||||
use rustfs_utils::http::headers::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
|
||||
};
|
||||
use rustfs_utils::http::metadata_compat::{
|
||||
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
|
||||
remove_str,
|
||||
};
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
let categories: [(Option<OffsetDateTime>, &str, &[&str]); 3] = [
|
||||
(opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP, &[AMZ_OBJECT_TAGGING]),
|
||||
(
|
||||
opts.replication_retention_timestamp,
|
||||
SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
||||
&[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
|
||||
),
|
||||
(
|
||||
opts.replication_legalhold_timestamp,
|
||||
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
|
||||
&[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER],
|
||||
),
|
||||
];
|
||||
|
||||
let mut changed = false;
|
||||
for (inbound_timestamp, timestamp_suffix, value_keys) in categories {
|
||||
let Some(inbound_timestamp) = inbound_timestamp else { continue };
|
||||
let is_category_value_key = |key: &str| value_keys.iter().any(|value_key| key.eq_ignore_ascii_case(value_key));
|
||||
let stored_timestamp = get_str(existing, timestamp_suffix).and_then(|value| OffsetDateTime::parse(&value, &Rfc3339).ok());
|
||||
if stored_timestamp.is_some_and(|stored| stored > inbound_timestamp) {
|
||||
inbound.retain(|key, _| !is_category_value_key(key));
|
||||
remove_str(inbound, timestamp_suffix);
|
||||
for (key, value) in existing {
|
||||
if is_category_value_key(key) {
|
||||
inbound.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
// Restore the winning timestamp via insert_str, not a verbatim key
|
||||
// copy: a MinIO-written version may carry only the
|
||||
// x-minio-internal- key, and the dual-key invariant requires every
|
||||
// write to produce both keys.
|
||||
if let Some(stored_value) = get_str(existing, timestamp_suffix) {
|
||||
rustfs_utils::http::insert_str(inbound, timestamp_suffix, stored_value);
|
||||
}
|
||||
changed = true;
|
||||
} else if let Ok(source_authored) = inbound_timestamp.format(&Rfc3339)
|
||||
&& get_str(inbound, timestamp_suffix).as_deref() != Some(source_authored.as_str())
|
||||
{
|
||||
rustfs_utils::http::insert_str(inbound, timestamp_suffix, source_authored);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
changed
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(in crate::set_disk) async fn persist_old_data_cleanup_receipts(
|
||||
&self,
|
||||
@@ -2562,6 +2666,22 @@ impl SetDisks {
|
||||
if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() {
|
||||
return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
|
||||
}
|
||||
// Receiver-side LWW (rustfs/backlog#1953): reuse this
|
||||
// commit-lock read of the destination version so a
|
||||
// category (tags / retention / legal hold) modified
|
||||
// more recently on this site is kept instead of being
|
||||
// overwritten by the inbound replication metadata.
|
||||
if replication_lww_applicable(opts) {
|
||||
let stored = stored_replication_category_metadata(&existing);
|
||||
let mut merged = parts_metadatas[response_metadata_slot].metadata.clone();
|
||||
if merge_replication_metadata_lww(&mut merged, &stored, opts) {
|
||||
for (pfi, disk) in parts_metadatas.iter_mut().zip(shuffle_disks.iter()) {
|
||||
if disk.is_some() {
|
||||
pfi.metadata = merged.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
|
||||
Err(err) => return Err(err),
|
||||
@@ -8066,6 +8186,357 @@ mod replication_quota_safety_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod replication_lww_tests {
|
||||
//! Receiver-side LWW for authorized replication writes (rustfs/backlog#1953,
|
||||
//! audit A4/P1-6): an inbound replication PUT whose per-category timestamp
|
||||
//! (tags / retention / legal hold) is older than the destination version's
|
||||
//! stored timestamp must keep the local category values instead of
|
||||
//! overwriting them; categories are judged independently and the write
|
||||
//! itself still succeeds.
|
||||
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
||||
use super::*;
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use rustfs_utils::http::headers::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
|
||||
insert_str,
|
||||
};
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
const T_OLD: &str = "2026-01-01T00:00:00Z";
|
||||
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
|
||||
const T_NEW: &str = "2026-03-01T00:00:00Z";
|
||||
|
||||
fn parse_ts(value: &str) -> OffsetDateTime {
|
||||
OffsetDateTime::parse(value, &Rfc3339).expect("test timestamp should parse")
|
||||
}
|
||||
|
||||
async fn make_bucket(disks: &[DiskStore], bucket: &str) {
|
||||
for disk in disks {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
}
|
||||
|
||||
async fn put_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str, opts: &ObjectOptions) {
|
||||
let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut reader, opts)
|
||||
.await
|
||||
.expect("versioned put should commit");
|
||||
assert_eq!(opts.version_id.as_deref(), Some(version_id));
|
||||
}
|
||||
|
||||
fn versioned_opts(version_id: &str, user_defined: HashMap<String, String>) -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version_id.to_string()),
|
||||
user_defined,
|
||||
// Explicit-version PUTs require the bucket Object Lock snapshot.
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Local state: version `version_id` with tags "site=local" stamped `T_LOCAL`.
|
||||
async fn seed_local_tagged_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) {
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
|
||||
insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
|
||||
put_version(set_disks, bucket, object, version_id, &versioned_opts(version_id, user_defined)).await;
|
||||
}
|
||||
|
||||
fn inbound_tagging_opts(version_id: &str, tags: &str, timestamp: &str) -> ObjectOptions {
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), tags.to_string());
|
||||
insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, timestamp.to_string());
|
||||
ObjectOptions {
|
||||
replication_request: true,
|
||||
replication_tagging_timestamp: Some(parse_ts(timestamp)),
|
||||
..versioned_opts(version_id, user_defined)
|
||||
}
|
||||
}
|
||||
|
||||
async fn version_info(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) -> ObjectInfo {
|
||||
set_disks
|
||||
.get_object_info(bucket, object, &versioned_opts(version_id, HashMap::new()))
|
||||
.await
|
||||
.expect("version should be readable")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inbound_stale_tagging_keeps_newer_local_tags() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "lww-tagging-stale";
|
||||
let object = "object";
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
|
||||
|
||||
put_version(
|
||||
&set_disks,
|
||||
bucket,
|
||||
object,
|
||||
&version_id,
|
||||
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
|
||||
)
|
||||
.await;
|
||||
|
||||
let info = version_info(&set_disks, bucket, object, &version_id).await;
|
||||
assert_eq!(
|
||||
info.user_tags.as_str(),
|
||||
"site=local",
|
||||
"older inbound tags must not overwrite newer local tags"
|
||||
);
|
||||
assert_eq!(
|
||||
get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(),
|
||||
Some(T_LOCAL),
|
||||
"the winning local tagging timestamp must be preserved"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inbound_newer_tagging_overwrites_local_tags() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "lww-tagging-newer";
|
||||
let object = "object";
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
|
||||
|
||||
put_version(
|
||||
&set_disks,
|
||||
bucket,
|
||||
object,
|
||||
&version_id,
|
||||
&inbound_tagging_opts(&version_id, "site=remote", T_NEW),
|
||||
)
|
||||
.await;
|
||||
|
||||
let info = version_info(&set_disks, bucket, object, &version_id).await;
|
||||
assert_eq!(
|
||||
info.user_tags.as_str(),
|
||||
"site=remote",
|
||||
"newer inbound tags must overwrite older local tags"
|
||||
);
|
||||
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_NEW));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inbound_wins_when_local_has_no_tagging_timestamp() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "lww-tagging-no-local-ts";
|
||||
let object = "object";
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
// Pre-P1-6 data: local tags without a stored tagging timestamp.
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
|
||||
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, user_defined)).await;
|
||||
|
||||
put_version(
|
||||
&set_disks,
|
||||
bucket,
|
||||
object,
|
||||
&version_id,
|
||||
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
|
||||
)
|
||||
.await;
|
||||
|
||||
let info = version_info(&set_disks, bucket, object, &version_id).await;
|
||||
assert_eq!(
|
||||
info.user_tags.as_str(),
|
||||
"site=remote",
|
||||
"without a local timestamp the inbound category must win (pre-LWW data compatibility)"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn categories_are_judged_independently() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "lww-category-independent";
|
||||
let object = "object";
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
|
||||
// Local: newer tags (T_LOCAL), older *cleared* retention (T_OLD) —
|
||||
// timestamp key only, the shape a replicated retention clear stores.
|
||||
// (An active local retention would already block the overwrite at the
|
||||
// WORM gate; the LWW-reachable retention states are cleared/expired.)
|
||||
let mut local = HashMap::new();
|
||||
local.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
|
||||
insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
|
||||
insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
|
||||
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
|
||||
|
||||
// Inbound: older tags (T_OLD), newer retention (T_NEW).
|
||||
let mut inbound = HashMap::new();
|
||||
inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
|
||||
insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
|
||||
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
|
||||
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
|
||||
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
|
||||
let opts = ObjectOptions {
|
||||
replication_request: true,
|
||||
replication_tagging_timestamp: Some(parse_ts(T_OLD)),
|
||||
replication_retention_timestamp: Some(parse_ts(T_NEW)),
|
||||
..versioned_opts(&version_id, inbound)
|
||||
};
|
||||
put_version(&set_disks, bucket, object, &version_id, &opts).await;
|
||||
|
||||
let info = version_info(&set_disks, bucket, object, &version_id).await;
|
||||
assert_eq!(info.user_tags.as_str(), "site=local", "the stale tagging category must keep local values");
|
||||
assert_eq!(
|
||||
info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str),
|
||||
Some("COMPLIANCE"),
|
||||
"the newer retention category must be applied in the same write"
|
||||
);
|
||||
assert_eq!(get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(), Some(T_NEW));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inbound_stale_legal_hold_keeps_local_value() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "lww-legalhold-stale";
|
||||
let object = "object";
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
|
||||
// Local: legal hold released (OFF) at T_LOCAL. (A local hold that is
|
||||
// still ON already blocks the overwrite at the WORM gate; the
|
||||
// LWW-reachable divergence is a stale inbound ON resurrecting a hold
|
||||
// that was released more recently on this site.)
|
||||
let mut local = HashMap::new();
|
||||
local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
|
||||
insert_str(&mut local, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_LOCAL.to_string());
|
||||
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
|
||||
|
||||
let mut inbound = HashMap::new();
|
||||
inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string());
|
||||
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string());
|
||||
let opts = ObjectOptions {
|
||||
replication_request: true,
|
||||
replication_legalhold_timestamp: Some(parse_ts(T_OLD)),
|
||||
..versioned_opts(&version_id, inbound)
|
||||
};
|
||||
put_version(&set_disks, bucket, object, &version_id, &opts).await;
|
||||
|
||||
let info = version_info(&set_disks, bucket, object, &version_id).await;
|
||||
assert_eq!(
|
||||
info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
|
||||
Some("OFF"),
|
||||
"a stale inbound legal hold must not resurrect a hold released more recently"
|
||||
);
|
||||
assert_eq!(
|
||||
get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(),
|
||||
Some(T_LOCAL)
|
||||
);
|
||||
}
|
||||
|
||||
/// Dual-key invariant under LWW: a MinIO-written destination version may
|
||||
/// carry only the x-minio-internal timestamp key; when the local category
|
||||
/// wins, the restored map must still hold BOTH compatibility keys.
|
||||
#[test]
|
||||
fn local_win_restores_both_internal_timestamp_keys_for_minio_only_metadata() {
|
||||
let mut inbound = HashMap::new();
|
||||
inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
|
||||
insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
|
||||
let existing = HashMap::from([
|
||||
(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string()),
|
||||
("X-Minio-Internal-Tagging-Timestamp".to_string(), T_LOCAL.to_string()),
|
||||
]);
|
||||
let opts = ObjectOptions {
|
||||
replication_request: true,
|
||||
replication_tagging_timestamp: Some(parse_ts(T_OLD)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(merge_replication_metadata_lww(&mut inbound, &existing, &opts));
|
||||
assert_eq!(inbound.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some("site=local"));
|
||||
assert_eq!(
|
||||
inbound.get("x-rustfs-internal-tagging-timestamp").map(String::as_str),
|
||||
Some(T_LOCAL),
|
||||
"the RustFS twin key must be materialized even when the source version only had the MinIO key"
|
||||
);
|
||||
assert_eq!(inbound.get("x-minio-internal-tagging-timestamp").map(String::as_str), Some(T_LOCAL));
|
||||
}
|
||||
|
||||
/// When the inbound category wins, the stored timestamp must be the
|
||||
/// source-authored one: the PUT path's eval_metadata stamps the
|
||||
/// object-lock timestamps with the receiver's clock
|
||||
/// (`parse_object_lock_retention`), which would otherwise make this
|
||||
/// replica's clock the LWW authority and wedge later convergence.
|
||||
#[tokio::test]
|
||||
async fn inbound_win_pins_stored_timestamp_to_source_authored_value() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "lww-retention-ts-pinned";
|
||||
let object = "object";
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
|
||||
// Local cleared retention at T_OLD.
|
||||
let mut local = HashMap::new();
|
||||
insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
|
||||
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
|
||||
|
||||
// Inbound newer retention: the source authored T_LOCAL, but the PUT
|
||||
// path's eval_metadata stomped the metadata key with receiver-now
|
||||
// (simulated by T_NEW here).
|
||||
let mut inbound = HashMap::new();
|
||||
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string());
|
||||
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
|
||||
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
|
||||
let opts = ObjectOptions {
|
||||
replication_request: true,
|
||||
replication_retention_timestamp: Some(parse_ts(T_LOCAL)),
|
||||
..versioned_opts(&version_id, inbound)
|
||||
};
|
||||
put_version(&set_disks, bucket, object, &version_id, &opts).await;
|
||||
|
||||
let info = version_info(&set_disks, bucket, object, &version_id).await;
|
||||
assert_eq!(
|
||||
get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(),
|
||||
Some(T_LOCAL),
|
||||
"the stored category timestamp must be the source-authored time, not the receiver's clock"
|
||||
);
|
||||
assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn newer_local_tag_deletion_survives_stale_inbound_tags() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "lww-tagging-deleted";
|
||||
let object = "object";
|
||||
let version_id = Uuid::new_v4().to_string();
|
||||
make_bucket(&disk_stores, bucket).await;
|
||||
// Local DeleteObjectTagging state: no tags, but a newer tagging timestamp.
|
||||
let mut local = HashMap::new();
|
||||
insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
|
||||
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
|
||||
|
||||
put_version(
|
||||
&set_disks,
|
||||
bucket,
|
||||
object,
|
||||
&version_id,
|
||||
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
|
||||
)
|
||||
.await;
|
||||
|
||||
let info = version_info(&set_disks, bucket, object, &version_id).await;
|
||||
assert!(
|
||||
info.user_tags.is_empty(),
|
||||
"a newer local tag deletion must not be resurrected by older inbound tags"
|
||||
);
|
||||
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod inline_put_commit_path_tests {
|
||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
||||
|
||||
@@ -16,14 +16,14 @@
|
||||
//!
|
||||
//! `scripts/test/vault_ha_kms_live.sh` owns the official Vault containers and
|
||||
//! kills the active node while this test continuously decrypts through a
|
||||
//! surviving standby. KV2 and Transit must recover after the bounded circuit
|
||||
//! interval, use a bounded number of attempts, and leave the circuit and
|
||||
//! in-flight gauges at zero after a new leader is elected.
|
||||
//! surviving standby. KV2 and Transit requests must remain successful, use a
|
||||
//! bounded number of attempts, and leave the circuit and in-flight gauges at
|
||||
//! zero after a new leader is elected.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use metrics_util::MetricKind;
|
||||
@@ -43,11 +43,6 @@ const OPERATION_ATTEMPTS: &str = "rustfs_kms_backend_operation_attempts";
|
||||
const IN_FLIGHT: &str = "rustfs_kms_backend_in_flight";
|
||||
const CIRCUIT_OPEN: &str = "rustfs_kms_backend_circuit_open";
|
||||
const MAX_ATTEMPTS: u32 = 10;
|
||||
const ATTEMPT_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
const HEALTHY_PROGRESS_TIMEOUT: Duration = Duration::from_secs(20);
|
||||
// The circuit remains open for 30s after five failed attempts.
|
||||
const POST_FAILOVER_PROGRESS_TIMEOUT: Duration = Duration::from_secs(35);
|
||||
const FAILOVER_ERROR_POLL_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
type MetricEntry = (
|
||||
metrics_util::CompositeKey,
|
||||
@@ -69,7 +64,7 @@ fn config(backend: KmsBackend, backend_config: BackendConfig) -> KmsConfig {
|
||||
backend,
|
||||
backend_config,
|
||||
allow_insecure_dev_defaults: true,
|
||||
timeout: ATTEMPT_TIMEOUT,
|
||||
timeout: Duration::from_secs(2),
|
||||
retry_attempts: MAX_ATTEMPTS,
|
||||
enable_cache: false,
|
||||
..KmsConfig::default()
|
||||
@@ -169,31 +164,14 @@ fn retryable_failures(snapshot: &[MetricEntry], operation: &str) -> u64 {
|
||||
.sum()
|
||||
}
|
||||
|
||||
async fn wait_for_count(
|
||||
counter: &AtomicU64,
|
||||
failure: &Mutex<Option<String>>,
|
||||
minimum: u64,
|
||||
description: &str,
|
||||
timeout: Duration,
|
||||
) {
|
||||
tokio::time::timeout(timeout, async {
|
||||
async fn wait_for_count(counter: &AtomicU64, minimum: u64, description: &str) {
|
||||
tokio::time::timeout(Duration::from_secs(20), async {
|
||||
while counter.load(Ordering::SeqCst) < minimum {
|
||||
if let Some(error) = failure.lock().expect("decrypt failure lock poisoned").as_ref() {
|
||||
panic!(
|
||||
"{description} worker failed after {} successful decrypts: {error}",
|
||||
counter.load(Ordering::SeqCst)
|
||||
);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
panic!(
|
||||
"timed out after {timeout:?} waiting for {description}: completed {}, expected {minimum}",
|
||||
counter.load(Ordering::SeqCst)
|
||||
)
|
||||
});
|
||||
.unwrap_or_else(|_| panic!("timed out waiting for {description}"));
|
||||
}
|
||||
|
||||
async fn wait_for_file(path: &Path, description: &str) {
|
||||
@@ -211,8 +189,7 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
|
||||
request: DecryptRequest,
|
||||
expected: Vec<u8>,
|
||||
completed: Arc<AtomicU64>,
|
||||
allow_failover_errors: Arc<AtomicBool>,
|
||||
failure: Arc<Mutex<Option<String>>>,
|
||||
failed: Arc<AtomicBool>,
|
||||
stop: CancellationToken,
|
||||
) {
|
||||
while !stop.is_cancelled() {
|
||||
@@ -220,18 +197,8 @@ async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
|
||||
Ok(response) if response.plaintext == expected => {
|
||||
completed.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
Ok(_) => {
|
||||
*failure.lock().expect("decrypt failure lock poisoned") =
|
||||
Some("decrypt returned unexpected plaintext".to_string());
|
||||
return;
|
||||
}
|
||||
Err(rustfs_kms::KmsError::BackendError { .. } | rustfs_kms::KmsError::OperationTimedOut { .. })
|
||||
if allow_failover_errors.load(Ordering::SeqCst) =>
|
||||
{
|
||||
tokio::time::sleep(FAILOVER_ERROR_POLL_INTERVAL).await;
|
||||
}
|
||||
Err(error) => {
|
||||
*failure.lock().expect("decrypt failure lock poisoned") = Some(error.to_string());
|
||||
Ok(_) | Err(_) => {
|
||||
failed.store(true, Ordering::SeqCst);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -329,9 +296,7 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
|
||||
);
|
||||
|
||||
let stop = CancellationToken::new();
|
||||
let allow_failover_errors = Arc::new(AtomicBool::new(false));
|
||||
let kv2_failure = Arc::new(Mutex::new(None));
|
||||
let transit_failure = Arc::new(Mutex::new(None));
|
||||
let failed = Arc::new(AtomicBool::new(false));
|
||||
let kv2_completed = Arc::new(AtomicU64::new(0));
|
||||
let transit_completed = Arc::new(AtomicU64::new(0));
|
||||
let kv2_worker = tokio::spawn(decrypt_loop(
|
||||
@@ -339,8 +304,7 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
|
||||
kv2_request,
|
||||
kv2_data_key.plaintext_key,
|
||||
Arc::clone(&kv2_completed),
|
||||
Arc::clone(&allow_failover_errors),
|
||||
Arc::clone(&kv2_failure),
|
||||
Arc::clone(&failed),
|
||||
stop.clone(),
|
||||
));
|
||||
let transit_worker = tokio::spawn(decrypt_loop(
|
||||
@@ -348,21 +312,12 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
|
||||
transit_request,
|
||||
transit_data_key.plaintext_key,
|
||||
Arc::clone(&transit_completed),
|
||||
Arc::clone(&allow_failover_errors),
|
||||
Arc::clone(&transit_failure),
|
||||
Arc::clone(&failed),
|
||||
stop.clone(),
|
||||
));
|
||||
|
||||
wait_for_count(&kv2_completed, &kv2_failure, 2, "two healthy KV2 decrypts", HEALTHY_PROGRESS_TIMEOUT).await;
|
||||
wait_for_count(
|
||||
&transit_completed,
|
||||
&transit_failure,
|
||||
2,
|
||||
"two healthy Transit decrypts",
|
||||
HEALTHY_PROGRESS_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
allow_failover_errors.store(true, Ordering::SeqCst);
|
||||
wait_for_count(&kv2_completed, 2, "two healthy KV2 decrypts").await;
|
||||
wait_for_count(&transit_completed, 2, "two healthy Transit decrypts").await;
|
||||
std::fs::write(&marker, b"ready").expect("publish failover readiness marker");
|
||||
|
||||
wait_for_file(&elected, "the replacement Vault leader").await;
|
||||
@@ -371,39 +326,18 @@ async fn exercise_failover(snapshotter: &Snapshotter) {
|
||||
|
||||
let kv2_after_election = kv2_completed.load(Ordering::SeqCst) + 2;
|
||||
let transit_after_election = transit_completed.load(Ordering::SeqCst) + 2;
|
||||
wait_for_count(
|
||||
&kv2_completed,
|
||||
&kv2_failure,
|
||||
kv2_after_election,
|
||||
"post-failover KV2 decrypts",
|
||||
POST_FAILOVER_PROGRESS_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
wait_for_count(
|
||||
&transit_completed,
|
||||
&transit_failure,
|
||||
transit_after_election,
|
||||
"post-failover Transit decrypts",
|
||||
POST_FAILOVER_PROGRESS_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
wait_for_count(&kv2_completed, kv2_after_election, "post-failover KV2 decrypts").await;
|
||||
wait_for_count(&transit_completed, transit_after_election, "post-failover Transit decrypts").await;
|
||||
|
||||
stop.cancel();
|
||||
kv2_worker.await.expect("KV2 decrypt worker must join");
|
||||
transit_worker.await.expect("Transit decrypt worker must join");
|
||||
assert!(
|
||||
kv2_failure.lock().expect("KV2 failure lock poisoned").is_none(),
|
||||
"no KV2 decrypt may fail or return different plaintext"
|
||||
);
|
||||
assert!(
|
||||
transit_failure.lock().expect("Transit failure lock poisoned").is_none(),
|
||||
"no Transit decrypt may fail or return different plaintext"
|
||||
);
|
||||
assert!(!failed.load(Ordering::SeqCst), "no decrypt may fail or return different plaintext");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires a real three-node Vault Raft cluster; run scripts/test/vault_ha_kms_live.sh"]
|
||||
fn vault_raft_leader_failure_recovers_kv2_and_transit_decrypts() {
|
||||
fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
@@ -415,6 +349,11 @@ fn vault_raft_leader_failure_recovers_kv2_and_transit_decrypts() {
|
||||
});
|
||||
let snapshot = snapshotter.snapshot().into_vec();
|
||||
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "circuit_open")]),
|
||||
0,
|
||||
"a bounded leader election must not open the circuit"
|
||||
);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]),
|
||||
0,
|
||||
|
||||
@@ -54,7 +54,6 @@ const ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS: &str =
|
||||
"Days must be a positive integer and Date must not be specified inside Expiration with ExpiredObjectAllVersions";
|
||||
const ERR_LIFECYCLE_INVALID_DEL_MARKER_EXPIRATION_DAYS: &str = "Days must be a positive integer with DelMarkerExpiration";
|
||||
const ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG: &str = "Rule ID must be at most 255 characters";
|
||||
const ERR_LIFECYCLE_INVALID_RULE_ID_EMPTY: &str = "Rule ID must not be empty";
|
||||
const ERR_LIFECYCLE_INVALID_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled";
|
||||
const ERR_LIFECYCLE_DEL_MARKER_WITH_TAGS: &str = "Rule with DelMarkerExpiration cannot have tags based filtering";
|
||||
const ERR_LIFECYCLE_EXPIRED_OBJECT_DELETE_MARKER_WITH_TAGS: &str =
|
||||
@@ -403,13 +402,10 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
NoncurrentVersionTransitionOps::validate(transition)?;
|
||||
}
|
||||
}
|
||||
if let Some(id) = &r.id {
|
||||
if id.is_empty() {
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_EMPTY));
|
||||
}
|
||||
if id.len() > 255 {
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG));
|
||||
}
|
||||
if let Some(id) = &r.id
|
||||
&& id.len() > 255
|
||||
{
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG));
|
||||
}
|
||||
r.validate()?;
|
||||
if let Some(object_lock_enabled) = lr.object_lock_enabled.as_ref()
|
||||
@@ -3734,31 +3730,6 @@ mod tests {
|
||||
.expect("empty prefix with filter should be valid");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_rejects_empty_rule_id() {
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(30),
|
||||
..Default::default()
|
||||
}),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: Some(String::new()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
};
|
||||
|
||||
let error = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
|
||||
assert_eq!(error.to_string(), ERR_LIFECYCLE_INVALID_RULE_ID_EMPTY);
|
||||
}
|
||||
|
||||
// --- TASK-004 tests: ExpiredObjectAllVersions ---
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -29,8 +29,7 @@ use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS;
|
||||
pub use rustfs_data_usage::{
|
||||
AllTierStats, BucketTargetUsageInfo, BucketUsageInfo, DATA_USAGE_OBJECT_NAME, DATA_USAGE_OBSERVED_OBJECT_NAME,
|
||||
DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageInfo, LEGACY_DATA_USAGE_OBJECT_NAME, PrefixUsageEntry,
|
||||
PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeReconciliationEntry, SizeReconciliationScope, SizeSummary,
|
||||
TierStats, hash_path, prefix_usage_in_cache,
|
||||
PrefixUsageQuery, PrefixUsageSummary, ReplTargetSizeSummary, SizeSummary, TierStats, hash_path, prefix_usage_in_cache,
|
||||
};
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
@@ -193,10 +192,6 @@ const MAX_DATA_USAGE_CACHE_DEPTH: usize = 1024;
|
||||
pub trait ScannerSizeSummaryExt {
|
||||
/// Fold one object's contribution into the summary, including its tier.
|
||||
fn actions_accounting(&mut self, oi: &ObjectInfo, size: i64, actual_size: i64);
|
||||
/// Fold counters and physical tier usage for an object whose metadata is
|
||||
/// valid but whose logical size is currently unavailable. Logical totals
|
||||
/// stay unchanged.
|
||||
fn actions_accounting_unknown(&mut self, oi: &ObjectInfo);
|
||||
}
|
||||
|
||||
impl ScannerSizeSummaryExt for SizeSummary {
|
||||
@@ -230,34 +225,6 @@ impl ScannerSizeSummaryExt for SizeSummary {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn actions_accounting_unknown(&mut self, oi: &ObjectInfo) {
|
||||
if oi.delete_marker {
|
||||
self.delete_markers = self.delete_markers.saturating_add(1);
|
||||
return;
|
||||
}
|
||||
|
||||
if oi.version_id.is_some_and(|v| !v.is_nil()) {
|
||||
self.versions = self.versions.saturating_add(1);
|
||||
}
|
||||
|
||||
if oi.transitioned_object.free_version {
|
||||
return;
|
||||
}
|
||||
|
||||
let tier = if oi.transitioned_object.status == TRANSITION_COMPLETE {
|
||||
oi.transitioned_object.tier.clone()
|
||||
} else {
|
||||
oi.storage_class.clone().unwrap_or_else(|| storageclass::STANDARD.to_string())
|
||||
};
|
||||
if let Some(tier_stats) = self.tier_stats.get_mut(&tier) {
|
||||
*tier_stats = tier_stats.add(&TierStats {
|
||||
total_size: u64::try_from(oi.size).unwrap_or(0),
|
||||
num_versions: 1,
|
||||
num_objects: u64::from(oi.is_latest),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== Cache-related data structures =====
|
||||
@@ -377,10 +344,6 @@ pub struct DataUsageCacheInfo {
|
||||
pub scan_plan_digest: Option<DataUsageScanPlanDigest>,
|
||||
#[serde(default)]
|
||||
pub cache_key_format: u16,
|
||||
/// Bounded durable debts for versions whose logical size was not trusted.
|
||||
/// The map key is an identity key, never a user-controlled metric label.
|
||||
#[serde(default)]
|
||||
pub size_reconciliation: HashMap<String, SizeReconciliationEntry>,
|
||||
}
|
||||
|
||||
impl Serialize for DataUsageCacheInfo {
|
||||
@@ -390,8 +353,7 @@ impl Serialize for DataUsageCacheInfo {
|
||||
{
|
||||
// Keep this metadata map-encoded so older readers can ignore fields
|
||||
// appended by newer scanner versions during rolling upgrades.
|
||||
let field_count = 16 + usize::from(!self.size_reconciliation.is_empty());
|
||||
let mut state = serializer.serialize_map(Some(field_count))?;
|
||||
let mut state = serializer.serialize_map(Some(16))?;
|
||||
state.serialize_entry("name", &self.name)?;
|
||||
state.serialize_entry("next_cycle", &self.next_cycle)?;
|
||||
state.serialize_entry("leader_epoch", &self.leader_epoch)?;
|
||||
@@ -408,9 +370,6 @@ impl Serialize for DataUsageCacheInfo {
|
||||
state.serialize_entry("snapshot_complete", &self.snapshot_complete)?;
|
||||
state.serialize_entry("scan_plan_digest", &self.scan_plan_digest)?;
|
||||
state.serialize_entry("cache_key_format", &self.cache_key_format)?;
|
||||
if !self.size_reconciliation.is_empty() {
|
||||
state.serialize_entry("size_reconciliation", &self.size_reconciliation)?;
|
||||
}
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
@@ -469,18 +428,14 @@ impl DataUsageCache {
|
||||
self.checked_flatten(name).is_some()
|
||||
});
|
||||
if !reusable {
|
||||
let (pending_heals, size_reconciliation) = if self.info.name == name {
|
||||
(
|
||||
std::mem::take(&mut self.info.pending_heals),
|
||||
std::mem::take(&mut self.info.size_reconciliation),
|
||||
)
|
||||
let pending_heals = if self.info.name == name {
|
||||
std::mem::take(&mut self.info.pending_heals)
|
||||
} else {
|
||||
(Vec::new(), HashMap::new())
|
||||
Vec::new()
|
||||
};
|
||||
*self = Self::default();
|
||||
self.info.name = name.to_string();
|
||||
self.info.pending_heals = pending_heals;
|
||||
self.info.size_reconciliation = size_reconciliation;
|
||||
}
|
||||
|
||||
self.info.next_cycle = next_cycle;
|
||||
|
||||
@@ -673,34 +673,6 @@ fn size_summary_actions_accounting_accumulates_tier_stats() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_summary_unknown_accounting_keeps_physical_tier_and_version_only() {
|
||||
let mut summary = SizeSummary::default();
|
||||
summary
|
||||
.tier_stats
|
||||
.insert(storageclass::STANDARD.to_string(), TierStats::default());
|
||||
let object = ObjectInfo {
|
||||
size: 12,
|
||||
storage_class: Some(storageclass::STANDARD.to_string()),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
is_latest: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
summary.actions_accounting_unknown(&object);
|
||||
|
||||
assert_eq!(summary.total_size, 0, "unknown logical size must not become zero or physical bytes");
|
||||
assert_eq!(summary.versions, 1);
|
||||
assert_eq!(
|
||||
summary.tier_stats.get(storageclass::STANDARD),
|
||||
Some(&TierStats {
|
||||
total_size: 12,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_usage_entry_merge_sums_failed_objects() {
|
||||
let mut left = DataUsageEntry {
|
||||
@@ -1107,16 +1079,6 @@ fn data_usage_cache_prepare_for_scan_preserves_pending_heal_only_progress() {
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
pending_heals: vec![pending_heal.clone()],
|
||||
size_reconciliation: HashMap::from([(
|
||||
"size-key".to_string(),
|
||||
SizeReconciliationEntry {
|
||||
key: "size-key".to_string(),
|
||||
bucket: "bucket".to_string(),
|
||||
object: "prefix/object".to_string(),
|
||||
reason: "invalid_declared_size".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
@@ -1126,7 +1088,6 @@ fn data_usage_cache_prepare_for_scan_preserves_pending_heal_only_progress() {
|
||||
|
||||
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reused);
|
||||
assert_eq!(cache.info.pending_heals, vec![pending_heal]);
|
||||
assert!(cache.info.size_reconciliation.contains_key("size-key"));
|
||||
assert!(cache.cache.is_empty());
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
}
|
||||
|
||||
@@ -20,9 +20,8 @@ use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use crate::ReplTargetSizeSummary;
|
||||
use crate::data_usage_define::{
|
||||
DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageHash, DataUsageHashMap,
|
||||
DataUsageScanCheckpoint, DataUsageScanCheckpointReason, PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt,
|
||||
SizeReconciliationEntry, SizeSummary, hash_path,
|
||||
DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageEntry, DataUsageHash, DataUsageHashMap, DataUsageScanCheckpoint,
|
||||
DataUsageScanCheckpointReason, PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeSummary, hash_path,
|
||||
};
|
||||
use crate::error::ScannerError;
|
||||
use crate::runtime_config::{
|
||||
@@ -98,9 +97,6 @@ const METRIC_SCANNER_EXCESS_FOLDERS_TOTAL: &str = "rustfs_scanner_excess_folders
|
||||
const METRIC_SCANNER_PENDING_HEAL_PRUNE_TOTAL: &str = "rustfs_scanner_pending_heal_prune_total";
|
||||
const METRIC_SCANNER_PENDING_HEAL_MALFORMED_TOTAL: &str = "rustfs_scanner_pending_heal_malformed_total";
|
||||
const MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET: usize = 128;
|
||||
const MAX_SIZE_RECONCILIATION_ENTRIES_PER_BUCKET: usize = 10_000;
|
||||
const MAX_SIZE_RECONCILIATION_BYTES_PER_BUCKET: usize = 8 * 1024 * 1024;
|
||||
const MAX_SIZE_RECONCILIATION_AGE_SECS: u64 = 7 * 24 * 60 * 60;
|
||||
|
||||
// --- scanner excess alerts as S3 notification events (rustfs/backlog#1868) --
|
||||
//
|
||||
@@ -368,7 +364,7 @@ impl PendingScannerAccounting<'_> {
|
||||
fn apply(self, size_summary: &mut SizeSummary, cumulative_size: &mut i64, queued: bool) {
|
||||
let size = if queued { self.expired_size } else { self.retained_size };
|
||||
size_summary.actions_accounting(self.object, size, self.retained_size);
|
||||
*cumulative_size = cumulative_size.saturating_add(size);
|
||||
*cumulative_size += size;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -675,65 +671,10 @@ pub struct FolderScanner {
|
||||
skip_heal: Arc<std::sync::atomic::AtomicBool>,
|
||||
local_disk: Arc<Disk>,
|
||||
pending_heals_changed: bool,
|
||||
pending_size_reconciliation_keys: HashSet<String>,
|
||||
pending_size_reconciliation_scopes: HashSet<String>,
|
||||
pending_size_reconciliation_truncated: bool,
|
||||
#[cfg(test)]
|
||||
list_path_raw_options_observer: Option<mpsc::UnboundedSender<ListPathRawTimeoutSnapshot>>,
|
||||
}
|
||||
|
||||
fn size_reconciliation_entry_bytes(entry: &SizeReconciliationEntry) -> usize {
|
||||
entry.key.len()
|
||||
+ entry.bucket.len()
|
||||
+ entry.object.len()
|
||||
+ entry.version_id.as_deref().map_or(0, str::len)
|
||||
+ entry.generation.as_deref().map_or(0, str::len)
|
||||
+ entry.reason.len()
|
||||
+ std::mem::size_of::<u64>()
|
||||
+ std::mem::size_of::<u32>()
|
||||
}
|
||||
|
||||
fn size_reconciliation_scope_key(bucket: &str, object: &str) -> String {
|
||||
format!("{}:{}|{}:{}", bucket.len(), bucket, object.len(), object)
|
||||
}
|
||||
|
||||
fn prune_size_reconciliation(info: &mut DataUsageCacheInfo, now: u64) {
|
||||
info.size_reconciliation.retain(|key, entry| {
|
||||
if entry.first_seen == 0 || entry.first_seen > now {
|
||||
entry.first_seen = now;
|
||||
}
|
||||
key == &entry.key
|
||||
&& entry.key.len() <= 4096
|
||||
&& entry.bucket.len() <= 512
|
||||
&& entry.object.len() <= 512
|
||||
&& entry.version_id.as_deref().is_none_or(|value| value.len() <= 64)
|
||||
&& entry.generation.as_deref().is_none_or(|value| value.len() <= 64)
|
||||
&& entry.reason.len() <= 64
|
||||
&& now.saturating_sub(entry.first_seen) <= MAX_SIZE_RECONCILIATION_AGE_SECS
|
||||
});
|
||||
|
||||
while info.size_reconciliation.len() > MAX_SIZE_RECONCILIATION_ENTRIES_PER_BUCKET
|
||||
|| info
|
||||
.size_reconciliation
|
||||
.values()
|
||||
.map(size_reconciliation_entry_bytes)
|
||||
.sum::<usize>()
|
||||
> MAX_SIZE_RECONCILIATION_BYTES_PER_BUCKET
|
||||
{
|
||||
let oldest = info
|
||||
.size_reconciliation
|
||||
.iter()
|
||||
.min_by(|(left_key, left), (right_key, right)| {
|
||||
left.first_seen.cmp(&right.first_seen).then_with(|| left_key.cmp(right_key))
|
||||
})
|
||||
.map(|(key, _)| key.clone());
|
||||
let Some(oldest) = oldest else {
|
||||
break;
|
||||
};
|
||||
info.size_reconciliation.remove(&oldest);
|
||||
}
|
||||
}
|
||||
|
||||
impl FolderScanner {
|
||||
fn now_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
@@ -807,60 +748,6 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply the per-object size-resolution ledger updates in one place. The
|
||||
/// scanner cache is the durable boundary; both working copies are updated
|
||||
/// so an incremental publication cannot lose a debt or its resolution.
|
||||
fn apply_size_reconciliation(&mut self, summary: &SizeSummary) {
|
||||
let now = Self::now_secs();
|
||||
self.pending_size_reconciliation_keys
|
||||
.extend(summary.size_reconciliation.iter().map(|entry| entry.key.clone()));
|
||||
self.pending_size_reconciliation_scopes.extend(
|
||||
summary
|
||||
.reconciliation_scopes
|
||||
.iter()
|
||||
.map(|scope| size_reconciliation_scope_key(&scope.bucket, &scope.object)),
|
||||
);
|
||||
self.pending_size_reconciliation_truncated |= summary.size_reconciliation_truncated;
|
||||
|
||||
for info in [&mut self.new_cache.info, &mut self.update_cache.info] {
|
||||
for incoming in &summary.size_reconciliation {
|
||||
if let Some(existing) = info.size_reconciliation.get_mut(&incoming.key) {
|
||||
existing.reason = incoming.reason.clone();
|
||||
existing.physical_size = incoming.physical_size;
|
||||
existing.generation = incoming.generation.clone();
|
||||
existing.version_id = incoming.version_id.clone();
|
||||
existing.attempts = existing.attempts.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
|
||||
if size_reconciliation_entry_bytes(incoming) > MAX_SIZE_RECONCILIATION_BYTES_PER_BUCKET {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut entry = incoming.clone();
|
||||
entry.first_seen = now;
|
||||
entry.attempts = 1;
|
||||
info.size_reconciliation.insert(entry.key.clone(), entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_size_reconciliation_batch(&mut self) {
|
||||
let now = Self::now_secs();
|
||||
let current_keys = std::mem::take(&mut self.pending_size_reconciliation_keys);
|
||||
let scopes = std::mem::take(&mut self.pending_size_reconciliation_scopes);
|
||||
let truncated = std::mem::replace(&mut self.pending_size_reconciliation_truncated, false);
|
||||
|
||||
for info in [&mut self.new_cache.info, &mut self.update_cache.info] {
|
||||
if !truncated {
|
||||
info.size_reconciliation.retain(|key, entry| {
|
||||
!scopes.contains(&size_reconciliation_scope_key(&entry.bucket, &entry.object)) || current_keys.contains(key)
|
||||
});
|
||||
}
|
||||
prune_size_reconciliation(info, now);
|
||||
}
|
||||
}
|
||||
|
||||
fn record_scan_resume_hint(&mut self, folder: &str) {
|
||||
self.new_cache.info.scan_resume_after = Some(folder.to_string());
|
||||
self.update_cache.info.scan_resume_after = Some(folder.to_string());
|
||||
@@ -1539,7 +1426,6 @@ impl FolderScanner {
|
||||
abandoned_children.remove(&path_join_buf(&[&item.bucket, &item.object_path()]));
|
||||
|
||||
apply_scanner_size_summary(into, &sz);
|
||||
self.apply_size_reconciliation(&sz);
|
||||
into.objects += 1;
|
||||
object_count += 1;
|
||||
self.budget.record_object_scanned();
|
||||
@@ -2219,7 +2105,6 @@ impl FolderScanner {
|
||||
}
|
||||
}
|
||||
|
||||
self.finish_size_reconciliation_batch();
|
||||
done_folder();
|
||||
let scanned_objects = u64::try_from(into.objects).unwrap_or(u64::MAX);
|
||||
emit_scanner_folder_trace(&self.root, &folder.name, scanned_objects, trace_started_at, "completed");
|
||||
@@ -2305,17 +2190,10 @@ pub async fn scan_data_folder(
|
||||
skip_heal,
|
||||
local_disk,
|
||||
pending_heals_changed: false,
|
||||
pending_size_reconciliation_keys: HashSet::new(),
|
||||
pending_size_reconciliation_scopes: HashSet::new(),
|
||||
pending_size_reconciliation_truncated: false,
|
||||
#[cfg(test)]
|
||||
list_path_raw_options_observer: None,
|
||||
};
|
||||
|
||||
let now = FolderScanner::now_secs();
|
||||
prune_size_reconciliation(&mut scanner.new_cache.info, now);
|
||||
prune_size_reconciliation(&mut scanner.update_cache.info, now);
|
||||
|
||||
// Check if context is cancelled
|
||||
if ctx.is_cancelled() {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
@@ -2339,9 +2217,7 @@ pub async fn scan_data_folder(
|
||||
new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN);
|
||||
new_cache.info.last_update = Some(SystemTime::now());
|
||||
new_cache.info.next_cycle = cache.info.next_cycle;
|
||||
let unresolved_objects = root.failed_objects > 0
|
||||
|| !new_cache.info.failed_objects.is_empty()
|
||||
|| !new_cache.info.size_reconciliation.is_empty();
|
||||
let unresolved_objects = root.failed_objects > 0 || !new_cache.info.failed_objects.is_empty();
|
||||
new_cache.info.snapshot_complete = !unresolved_objects;
|
||||
let had_scan_checkpoint = cache.info.scan_checkpoint.is_some() || new_cache.info.scan_checkpoint.is_some();
|
||||
new_cache.info.scan_resume_after = None;
|
||||
@@ -2369,7 +2245,7 @@ pub async fn scan_data_folder(
|
||||
if root_has_progress {
|
||||
new_cache.replace_hashed(&root_hash, &None, &root);
|
||||
}
|
||||
if partial_cache_is_useful(&root, pending_heals_changed) || !new_cache.info.size_reconciliation.is_empty() {
|
||||
if partial_cache_is_useful(&root, pending_heals_changed) {
|
||||
if new_cache.root().is_some() {
|
||||
new_cache.force_compact(DATA_SCANNER_COMPACT_AT_CHILDREN);
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
// limitations under the License.
|
||||
/// Per-object scan actions: ScannerItem, the get-size failure policy, and the heal/ILM admission helpers.
|
||||
use super::*;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
/// Cached folder information for scanning
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -33,263 +32,6 @@ pub(super) enum GetSizeFailureAction {
|
||||
HealMetadata { object: String },
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(super) enum SizeResolutionReason {
|
||||
CompressedSizeUnknown,
|
||||
InvalidPhysicalSize,
|
||||
UnsupportedCompression,
|
||||
InvalidObjectSize,
|
||||
InvalidPartSize,
|
||||
InvalidDeclaredSize,
|
||||
SizeOverflowOrMismatch,
|
||||
}
|
||||
|
||||
impl SizeResolutionReason {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::CompressedSizeUnknown => "compressed_size_unknown",
|
||||
Self::InvalidPhysicalSize => "invalid_physical_size",
|
||||
Self::UnsupportedCompression => "unsupported_compression",
|
||||
Self::InvalidObjectSize => "invalid_object_size",
|
||||
Self::InvalidPartSize => "invalid_part_size",
|
||||
Self::InvalidDeclaredSize => "invalid_declared_size",
|
||||
Self::SizeOverflowOrMismatch => "size_overflow_or_mismatch",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(super) enum SizeResolution {
|
||||
Known { logical: i64, physical: i64 },
|
||||
Unknown { physical: i64, reason: SizeResolutionReason },
|
||||
Corrupt { physical: i64, reason: SizeResolutionReason },
|
||||
}
|
||||
|
||||
impl SizeResolution {
|
||||
fn known_size(&self) -> Option<i64> {
|
||||
match self {
|
||||
Self::Known { logical, .. } => Some(*logical),
|
||||
Self::Unknown { .. } | Self::Corrupt { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn size_reconciliation_key(oi: &ObjectInfo, reason: SizeResolutionReason) -> String {
|
||||
let version = oi
|
||||
.version_id
|
||||
.filter(|version| !version.is_nil())
|
||||
.map(|version| version.to_string())
|
||||
.unwrap_or_default();
|
||||
let generation = oi
|
||||
.data_dir
|
||||
.filter(|generation| !generation.is_nil())
|
||||
.map(|generation| generation.to_string())
|
||||
.unwrap_or_default();
|
||||
// Length-prefix each component so an object key containing the separator
|
||||
// cannot alias another identity. S3 keys are bounded in normal operation;
|
||||
// oversized persisted values use a digest so a corrupt metadata record
|
||||
// cannot grow the ledger without bound.
|
||||
fn component(value: &str) -> String {
|
||||
const MAX_COMPONENT_LEN: usize = 512;
|
||||
if value.len() <= MAX_COMPONENT_LEN {
|
||||
return format!("{}:{}", value.len(), value);
|
||||
}
|
||||
let digest = Sha256::digest(value.as_bytes());
|
||||
let digest = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
|
||||
format!("hash:{}:{}", value.len(), digest)
|
||||
}
|
||||
format!(
|
||||
"{}|{}|{}|{}|{}",
|
||||
component(&oi.bucket),
|
||||
component(&oi.name),
|
||||
component(&version),
|
||||
component(&generation),
|
||||
component(reason.as_str())
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn bounded_reconciliation_field(value: &str) -> String {
|
||||
const MAX_FIELD_LEN: usize = 512;
|
||||
if value.len() <= MAX_FIELD_LEN {
|
||||
return value.to_string();
|
||||
}
|
||||
let digest = hex_simd::encode_to_string(Sha256::digest(value.as_bytes()), hex_simd::AsciiCase::Lower);
|
||||
let prefix_len = MAX_FIELD_LEN - 65;
|
||||
let prefix = value
|
||||
.char_indices()
|
||||
.take_while(|(offset, ch)| offset.saturating_add(ch.len_utf8()) <= prefix_len)
|
||||
.map(|(_, ch)| ch)
|
||||
.collect::<String>();
|
||||
format!("{}~{}", prefix, digest)
|
||||
}
|
||||
|
||||
fn record_size_resolution(summary: &mut SizeSummary, oi: &ObjectInfo, resolution: &SizeResolution) {
|
||||
match resolution {
|
||||
SizeResolution::Known { .. } => {}
|
||||
SizeResolution::Unknown { physical, reason } | SizeResolution::Corrupt { physical, reason } => {
|
||||
summary.record_size_reconciliation(SizeReconciliationEntry {
|
||||
key: size_reconciliation_key(oi, *reason),
|
||||
bucket: bounded_reconciliation_field(&oi.bucket),
|
||||
object: bounded_reconciliation_field(&oi.name),
|
||||
version_id: oi
|
||||
.version_id
|
||||
.filter(|version| !version.is_nil())
|
||||
.map(|version| version.to_string()),
|
||||
generation: oi
|
||||
.data_dir
|
||||
.filter(|generation| !generation.is_nil())
|
||||
.map(|generation| generation.to_string()),
|
||||
reason: reason.as_str().to_string(),
|
||||
physical_size: u64::try_from(*physical).ok(),
|
||||
first_seen: 0,
|
||||
attempts: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the size metadata once at the scanner trust boundary. A compressed
|
||||
/// -1 sentinel is valid legacy metadata, but it cannot participate in normal
|
||||
/// logical-size accounting or size-filtered lifecycle rules.
|
||||
pub(super) fn resolve_size(oi: &ObjectInfo) -> SizeResolution {
|
||||
let physical = oi.size;
|
||||
if physical < 0 {
|
||||
return SizeResolution::Corrupt {
|
||||
physical,
|
||||
reason: SizeResolutionReason::InvalidPhysicalSize,
|
||||
};
|
||||
}
|
||||
|
||||
let compressed = match oi.compression_read_plan() {
|
||||
Ok((_, _, compressed)) => compressed,
|
||||
Err(_) => {
|
||||
return SizeResolution::Corrupt {
|
||||
physical,
|
||||
reason: SizeResolutionReason::UnsupportedCompression,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
if oi.actual_size < -1 || (oi.actual_size == -1 && !compressed) {
|
||||
return SizeResolution::Corrupt {
|
||||
physical,
|
||||
reason: SizeResolutionReason::InvalidObjectSize,
|
||||
};
|
||||
}
|
||||
|
||||
// Match ObjectInfo::get_actual_size: a positive in-memory value is the
|
||||
// authoritative decoded size. Stale declared/part metadata must not turn
|
||||
// an otherwise valid object into a false corruption report.
|
||||
if oi.actual_size > 0 {
|
||||
return SizeResolution::Known {
|
||||
logical: oi.actual_size,
|
||||
physical,
|
||||
};
|
||||
}
|
||||
|
||||
if oi
|
||||
.parts
|
||||
.iter()
|
||||
.any(|part| part.actual_size < -1 || (part.actual_size < 0 && !compressed))
|
||||
{
|
||||
return SizeResolution::Corrupt {
|
||||
physical,
|
||||
reason: SizeResolutionReason::InvalidPartSize,
|
||||
};
|
||||
}
|
||||
|
||||
let declared = rustfs_utils::http::get_str(&oi.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE);
|
||||
let declared = match declared {
|
||||
Some(value) if value.is_empty() => {
|
||||
return SizeResolution::Corrupt {
|
||||
physical,
|
||||
reason: SizeResolutionReason::InvalidDeclaredSize,
|
||||
};
|
||||
}
|
||||
Some(value) => match value.parse::<i64>() {
|
||||
Ok(value) if value >= 0 => Some(value),
|
||||
_ => {
|
||||
return SizeResolution::Corrupt {
|
||||
physical,
|
||||
reason: SizeResolutionReason::InvalidDeclaredSize,
|
||||
};
|
||||
}
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
|
||||
let logical = match oi.get_actual_size() {
|
||||
Ok(size) if size == -1 && compressed && declared.is_none() => {
|
||||
return SizeResolution::Unknown {
|
||||
physical,
|
||||
reason: SizeResolutionReason::CompressedSizeUnknown,
|
||||
};
|
||||
}
|
||||
Ok(size) if size >= 0 => size,
|
||||
Ok(_) | Err(_) => {
|
||||
return SizeResolution::Corrupt {
|
||||
physical,
|
||||
reason: SizeResolutionReason::SizeOverflowOrMismatch,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
if compressed && logical == 0 && physical != 0 && oi.parts.is_empty() && declared.is_none() {
|
||||
return SizeResolution::Corrupt {
|
||||
physical,
|
||||
reason: SizeResolutionReason::SizeOverflowOrMismatch,
|
||||
};
|
||||
}
|
||||
|
||||
SizeResolution::Known { logical, physical }
|
||||
}
|
||||
|
||||
fn resolve_sizes(object_infos: &[ObjectInfo]) -> Vec<SizeResolution> {
|
||||
object_infos.iter().map(resolve_size).collect()
|
||||
}
|
||||
|
||||
fn lifecycle_rule_has_size_filter(lifecycle: &BucketLifecycleConfiguration, rule_id: &str) -> bool {
|
||||
let filter_has_size = |filter: &s3s::dto::LifecycleRuleFilter| {
|
||||
filter.object_size_greater_than.is_some()
|
||||
|| filter.object_size_less_than.is_some()
|
||||
|| filter
|
||||
.and
|
||||
.as_ref()
|
||||
.is_some_and(|and| and.object_size_greater_than.is_some() || and.object_size_less_than.is_some())
|
||||
};
|
||||
lifecycle
|
||||
.rules
|
||||
.iter()
|
||||
.find(|rule| {
|
||||
if rule_id.is_empty() {
|
||||
rule.id.as_deref().is_none_or(str::is_empty)
|
||||
} else {
|
||||
rule.id.as_deref() == Some(rule_id)
|
||||
}
|
||||
})
|
||||
.and_then(|rule| rule.filter.as_ref())
|
||||
.is_some_and(filter_has_size)
|
||||
}
|
||||
|
||||
fn lifecycle_event_allowed(resolution: &SizeResolution, event: &Event, lifecycle: &BucketLifecycleConfiguration) -> bool {
|
||||
match resolution {
|
||||
// Missing or invalid logical size only defers actions whose selected
|
||||
// rule actually depends on that size. Time/version-only actions retain
|
||||
// their existing semantics, including intrinsic events without a rule ID.
|
||||
SizeResolution::Unknown { .. } | SizeResolution::Corrupt { .. } => {
|
||||
!lifecycle_rule_has_size_filter(lifecycle, &event.rule_id)
|
||||
}
|
||||
SizeResolution::Known { .. } => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// A successful newer-noncurrent batch consumes both known and unresolved
|
||||
/// versions from the retained-version alert count. The two accounting paths
|
||||
/// are separate because only known sizes can contribute byte totals.
|
||||
fn remaining_versions_after_queued_noncurrent(remaining_versions: usize, known_count: usize, unknown_count: usize) -> usize {
|
||||
remaining_versions.saturating_sub(known_count.saturating_add(unknown_count))
|
||||
}
|
||||
|
||||
/// How the corrupt-metadata branch records the repair after attempting an
|
||||
/// MRF intent (backlog#1894 axis A).
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
@@ -577,48 +319,34 @@ impl ScannerItem {
|
||||
"Scanner lifecycle evaluation started"
|
||||
);
|
||||
|
||||
let resolved_sizes = resolve_sizes(&object_infos);
|
||||
if let Some(first) = object_infos.first() {
|
||||
size_summary.record_reconciliation_scope(
|
||||
&bounded_reconciliation_field(&first.bucket),
|
||||
&bounded_reconciliation_field(&first.name),
|
||||
);
|
||||
}
|
||||
for (oi, resolution) in object_infos.iter().zip(resolved_sizes.iter()) {
|
||||
record_size_resolution(size_summary, oi, resolution);
|
||||
}
|
||||
let has_corrupt_size = resolved_sizes
|
||||
.iter()
|
||||
.any(|resolution| matches!(resolution, SizeResolution::Corrupt { .. }));
|
||||
|
||||
// `versioning_config` is resolved once per object by the caller
|
||||
// (`get_size`) and handed in; only `prefix_enabled` is consulted here.
|
||||
|
||||
let Some(lifecycle) = self.lifecycle.clone() else {
|
||||
let mut cumulative_size: i64 = 0;
|
||||
for (oi, resolved_size) in object_infos.iter().zip(resolved_sizes.iter()) {
|
||||
let accounting_size = match resolved_size {
|
||||
SizeResolution::Known { logical, .. } => *logical,
|
||||
// A valid compressed legacy sentinel has no logical size,
|
||||
// but heal and replication still need to run. The
|
||||
// physical size is only an input to those operations; it
|
||||
// is not folded into the logical total below.
|
||||
SizeResolution::Unknown { physical, .. } => {
|
||||
self.heal_actions(oi, *physical, size_summary).await;
|
||||
size_summary.actions_accounting_unknown(oi);
|
||||
continue;
|
||||
}
|
||||
SizeResolution::Corrupt { .. } => {
|
||||
size_summary.actions_accounting_unknown(oi);
|
||||
let Some(lifecycle) = self.lifecycle.as_ref() else {
|
||||
let mut cumulative_size = 0;
|
||||
for oi in object_infos.iter() {
|
||||
let actual_size = match oi.get_actual_size() {
|
||||
Ok(size) => size,
|
||||
Err(_) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_LIFECYCLE_ACTION,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %self.bucket,
|
||||
object = %oi.name,
|
||||
state = "size_lookup_failed",
|
||||
"Scanner lifecycle action used fallback size"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let size = self.heal_actions(oi, accounting_size, size_summary).await;
|
||||
let size = self.heal_actions(oi, actual_size, size_summary).await;
|
||||
|
||||
size_summary.actions_accounting(oi, size, accounting_size);
|
||||
size_summary.actions_accounting(oi, size, actual_size);
|
||||
|
||||
cumulative_size = cumulative_size.saturating_add(size);
|
||||
cumulative_size += size;
|
||||
}
|
||||
|
||||
self.alert_excessive_versions(object_infos.len(), cumulative_size);
|
||||
@@ -672,108 +400,25 @@ impl ScannerItem {
|
||||
let mut to_delete_objs: Vec<ObjectToDelete> = Vec::new();
|
||||
let mut noncurrent_events: Vec<Event> = Vec::new();
|
||||
let mut noncurrent_accounting: Vec<PendingScannerAccounting<'_>> = Vec::new();
|
||||
let mut noncurrent_unknown: Vec<&ObjectInfo> = Vec::new();
|
||||
let mut cumulative_size = 0;
|
||||
let mut remaining_versions = object_infos.len();
|
||||
'eventLoop: {
|
||||
for (i, event) in events.iter().enumerate() {
|
||||
let oi = &object_infos[i];
|
||||
let known_size = resolved_sizes[i].known_size();
|
||||
if has_corrupt_size
|
||||
&& matches!(
|
||||
event.action,
|
||||
IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction
|
||||
)
|
||||
{
|
||||
// An all-version delete would also remove a corrupt
|
||||
// sibling that could not be reconciled safely.
|
||||
continue;
|
||||
}
|
||||
if !lifecycle_event_allowed(&resolved_sizes[i], event, &lifecycle) {
|
||||
// An unknown logical size must not make an otherwise
|
||||
// non-destructive scan disappear from heal/physical-tier
|
||||
// accounting. Size-filtered or deferred events remain
|
||||
// pending, so retain the version-only physical counters.
|
||||
if let SizeResolution::Unknown { physical, .. } = &resolved_sizes[i] {
|
||||
self.heal_actions(oi, *physical, size_summary).await;
|
||||
size_summary.actions_accounting_unknown(oi);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let actual_size = match known_size {
|
||||
Some(size) => size,
|
||||
None => {
|
||||
match event.action {
|
||||
IlmAction::DeleteAction
|
||||
| IlmAction::DeleteRestoredAction
|
||||
| IlmAction::DeleteRestoredVersionAction
|
||||
| IlmAction::DeleteAllVersionsAction
|
||||
| IlmAction::DelMarkerDeleteAllVersionsAction => {
|
||||
let done_ilm = Metrics::time_ilm(event.action);
|
||||
let trace_started_at = trace_start_instant();
|
||||
let queued = apply_expiry_rule(event, &LcEventSrc::Scanner, oi).await;
|
||||
emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at);
|
||||
if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
|
||||
done_ilm(1)();
|
||||
if event.action == IlmAction::DeleteAllVersionsAction
|
||||
|| event.action == IlmAction::DelMarkerDeleteAllVersionsAction
|
||||
{
|
||||
remaining_versions = 0;
|
||||
}
|
||||
} else if matches!(
|
||||
event.action,
|
||||
IlmAction::DeleteAction
|
||||
| IlmAction::DeleteRestoredAction
|
||||
| IlmAction::DeleteRestoredVersionAction
|
||||
) {
|
||||
size_summary.actions_accounting_unknown(oi);
|
||||
} else {
|
||||
size_summary.actions_accounting_unknown(oi);
|
||||
for (j, retained) in object_infos.iter().enumerate().skip(i + 1) {
|
||||
match &resolved_sizes[j] {
|
||||
SizeResolution::Known { logical, .. } => PendingScannerAccounting {
|
||||
object: retained,
|
||||
retained_size: *logical,
|
||||
expired_size: 0,
|
||||
}
|
||||
.apply(size_summary, &mut cumulative_size, false),
|
||||
SizeResolution::Unknown { .. } => {
|
||||
size_summary.actions_accounting_unknown(retained);
|
||||
}
|
||||
SizeResolution::Corrupt { .. } => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
IlmAction::DeleteVersionAction => {
|
||||
if let Some(opt) = object_opts.get(i) {
|
||||
to_delete_objs.push(ObjectToDelete {
|
||||
object_name: opt.name.clone(),
|
||||
version_id: opt.version_id,
|
||||
..Default::default()
|
||||
});
|
||||
noncurrent_events.push(event.clone());
|
||||
noncurrent_unknown.push(oi);
|
||||
}
|
||||
}
|
||||
IlmAction::TransitionAction | IlmAction::TransitionVersionAction => {
|
||||
let trace_started_at = trace_start_instant();
|
||||
let queued = apply_transition_rule(event, &LcEventSrc::Scanner, oi).await;
|
||||
emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at);
|
||||
if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
|
||||
let done_ilm = Metrics::time_ilm(event.action);
|
||||
done_ilm(1)();
|
||||
}
|
||||
size_summary.actions_accounting_unknown(oi);
|
||||
}
|
||||
IlmAction::NoneAction | IlmAction::ActionCount => {
|
||||
if let SizeResolution::Unknown { physical, .. } = &resolved_sizes[i] {
|
||||
self.heal_actions(oi, *physical, size_summary).await;
|
||||
}
|
||||
size_summary.actions_accounting_unknown(oi);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
let actual_size = match oi.get_actual_size() {
|
||||
Ok(size) => size,
|
||||
Err(_) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_LIFECYCLE_ACTION,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %self.bucket,
|
||||
object = %oi.name,
|
||||
state = "size_lookup_failed",
|
||||
"Scanner lifecycle action used fallback size"
|
||||
);
|
||||
0
|
||||
}
|
||||
};
|
||||
|
||||
@@ -801,24 +446,36 @@ impl ScannerItem {
|
||||
done_ilm(1)();
|
||||
remaining_versions = 0;
|
||||
} else {
|
||||
if let Some(actual_size) = known_size {
|
||||
PendingScannerAccounting {
|
||||
object: oi,
|
||||
retained_size: actual_size,
|
||||
expired_size: 0,
|
||||
}
|
||||
.apply(size_summary, &mut cumulative_size, false);
|
||||
for retained in object_infos.iter().skip(i + 1) {
|
||||
let retained_size = match retained.get_actual_size() {
|
||||
Ok(size) => size,
|
||||
Err(_) => {
|
||||
warn!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_LIFECYCLE_ACTION,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %self.bucket,
|
||||
object = %retained.name,
|
||||
state = "size_lookup_failed",
|
||||
"Scanner lifecycle action used fallback size"
|
||||
);
|
||||
0
|
||||
}
|
||||
};
|
||||
PendingScannerAccounting {
|
||||
object: oi,
|
||||
retained_size: actual_size,
|
||||
object: retained,
|
||||
retained_size,
|
||||
expired_size: 0,
|
||||
}
|
||||
.apply(size_summary, &mut cumulative_size, false);
|
||||
}
|
||||
for (j, retained) in object_infos.iter().enumerate().skip(i + 1) {
|
||||
if let Some(retained_size) = resolved_sizes[j].known_size() {
|
||||
PendingScannerAccounting {
|
||||
object: retained,
|
||||
retained_size,
|
||||
expired_size: 0,
|
||||
}
|
||||
.apply(size_summary, &mut cumulative_size, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
break 'eventLoop;
|
||||
}
|
||||
@@ -854,13 +511,11 @@ impl ScannerItem {
|
||||
version_id: opt.version_id,
|
||||
..Default::default()
|
||||
});
|
||||
if let Some(actual_size) = known_size {
|
||||
noncurrent_accounting.push(PendingScannerAccounting {
|
||||
object: oi,
|
||||
retained_size: actual_size,
|
||||
expired_size: 0,
|
||||
});
|
||||
}
|
||||
noncurrent_accounting.push(PendingScannerAccounting {
|
||||
object: oi,
|
||||
retained_size: actual_size,
|
||||
expired_size: 0,
|
||||
});
|
||||
account_now = false;
|
||||
}
|
||||
noncurrent_events.push(event.clone());
|
||||
@@ -893,7 +548,7 @@ impl ScannerItem {
|
||||
|
||||
if account_now {
|
||||
size_summary.actions_accounting(oi, size, actual_size);
|
||||
cumulative_size = cumulative_size.saturating_add(size);
|
||||
cumulative_size += size;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -921,20 +576,11 @@ impl ScannerItem {
|
||||
}
|
||||
if record_scanner_ilm_action_if_queued(global_metrics(), action, count, queued) {
|
||||
done_ilm(count)();
|
||||
remaining_versions = remaining_versions_after_queued_noncurrent(
|
||||
remaining_versions,
|
||||
noncurrent_accounting.len(),
|
||||
noncurrent_unknown.len(),
|
||||
);
|
||||
remaining_versions = remaining_versions.saturating_sub(noncurrent_accounting.len());
|
||||
}
|
||||
for pending in noncurrent_accounting {
|
||||
pending.apply(size_summary, &mut cumulative_size, queued);
|
||||
}
|
||||
if !queued {
|
||||
for object in noncurrent_unknown {
|
||||
size_summary.actions_accounting_unknown(object);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.alert_excessive_versions(remaining_versions, cumulative_size);
|
||||
}
|
||||
@@ -1283,394 +929,4 @@ mod tests {
|
||||
assert_eq!(item.object_name, "object");
|
||||
assert_eq!(item.object_path(), "object");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_resolution_rejects_negative_overflow_and_unknown_compression() {
|
||||
let compressed = |actual_size: i64, declared: Option<&str>| {
|
||||
let mut user_defined = HashMap::new();
|
||||
rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
|
||||
if let Some(declared) = declared {
|
||||
rustfs_utils::http::insert_str(&mut user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, declared.to_string());
|
||||
}
|
||||
ObjectInfo {
|
||||
size: 12,
|
||||
actual_size,
|
||||
user_defined: Arc::new(user_defined),
|
||||
..Default::default()
|
||||
}
|
||||
};
|
||||
|
||||
let normal = ObjectInfo {
|
||||
size: 12,
|
||||
actual_size: 10,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
resolve_size(&normal),
|
||||
SizeResolution::Known {
|
||||
logical: 10,
|
||||
physical: 12
|
||||
}
|
||||
);
|
||||
|
||||
let stale_declared_metadata = ObjectInfo {
|
||||
size: 12,
|
||||
actual_size: 10,
|
||||
user_defined: Arc::new(HashMap::from([("x-rustfs-internal-actual-size".to_string(), "not-a-size".to_string())])),
|
||||
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
|
||||
actual_size: -2,
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
resolve_size(&stale_declared_metadata),
|
||||
SizeResolution::Known {
|
||||
logical: 10,
|
||||
physical: 12
|
||||
}
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
resolve_size(&compressed(0, Some("9"))),
|
||||
SizeResolution::Known {
|
||||
logical: 9,
|
||||
physical: 12
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_size(&compressed(-1, None)),
|
||||
SizeResolution::Unknown {
|
||||
physical: 12,
|
||||
reason: SizeResolutionReason::CompressedSizeUnknown,
|
||||
}
|
||||
);
|
||||
assert!(matches!(
|
||||
resolve_size(&compressed(0, Some("not-a-size"))),
|
||||
SizeResolution::Corrupt {
|
||||
reason: SizeResolutionReason::InvalidDeclaredSize,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert!(matches!(
|
||||
resolve_size(&ObjectInfo {
|
||||
size: 12,
|
||||
actual_size: -2,
|
||||
..Default::default()
|
||||
}),
|
||||
SizeResolution::Corrupt { .. }
|
||||
));
|
||||
assert!(matches!(resolve_size(&compressed(0, Some("-1"))), SizeResolution::Corrupt { .. }));
|
||||
assert!(matches!(resolve_size(&compressed(0, Some(""))), SizeResolution::Corrupt { .. }));
|
||||
|
||||
let unsupported = {
|
||||
let mut object = compressed(0, None);
|
||||
let mut metadata = (*object.user_defined).clone();
|
||||
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "unsupported".to_string());
|
||||
object.user_defined = Arc::new(metadata);
|
||||
object
|
||||
};
|
||||
assert!(matches!(resolve_size(&unsupported), SizeResolution::Corrupt { .. }));
|
||||
|
||||
let invalid_part = {
|
||||
let mut object = compressed(0, None);
|
||||
object.parts = Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
|
||||
size: 12,
|
||||
actual_size: -2,
|
||||
..Default::default()
|
||||
}]);
|
||||
object
|
||||
};
|
||||
assert!(matches!(resolve_size(&invalid_part), SizeResolution::Corrupt { .. }));
|
||||
|
||||
let overflow = {
|
||||
let mut object = compressed(0, None);
|
||||
object.parts = Arc::new(vec![
|
||||
rustfs_filemeta::ObjectPartInfo {
|
||||
size: 1,
|
||||
actual_size: i64::MAX,
|
||||
..Default::default()
|
||||
},
|
||||
rustfs_filemeta::ObjectPartInfo {
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
},
|
||||
]);
|
||||
object
|
||||
};
|
||||
assert!(matches!(resolve_size(&overflow), SizeResolution::Corrupt { .. }));
|
||||
|
||||
let mismatch = compressed(0, None);
|
||||
assert!(matches!(resolve_size(&mismatch), SizeResolution::Corrupt { .. }));
|
||||
assert_eq!(
|
||||
resolve_size(&ObjectInfo {
|
||||
size: 0,
|
||||
actual_size: 0,
|
||||
..Default::default()
|
||||
}),
|
||||
SizeResolution::Known { logical: 0, physical: 0 }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_resolution_records_and_replays_one_identity() {
|
||||
let version_id = uuid::Uuid::new_v4();
|
||||
let generation = uuid::Uuid::new_v4();
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
|
||||
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "not-a-number".to_string());
|
||||
let corrupt = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 12,
|
||||
version_id: Some(version_id),
|
||||
data_dir: Some(generation),
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut summary = SizeSummary::default();
|
||||
let resolution = resolve_size(&corrupt);
|
||||
record_size_resolution(&mut summary, &corrupt, &resolution);
|
||||
record_size_resolution(&mut summary, &corrupt, &resolution);
|
||||
assert_eq!(summary.size_reconciliation.len(), 1);
|
||||
assert_eq!(summary.size_reconciliation[0].reason, "invalid_declared_size");
|
||||
assert_eq!(summary.size_reconciliation[0].physical_size, Some(12));
|
||||
|
||||
let known = ObjectInfo {
|
||||
actual_size: 12,
|
||||
user_defined: Arc::new(HashMap::new()),
|
||||
..corrupt.clone()
|
||||
};
|
||||
record_size_resolution(&mut summary, &known, &resolve_size(&known));
|
||||
summary.record_reconciliation_scope(&known.bucket, &known.name);
|
||||
assert_eq!(summary.reconciliation_scopes.len(), 1);
|
||||
assert_eq!(summary.reconciliation_scopes[0].bucket, "bucket");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_size_has_same_ilm_accounting() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
|
||||
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "invalid".to_string());
|
||||
let object = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 12,
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
};
|
||||
let resolution = resolve_size(&object);
|
||||
let mut without_ilm = SizeSummary::default();
|
||||
let mut with_ilm = SizeSummary::default();
|
||||
record_size_resolution(&mut without_ilm, &object, &resolution);
|
||||
record_size_resolution(&mut with_ilm, &object, &resolution);
|
||||
assert_eq!(without_ilm.size_reconciliation, with_ilm.size_reconciliation);
|
||||
assert_eq!(without_ilm.total_size, 0);
|
||||
assert_eq!(with_ilm.total_size, 0);
|
||||
assert!(without_ilm.tier_stats.is_empty());
|
||||
assert!(with_ilm.tier_stats.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_resolution_parses_once_per_version() {
|
||||
let objects = vec![
|
||||
ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "one".to_string(),
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
},
|
||||
ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "two".to_string(),
|
||||
size: 2,
|
||||
actual_size: -2,
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
let resolutions = resolve_sizes(&objects);
|
||||
assert_eq!(resolutions.len(), objects.len());
|
||||
assert!(matches!(resolutions[0], SizeResolution::Known { logical: 1, .. }));
|
||||
assert!(matches!(resolutions[1], SizeResolution::Corrupt { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queued_unknown_noncurrent_versions_are_removed_from_alert_count() {
|
||||
assert_eq!(remaining_versions_after_queued_noncurrent(3, 1, 2), 0);
|
||||
assert_eq!(remaining_versions_after_queued_noncurrent(7, 2, 1), 4);
|
||||
assert_eq!(remaining_versions_after_queued_noncurrent(usize::MAX, usize::MAX, usize::MAX), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_size_blocks_size_dependent_transition_but_allows_time_only_expiry() {
|
||||
let size_filtered = BucketLifecycleConfiguration {
|
||||
rules: vec![s3s::dto::LifecycleRule {
|
||||
status: s3s::dto::ExpirationStatus::from_static(s3s::dto::ExpirationStatus::ENABLED),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
id: Some("size".to_string()),
|
||||
filter: Some(s3s::dto::LifecycleRuleFilter {
|
||||
object_size_greater_than: Some(1),
|
||||
..Default::default()
|
||||
}),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
let unknown = SizeResolution::Unknown {
|
||||
physical: 12,
|
||||
reason: SizeResolutionReason::CompressedSizeUnknown,
|
||||
};
|
||||
let size_event = Event {
|
||||
action: IlmAction::DeleteAction,
|
||||
rule_id: "size".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!lifecycle_event_allowed(&unknown, &size_event, &size_filtered));
|
||||
assert!(!lifecycle_event_allowed(
|
||||
&unknown,
|
||||
&Event {
|
||||
action: IlmAction::TransitionAction,
|
||||
rule_id: "size".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
&size_filtered
|
||||
));
|
||||
let mixed_filters = BucketLifecycleConfiguration {
|
||||
rules: vec![
|
||||
size_filtered.rules[0].clone(),
|
||||
s3s::dto::LifecycleRule {
|
||||
status: s3s::dto::ExpirationStatus::from_static(s3s::dto::ExpirationStatus::ENABLED),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
id: Some("time".to_string()),
|
||||
filter: None,
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
},
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(lifecycle_event_allowed(
|
||||
&unknown,
|
||||
&Event {
|
||||
action: IlmAction::DeleteAction,
|
||||
rule_id: "time".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
&mixed_filters
|
||||
));
|
||||
assert!(lifecycle_event_allowed(
|
||||
&unknown,
|
||||
&Event {
|
||||
action: IlmAction::TransitionAction,
|
||||
..Default::default()
|
||||
},
|
||||
&BucketLifecycleConfiguration::default()
|
||||
));
|
||||
assert!(lifecycle_event_allowed(
|
||||
&SizeResolution::Corrupt {
|
||||
physical: 12,
|
||||
reason: SizeResolutionReason::InvalidDeclaredSize,
|
||||
},
|
||||
&Event {
|
||||
action: IlmAction::DeleteAction,
|
||||
..Default::default()
|
||||
},
|
||||
&BucketLifecycleConfiguration::default()
|
||||
));
|
||||
assert!(!lifecycle_event_allowed(
|
||||
&SizeResolution::Corrupt {
|
||||
physical: 12,
|
||||
reason: SizeResolutionReason::InvalidDeclaredSize,
|
||||
},
|
||||
&Event {
|
||||
action: IlmAction::DeleteAction,
|
||||
rule_id: "size".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
&size_filtered
|
||||
));
|
||||
assert!(lifecycle_rule_has_size_filter(
|
||||
&BucketLifecycleConfiguration {
|
||||
rules: vec![s3s::dto::LifecycleRule {
|
||||
status: s3s::dto::ExpirationStatus::from_static(s3s::dto::ExpirationStatus::ENABLED),
|
||||
expiration: None,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
id: None,
|
||||
filter: Some(s3s::dto::LifecycleRuleFilter {
|
||||
object_size_greater_than: Some(1),
|
||||
..Default::default()
|
||||
}),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
..Default::default()
|
||||
},
|
||||
""
|
||||
));
|
||||
assert!(lifecycle_event_allowed(
|
||||
&SizeResolution::Known {
|
||||
logical: 10,
|
||||
physical: 12,
|
||||
},
|
||||
&Event {
|
||||
action: IlmAction::DeleteAllVersionsAction,
|
||||
..Default::default()
|
||||
},
|
||||
&BucketLifecycleConfiguration::default()
|
||||
));
|
||||
assert!(lifecycle_event_allowed(
|
||||
&unknown,
|
||||
&Event {
|
||||
action: IlmAction::DeleteAction,
|
||||
rule_id: "time-only".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
&BucketLifecycleConfiguration::default()
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn long_object_size_reconciliation_scope_uses_bounded_identity() {
|
||||
let object_name = "o".repeat(600);
|
||||
let mut item = scanner_item_with_prefix("");
|
||||
item.object_name = object_name.clone();
|
||||
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
|
||||
let object = ObjectInfo {
|
||||
bucket: item.bucket.clone(),
|
||||
name: object_name.clone(),
|
||||
size: 12,
|
||||
actual_size: -1,
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
};
|
||||
let mut summary = SizeSummary::default();
|
||||
item.apply_actions(vec![object], None, VersioningConfiguration::default(), &mut summary)
|
||||
.await;
|
||||
|
||||
let bounded_bucket = bounded_reconciliation_field(&item.bucket);
|
||||
let bounded_object = bounded_reconciliation_field(&object_name);
|
||||
assert_eq!(summary.reconciliation_scopes[0].bucket, bounded_bucket);
|
||||
assert_eq!(summary.reconciliation_scopes[0].object, bounded_object);
|
||||
assert_eq!(summary.size_reconciliation[0].object, bounded_object);
|
||||
assert_eq!(summary.versions, 1);
|
||||
assert_eq!(summary.total_size, 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -326,9 +326,6 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
|
||||
skip_heal: Arc::new(AtomicBool::new(false)),
|
||||
local_disk: disk,
|
||||
pending_heals_changed: false,
|
||||
pending_size_reconciliation_keys: HashSet::new(),
|
||||
pending_size_reconciliation_scopes: HashSet::new(),
|
||||
pending_size_reconciliation_truncated: false,
|
||||
list_path_raw_options_observer: None,
|
||||
};
|
||||
|
||||
@@ -391,66 +388,6 @@ async fn test_record_failed_ttl_zero_noop() {
|
||||
assert!(!scanner.should_skip_failed("path2"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_size_reconciliation_replays_after_restart() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir);
|
||||
|
||||
let entry = SizeReconciliationEntry {
|
||||
key: "1:b|6:object|0:|0:".to_string(),
|
||||
bucket: "b".to_string(),
|
||||
object: "object".to_string(),
|
||||
reason: "invalid_declared_size".to_string(),
|
||||
physical_size: Some(12),
|
||||
..Default::default()
|
||||
};
|
||||
let mut summary = SizeSummary::default();
|
||||
summary.record_size_reconciliation(entry.clone());
|
||||
summary.record_reconciliation_scope("b", "object");
|
||||
scanner.apply_size_reconciliation(&summary);
|
||||
scanner.apply_size_reconciliation(&summary);
|
||||
|
||||
assert_eq!(scanner.new_cache.info.size_reconciliation.len(), 1);
|
||||
assert_eq!(scanner.update_cache.info.size_reconciliation.len(), 1);
|
||||
assert_eq!(scanner.new_cache.info.size_reconciliation[&entry.key].attempts, 2);
|
||||
|
||||
let encoded = rmp_serde::to_vec_named(&scanner.new_cache.info).expect("size ledger should encode");
|
||||
let decoded: crate::data_usage_define::DataUsageCacheInfo =
|
||||
rmp_serde::from_slice(&encoded).expect("size ledger should decode");
|
||||
assert_eq!(decoded.size_reconciliation.len(), 1);
|
||||
assert_eq!(decoded.size_reconciliation[&entry.key].reason, "invalid_declared_size");
|
||||
|
||||
let mut resolved = SizeSummary::default();
|
||||
resolved.record_reconciliation_scope("b", "object");
|
||||
scanner.apply_size_reconciliation(&resolved);
|
||||
assert!(scanner.new_cache.info.size_reconciliation.is_empty());
|
||||
assert!(scanner.update_cache.info.size_reconciliation.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_size_reconciliation_clears_bounded_long_object_scope() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir);
|
||||
let long_object = "o".repeat(600);
|
||||
let bounded_object = item_actions::bounded_reconciliation_field(&long_object);
|
||||
let entry = SizeReconciliationEntry {
|
||||
key: "long-object-key".to_string(),
|
||||
bucket: "b".to_string(),
|
||||
object: bounded_object,
|
||||
reason: "invalid_declared_size".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut summary = SizeSummary::default();
|
||||
summary.record_size_reconciliation(entry);
|
||||
scanner.apply_size_reconciliation(&summary);
|
||||
assert_eq!(scanner.new_cache.info.size_reconciliation.len(), 1);
|
||||
|
||||
let mut resolved = SizeSummary::default();
|
||||
resolved.record_reconciliation_scope("b", &long_object);
|
||||
scanner.apply_size_reconciliation(&resolved);
|
||||
assert!(scanner.new_cache.info.size_reconciliation.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_classify_get_size_failure_marks_metadata_heal_object_path() {
|
||||
let temp_dir = std::env::temp_dir();
|
||||
|
||||
@@ -554,10 +554,11 @@ fn apply_replication_timestamps_from_headers(headers: &HeaderMap<HeaderValue>, o
|
||||
|
||||
// Persist into the internal metadata keys so a later outbound replication
|
||||
// pass (replication_target_boundary) reads the source's modification
|
||||
// times instead of falling back to mod_time.
|
||||
// TODO(P1-6): receiver-side LWW is still missing — when the stored
|
||||
// per-category timestamp is newer than the inbound one, the existing
|
||||
// tags/retention/legal-hold should win instead of being overwritten.
|
||||
// times instead of falling back to mod_time. Receiver-side LWW happens at
|
||||
// the set layer under the object write lock
|
||||
// (ecstore set_disk::ops::object::merge_replication_metadata_lww,
|
||||
// rustfs/backlog#1953): a category whose stored timestamp is newer than
|
||||
// the inbound one keeps the local values.
|
||||
for (timestamp, suffix) in [
|
||||
(opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP),
|
||||
(opts.replication_retention_timestamp, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP),
|
||||
|
||||
@@ -241,7 +241,7 @@ env \
|
||||
RUSTFS_TEST_VAULT_FAILOVER_MARKER="$MARKER" \
|
||||
RUSTFS_TEST_VAULT_OLD_LEADER="$OLD_LEADER" \
|
||||
cargo test -p rustfs-kms --test vault_ha_failover_live \
|
||||
vault_raft_leader_failure_recovers_kv2_and_transit_decrypts -- \
|
||||
vault_raft_leader_failure_preserves_kv2_and_transit_decrypts -- \
|
||||
--ignored --nocapture --test-threads=1 &
|
||||
TEST_PID=$!
|
||||
|
||||
|
||||
Reference in New Issue
Block a user