Compare commits

..

5 Commits

Author SHA1 Message Date
houseme 5a5a759176 Merge branch 'main' into fix/b5-t6-a4-metadata-lww 2026-08-23 12:29:23 +08:00
唐小鸭 290662bc3a test(e2e): drop serial_test from receiver LWW e2e
main removed the serial_test dependency from e2e_test (#6344); the test
uses an ephemeral-port RustFSTestEnvironment and needs no cross-test
serialization, so the attribute and import go.
2026-08-23 00:26:43 +08:00
唐小鸭 858cf20d9a fix(replication): keep first-replication miss quiet in multipart LWW log
Version-absent is the normal first delivery of a version, not a
degraded comparison skip; only real read errors (quorum loss) warrant
the warn added in the previous commit.
2026-08-21 19:35:22 +08:00
唐小鸭 66e63d20dd fix(replication): surface skipped multipart LWW comparison at warn level
When the complete-multipart LWW gate cannot read the destination
version (quorum error), the inbound metadata is applied without
comparison — the exact overwrite rustfs/backlog#1953 exists to prevent.
Keep the fail-open semantics (failing the complete would loop through
MRF) but log the degraded path at warn so operators can see it; the
version-absent first replication keeps riding the same branch.
2026-08-21 19:33:59 +08:00
唐小鸭 81ef2882df fix(replication): apply receiver-side LWW to inbound metadata categories
Metadata-only replication reuses the whole-object PUT/multipart
transports, so in active-active topologies an inbound authorized
replication write carried the source's tags, retention, and legal hold
verbatim and unconditionally overwrote a category the destination had
modified more recently — both sites ended permanently diverged while
reporting COMPLETED (rustfs/backlog#1953, audit A4/P1-6).

The receiver now judges each category independently under the object
write lock: when the destination version's stored internal timestamp is
strictly newer than the inbound source timestamp, the local category
values and timestamp are kept; the rest of the write proceeds per the
inbound metadata and the object-level result stays successful, so a
local win never feeds an MRF retry loop. When the inbound category wins,
its internal timestamp key is pinned to the source-authored time instead
of the receiver-now() value stamped by the object-lock eval_metadata
path. No stored timestamp (pre-P1-6 data) or no inbound timestamp keeps
today's overwrite behavior.

The put_object hook reuses the existing commit-lock WORM-gate read (no
extra fanout); complete_multipart_upload adds one gated read under its
held lock, and the sender's complete options now carry the three
category timestamps so the multipart transport gets the same receiver
behavior. Local wins restore the timestamp via insert_str so a
MinIO-written single-key version still yields both compatibility keys.
2026-08-21 19:29:26 +08:00
29 changed files with 1051 additions and 1936 deletions
+5
View File
@@ -61,6 +61,11 @@ mod get_codec_streaming_compat_test;
#[cfg(test)]
mod version_id_regression_test;
// Receiver-side replication LWW (rustfs/backlog#1953): stale inbound
// replication metadata must not overwrite a newer local category state.
#[cfg(test)]
mod replication_lww_receiver_test;
// Data usage regression tests
#[cfg(test)]
mod data_usage_test;
@@ -0,0 +1,150 @@
#![cfg(test)]
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Receiver-side replication LWW over the wire (rustfs/backlog#1953, audit
//! A4/P1-6).
//!
//! In an active-active topology both sites' metadata states arrive at the
//! peer as authorized replication PUTs carrying per-category source
//! timestamps (`x-rustfs-source-replication-tagging-timestamp` header
//! family). Before the fix the receiver applied them unconditionally, so a
//! stale delivery overwrote a newer local state and the two sites diverged
//! permanently while both reported COMPLETED. This test drives one live
//! `rustfs` server with simulated inbound replication PUTs for the same
//! object version and asserts the newer tagging state wins regardless of
//! delivery order, while a stale delivery still succeeds at the object level
//! (a failure would loop through MRF re-delivering the stale value).
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-request";
const HDR_SOURCE_VERSION_ID: &str = "x-rustfs-source-version-id";
const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
const HDR_SOURCE_TAGGING_TIMESTAMP: &str = "x-rustfs-source-replication-tagging-timestamp";
const SOURCE_MTIME: &str = "2026-01-01T00:00:00Z";
const T_STALE: &str = "2026-01-01T00:00:01Z";
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
const T_NEWER: &str = "2026-03-01T00:00:00Z";
/// Simulated inbound authorized replication PUT: same object version, tags and
/// the source-authored tagging timestamp carried in transport headers.
async fn inbound_replication_put(
client: &Client,
bucket: &str,
key: &str,
version_id: &str,
tags: &str,
tagging_timestamp: &str,
) -> TestResult {
let version_id = version_id.to_string();
let tagging_timestamp = tagging_timestamp.to_string();
client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from_static(b"lww-e2e-body"))
.tagging(tags)
.customize()
.mutate_request(move |req| {
req.headers_mut().insert(HDR_SOURCE_REPLICATION_REQUEST, "true");
req.headers_mut().insert(HDR_SOURCE_VERSION_ID, version_id.clone());
req.headers_mut().insert(HDR_SOURCE_MTIME, SOURCE_MTIME);
req.headers_mut()
.insert(HDR_SOURCE_TAGGING_TIMESTAMP, tagging_timestamp.clone());
})
.send()
.await?;
Ok(())
}
async fn tag_value(client: &Client, bucket: &str, key: &str, version_id: &str, tag_key: &str) -> Option<String> {
let tagging = client
.get_object_tagging()
.bucket(bucket)
.key(key)
.version_id(version_id)
.send()
.await
.expect("object tagging should be readable");
tagging
.tag_set()
.iter()
.find(|tag| tag.key() == tag_key)
.map(|tag| tag.value().to_string())
}
#[tokio::test(flavor = "multi_thread")]
async fn receiver_lww_keeps_newer_tags_across_delivery_orders() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let client = env.create_s3_client();
let bucket = "replication-lww-receiver";
let key = "object";
client.create_bucket().bucket(bucket).send().await?;
client
.put_bucket_versioning()
.bucket(bucket)
.versioning_configuration(
VersioningConfiguration::builder()
.status(BucketVersioningStatus::Enabled)
.build(),
)
.send()
.await?;
// First delivery establishes version V with tags stamped T_LOCAL.
let version_id = uuid::Uuid::new_v4().to_string();
inbound_replication_put(&client, bucket, key, &version_id, "site=local", T_LOCAL).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("local"),
"the first delivery must establish the tagged version"
);
// A stale delivery (older source timestamp) must succeed at the object
// level but must NOT overwrite the newer tags.
inbound_replication_put(&client, bucket, key, &version_id, "site=stale", T_STALE).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("local"),
"a stale inbound delivery must not overwrite newer tags (rustfs/backlog#1953)"
);
// A newer delivery still converges the version onto the newest state.
inbound_replication_put(&client, bucket, key, &version_id, "site=newer", T_NEWER).await?;
assert_eq!(
tag_value(&client, bucket, key, &version_id, "site").await.as_deref(),
Some("newer"),
"a newer inbound delivery must overwrite older tags"
);
client
.delete_object()
.bucket(bucket)
.key(key)
.version_id(&version_id)
.send()
.await?;
env.delete_test_bucket(bucket).await.ok();
Ok(())
}
@@ -3868,6 +3868,7 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
actual_size,
object_info.etag.clone().unwrap_or_default(),
object_info.mod_time,
&put_opts.internal,
),
)
.await
@@ -472,6 +472,7 @@ pub(crate) fn replication_complete_multipart_options(
actual_size: String,
source_etag: String,
source_mtime: Option<OffsetDateTime>,
source_internal: &AdvancedPutOptions,
) -> PutObjectOptions {
let mut user_metadata = HashMap::new();
insert_header_map(&mut user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, actual_size);
@@ -484,6 +485,14 @@ pub(crate) fn replication_complete_multipart_options(
// mtime must degrade to epoch so header() suppresses the header
// instead of asserting the replication time as the object's mtime.
source_mtime: source_mtime.unwrap_or(OffsetDateTime::UNIX_EPOCH),
// Carry the per-category LWW timestamps on the complete request as
// well: the receiver's CompleteMultipartUpload options builder
// parses the same headers, so the multipart transport gets the
// same receiver-side LWW as the single-PUT transport
// (rustfs/backlog#1953). Epoch values keep the headers suppressed.
tagging_timestamp: source_internal.tagging_timestamp,
retention_timestamp: source_internal.retention_timestamp,
legalhold_timestamp: source_internal.legalhold_timestamp,
replication_status: ReplicationStatusType::Replica,
replication_request: true,
..Default::default()
@@ -663,20 +672,39 @@ mod tests {
#[test]
fn replication_complete_multipart_options_sets_actual_size() {
let source_mtime = OffsetDateTime::from_unix_timestamp(1_716_170_000).expect("valid test timestamp");
let source_internal = AdvancedPutOptions {
tagging_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_100).expect("valid test timestamp"),
retention_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_200).expect("valid test timestamp"),
legalhold_timestamp: OffsetDateTime::from_unix_timestamp(1_716_170_300).expect("valid test timestamp"),
..Default::default()
};
let options = replication_complete_multipart_options(
"1024".to_string(),
"0123456789abcdef0123456789abcdef-3".to_string(),
Some(source_mtime),
&source_internal,
);
assert_eq!(options.internal.source_etag, "0123456789abcdef0123456789abcdef-3");
assert_eq!(options.internal.source_mtime, source_mtime);
// The complete request must carry the same per-category LWW timestamps
// as the initiate request; the receiver reads them from the complete
// headers (rustfs/backlog#1953).
assert_eq!(options.internal.tagging_timestamp, source_internal.tagging_timestamp);
assert_eq!(options.internal.retention_timestamp, source_internal.retention_timestamp);
assert_eq!(options.internal.legalhold_timestamp, source_internal.legalhold_timestamp);
// Absent source mtime must degrade to epoch (header suppressed), not
// the AdvancedPutOptions default of now_utc() — that default would
// stamp the replication time as the replica's mtime and break the
// multipart HEAD convergence.
let options_no_mtime = replication_complete_multipart_options("1024".to_string(), String::new(), None);
// multipart HEAD convergence. Unset category timestamps stay epoch so
// header() keeps suppressing them.
let options_no_mtime =
replication_complete_multipart_options("1024".to_string(), String::new(), None, &AdvancedPutOptions::default());
assert_eq!(options_no_mtime.internal.source_mtime.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.tagging_timestamp.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.retention_timestamp.unix_timestamp(), 0);
assert_eq!(options_no_mtime.internal.legalhold_timestamp.unix_timestamp(), 0);
assert_eq!(
get_header_map(&options.user_metadata, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE).as_deref(),
@@ -1089,9 +1089,7 @@ impl PeerRestClient {
.await?
.max_decoding_message_size(BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE);
let response = match client
.background_heal_status(Request::new(BackgroundHealStatusRequest {
protocol_version: rustfs_protos::BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION,
}))
.background_heal_status(Request::new(BackgroundHealStatusRequest::default()))
.await
{
Ok(response) => response.into_inner(),
+6 -38
View File
@@ -784,24 +784,6 @@ pub(crate) fn create_deferred_bitrot_reader_with_stripe_handle(
///
/// # Returns
/// A Result containing the BitrotWriterWrapper or an error
/// Size hint handed to `DiskAPI::create_file` for a bitrot-wrapped shard.
///
/// A known length is grown by one checksum per shard so the on-disk file size
/// matches what the bitrot writer emits. A negative length is the
/// unknown-size sentinel (`HashReader::SIZE_PRESERVE_LAYER`, used by SSE and
/// compression) and must be preserved: `RemoteDisk::create_file` forwards it
/// in the `put_file_stream` query, and the receiver only treats `size > 0` as
/// a fixed body length when locating the authenticated trailer. Clamping it
/// to `0` would claim an empty body and misframe the stream. `0` stays `0`
/// because a genuinely empty object still means an empty body.
fn bitrot_create_file_size(length: i64, shard_size: usize, checksum_algo: &HashAlgorithm) -> i64 {
if length <= 0 {
return length;
}
let length = length as usize;
(length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
}
pub async fn create_bitrot_writer(
is_inline_buffer: bool,
disk: Option<&DiskStore>,
@@ -814,7 +796,12 @@ pub async fn create_bitrot_writer(
let writer = if is_inline_buffer {
CustomWriter::new_inline_buffer()
} else if let Some(disk) = disk {
let length = bitrot_create_file_size(length, shard_size, &checksum_algo);
let length = if length > 0 {
let length = length as usize;
(length.div_ceil(shard_size) * checksum_algo.size() + length) as i64
} else {
0
};
let file = disk.create_file("", volume, path, length).await?;
#[cfg(feature = "hotpath")]
@@ -833,25 +820,6 @@ mod tests {
use rustfs_rio::ChunkReader;
use std::collections::VecDeque;
#[test]
fn bitrot_create_file_size_grows_known_length_by_checksums() {
// 10 bytes over 4-byte shards = 3 shards, each followed by a 32-byte hash.
assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::HighwayHash256), 10 + 3 * 32);
assert_eq!(bitrot_create_file_size(10, 4, &HashAlgorithm::None), 10);
}
#[test]
fn bitrot_create_file_size_keeps_empty_and_unknown_distinct() {
assert_eq!(bitrot_create_file_size(0, 4, &HashAlgorithm::HighwayHash256), 0);
// SSE/compression streams advertise SIZE_PRESERVE_LAYER (-1); the remote
// put_file_stream receiver relies on a non-positive size to parse the auth
// trailer from the stream tail, so the sentinel must survive untouched.
assert_eq!(
bitrot_create_file_size(rustfs_rio::HashReader::SIZE_PRESERVE_LAYER, 4, &HashAlgorithm::HighwayHash256),
rustfs_rio::HashReader::SIZE_PRESERVE_LAYER
);
}
struct TestChunkReader {
chunks: VecDeque<Bytes>,
}
@@ -2318,6 +2318,57 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
fi.set_data_moved();
}
// Receiver-side LWW (rustfs/backlog#1953): the multipart replication
// transport carries the category values at CreateMultipartUpload (in
// the staged upload metadata) and the source category timestamps on
// the complete request. Read the destination version under the held
// object write lock and keep any category this site modified more
// recently. Read failures (version absent on first replication, quorum
// errors) keep today's overwrite semantics: failing the complete would
// loop through MRF, re-delivering the stale value forever.
if crate::set_disk::ops::object::replication_lww_applicable(opts)
&& let Some(version_id) = fi.version_id
{
match self
.get_object_info(
bucket,
object,
&ObjectOptions {
version_id: Some(version_id.to_string()),
no_lock: true,
metadata_cache_safe: false,
versioned: opts.versioned,
version_suspended: opts.version_suspended,
..Default::default()
},
)
.await
{
Ok(existing) => {
let stored = crate::set_disk::ops::object::stored_replication_category_metadata(&existing);
crate::set_disk::ops::object::merge_replication_metadata_lww(&mut fi.metadata, &stored, opts);
}
// Version absent: first replication of this version, nothing
// local to compare — the normal path, not a degraded one.
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
Err(err) => {
// Degraded path: without the stored state the inbound
// metadata is applied unchanged — exactly the overwrite
// LWW exists to prevent — so this must be operator-visible.
warn!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_SET_DISK,
bucket,
object,
version_id = %version_id,
error = %err,
state = "replication_lww_read_unavailable",
"SetDisk multipart replication LWW read skipped; inbound metadata applied without comparison"
);
}
}
}
for meta in parts_metadatas.iter_mut() {
if meta.has_valid_erasure_geometry() {
meta.size = fi.size;
@@ -7055,6 +7106,97 @@ mod tests {
.await
}
/// Receiver-side LWW on the multipart replication transport
/// (rustfs/backlog#1953): a metadata-only replication of a multipart
/// source object rides CreateMultipartUpload (category values in the
/// upload metadata) + CompleteMultipartUpload (category timestamps in
/// the complete options). A stale inbound tagging timestamp must not
/// overwrite a newer locally-tagged destination version.
#[tokio::test]
#[serial]
async fn complete_multipart_upload_stale_replication_tags_keep_local() {
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::{SUFFIX_TAGGING_TIMESTAMP, get_str};
use time::format_description::well_known::Rfc3339;
const T_OLD: &str = "2026-01-01T00:00:00Z";
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "multipart-replication-lww-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
// Local destination version with newer tags.
let version_id = Uuid::new_v4();
let mut local_metadata = HashMap::new();
local_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
rustfs_utils::http::insert_str(&mut local_metadata, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
let mut local_reader = PutObjReader::from_vec(b"local body".to_vec());
set_disks
.put_object(
bucket,
object,
&mut local_reader,
&ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
user_defined: local_metadata,
// Explicit-version PUTs require the bucket Object Lock snapshot.
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
},
)
.await
.expect("local versioned put should commit");
// Inbound replication upload carrying older tags for the same version.
let mut inbound_metadata = HashMap::new();
inbound_metadata.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
rustfs_utils::http::insert_str(&mut inbound_metadata, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
let create_opts = ObjectOptions {
versioned: true,
user_defined: inbound_metadata,
..Default::default()
};
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &payload(0x5a), &create_opts).await;
rewrite_staged_upload_version_id(&set_disks, bucket, object, &upload_id, Some(version_id)).await;
let complete_opts = ObjectOptions {
versioned: true,
replication_request: true,
replication_tagging_timestamp: Some(OffsetDateTime::parse(T_OLD, &Rfc3339).expect("test timestamp should parse")),
..Default::default()
};
set_disks
.clone()
.complete_multipart_upload(bucket, object, &upload_id, parts, &complete_opts)
.await
.expect("replication multipart completion should succeed even when a category keeps local values");
let info = set_disks
.get_object_info(
bucket,
object,
&ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
..Default::default()
},
)
.await
.expect("completed version should be readable");
assert_eq!(
info.user_tags.as_str(),
"site=local",
"older inbound multipart tags must not overwrite newer local tags"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
}
#[tokio::test]
#[serial]
async fn complete_multipart_upload_assigns_completion_version_id() {
+471
View File
@@ -1881,6 +1881,110 @@ fn delete_file_info_with_replication_transport_metadata(fi: &FileInfo) -> FileIn
transported
}
/// True when an authorized replication write carries at least one per-category
/// source timestamp, i.e. receiver-side LWW has something to judge.
pub(in crate::set_disk) fn replication_lww_applicable(opts: &ObjectOptions) -> bool {
opts.replication_request
&& (opts.replication_tagging_timestamp.is_some()
|| opts.replication_retention_timestamp.is_some()
|| opts.replication_legalhold_timestamp.is_some())
}
/// The stored per-category state of a destination version, as compared by
/// [`merge_replication_metadata_lww`]. `ObjectInfo::from_file_info`
/// externalizes tags into `user_tags` (stripping the metadata key), so the
/// tag value is folded back into map form here.
pub(in crate::set_disk) fn stored_replication_category_metadata(existing: &ObjectInfo) -> HashMap<String, String> {
let mut stored = (*existing.user_defined).clone();
if !existing.user_tags.is_empty() {
stored.insert(rustfs_utils::http::headers::AMZ_OBJECT_TAGGING.to_string(), (*existing.user_tags).clone());
}
stored
}
/// Receiver-side last-writer-wins for authorized replication writes
/// (rustfs/backlog#1953, audit A4/P1-6). Metadata-only replication reuses the
/// whole-object transports, so in active-active topologies an inbound write
/// carries the source's tags / retention / legal hold verbatim and would
/// otherwise overwrite a category the destination modified more recently —
/// both sites end up permanently diverged while reporting COMPLETED.
///
/// Judged per category, only when the inbound request carries that category's
/// source timestamp (`ObjectOptions::replication_*_timestamp`):
/// - stored timestamp newer than inbound: the local category values and
/// timestamp are kept; the rest of the write proceeds per the inbound
/// metadata and the object-level result stays successful (failing the write
/// instead would loop through MRF, re-delivering the stale value forever);
/// - otherwise the inbound category wins and its internal timestamp key is
/// pinned to the source-authored time — the PUT path re-stamps the
/// object-lock timestamps with the receiver's clock
/// (`parse_object_lock_retention` / `parse_object_lock_legal_hold` insert
/// `now()` via `eval_metadata`), which would make the replica's clock the
/// LWW authority and wedge later convergence;
/// - no stored timestamp (pre-P1-6 data) or no inbound timestamp: the current
/// overwrite behavior is preserved.
///
/// Returns whether `inbound` was modified. Callers must hold the object write
/// lock so the stored values compared here are the ones being replaced.
pub(in crate::set_disk) fn merge_replication_metadata_lww(
inbound: &mut HashMap<String, String>,
existing: &HashMap<String, String>,
opts: &ObjectOptions,
) -> bool {
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
};
use rustfs_utils::http::metadata_compat::{
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
remove_str,
};
use time::format_description::well_known::Rfc3339;
let categories: [(Option<OffsetDateTime>, &str, &[&str]); 3] = [
(opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP, &[AMZ_OBJECT_TAGGING]),
(
opts.replication_retention_timestamp,
SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
&[AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER],
),
(
opts.replication_legalhold_timestamp,
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
&[AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER],
),
];
let mut changed = false;
for (inbound_timestamp, timestamp_suffix, value_keys) in categories {
let Some(inbound_timestamp) = inbound_timestamp else { continue };
let is_category_value_key = |key: &str| value_keys.iter().any(|value_key| key.eq_ignore_ascii_case(value_key));
let stored_timestamp = get_str(existing, timestamp_suffix).and_then(|value| OffsetDateTime::parse(&value, &Rfc3339).ok());
if stored_timestamp.is_some_and(|stored| stored > inbound_timestamp) {
inbound.retain(|key, _| !is_category_value_key(key));
remove_str(inbound, timestamp_suffix);
for (key, value) in existing {
if is_category_value_key(key) {
inbound.insert(key.clone(), value.clone());
}
}
// Restore the winning timestamp via insert_str, not a verbatim key
// copy: a MinIO-written version may carry only the
// x-minio-internal- key, and the dual-key invariant requires every
// write to produce both keys.
if let Some(stored_value) = get_str(existing, timestamp_suffix) {
rustfs_utils::http::insert_str(inbound, timestamp_suffix, stored_value);
}
changed = true;
} else if let Ok(source_authored) = inbound_timestamp.format(&Rfc3339)
&& get_str(inbound, timestamp_suffix).as_deref() != Some(source_authored.as_str())
{
rustfs_utils::http::insert_str(inbound, timestamp_suffix, source_authored);
changed = true;
}
}
changed
}
impl SetDisks {
pub(in crate::set_disk) async fn persist_old_data_cleanup_receipts(
&self,
@@ -2562,6 +2666,22 @@ impl SetDisks {
if check_object_lock_for_deletion_with_state(object_lock_config.state(), &existing, false)?.is_some() {
return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object.to_string()));
}
// Receiver-side LWW (rustfs/backlog#1953): reuse this
// commit-lock read of the destination version so a
// category (tags / retention / legal hold) modified
// more recently on this site is kept instead of being
// overwritten by the inbound replication metadata.
if replication_lww_applicable(opts) {
let stored = stored_replication_category_metadata(&existing);
let mut merged = parts_metadatas[response_metadata_slot].metadata.clone();
if merge_replication_metadata_lww(&mut merged, &stored, opts) {
for (pfi, disk) in parts_metadatas.iter_mut().zip(shuffle_disks.iter()) {
if disk.is_some() {
pfi.metadata = merged.clone();
}
}
}
}
}
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
Err(err) => return Err(err),
@@ -8066,6 +8186,357 @@ mod replication_quota_safety_tests {
}
}
#[cfg(test)]
mod replication_lww_tests {
//! Receiver-side LWW for authorized replication writes (rustfs/backlog#1953,
//! audit A4/P1-6): an inbound replication PUT whose per-category timestamp
//! (tags / retention / legal hold) is older than the destination version's
//! stored timestamp must keep the local category values instead of
//! overwriting them; categories are judged independently and the write
//! itself still succeeds.
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
use super::*;
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
};
use rustfs_utils::http::{
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_TAGGING_TIMESTAMP, get_str,
insert_str,
};
use time::format_description::well_known::Rfc3339;
const T_OLD: &str = "2026-01-01T00:00:00Z";
const T_LOCAL: &str = "2026-02-01T00:00:00Z";
const T_NEW: &str = "2026-03-01T00:00:00Z";
fn parse_ts(value: &str) -> OffsetDateTime {
OffsetDateTime::parse(value, &Rfc3339).expect("test timestamp should parse")
}
async fn make_bucket(disks: &[DiskStore], bucket: &str) {
for disk in disks {
disk.make_volume(bucket).await.expect("bucket volume should be created");
}
}
async fn put_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str, opts: &ObjectOptions) {
let mut reader = PutObjReader::from_vec(b"lww-body".to_vec());
set_disks
.put_object(bucket, object, &mut reader, opts)
.await
.expect("versioned put should commit");
assert_eq!(opts.version_id.as_deref(), Some(version_id));
}
fn versioned_opts(version_id: &str, user_defined: HashMap<String, String>) -> ObjectOptions {
ObjectOptions {
versioned: true,
version_id: Some(version_id.to_string()),
user_defined,
// Explicit-version PUTs require the bucket Object Lock snapshot.
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
}
}
/// Local state: version `version_id` with tags "site=local" stamped `T_LOCAL`.
async fn seed_local_tagged_version(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) {
let mut user_defined = HashMap::new();
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
put_version(set_disks, bucket, object, version_id, &versioned_opts(version_id, user_defined)).await;
}
fn inbound_tagging_opts(version_id: &str, tags: &str, timestamp: &str) -> ObjectOptions {
let mut user_defined = HashMap::new();
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), tags.to_string());
insert_str(&mut user_defined, SUFFIX_TAGGING_TIMESTAMP, timestamp.to_string());
ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(parse_ts(timestamp)),
..versioned_opts(version_id, user_defined)
}
}
async fn version_info(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, version_id: &str) -> ObjectInfo {
set_disks
.get_object_info(bucket, object, &versioned_opts(version_id, HashMap::new()))
.await
.expect("version should be readable")
}
#[tokio::test]
async fn inbound_stale_tagging_keeps_newer_local_tags() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-stale";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_tags.as_str(),
"site=local",
"older inbound tags must not overwrite newer local tags"
);
assert_eq!(
get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(),
Some(T_LOCAL),
"the winning local tagging timestamp must be preserved"
);
}
#[tokio::test]
async fn inbound_newer_tagging_overwrites_local_tags() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-newer";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
seed_local_tagged_version(&set_disks, bucket, object, &version_id).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_NEW),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_tags.as_str(),
"site=remote",
"newer inbound tags must overwrite older local tags"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_NEW));
}
#[tokio::test]
async fn inbound_wins_when_local_has_no_tagging_timestamp() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-no-local-ts";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Pre-P1-6 data: local tags without a stored tagging timestamp.
let mut user_defined = HashMap::new();
user_defined.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, user_defined)).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_tags.as_str(),
"site=remote",
"without a local timestamp the inbound category must win (pre-LWW data compatibility)"
);
}
#[tokio::test]
async fn categories_are_judged_independently() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-category-independent";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local: newer tags (T_LOCAL), older *cleared* retention (T_OLD) —
// timestamp key only, the shape a replicated retention clear stores.
// (An active local retention would already block the overwrite at the
// WORM gate; the LWW-reachable retention states are cleared/expired.)
let mut local = HashMap::new();
local.insert(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string());
insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
// Inbound: older tags (T_OLD), newer retention (T_NEW).
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "COMPLIANCE".to_string());
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
let opts = ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(parse_ts(T_OLD)),
replication_retention_timestamp: Some(parse_ts(T_NEW)),
..versioned_opts(&version_id, inbound)
};
put_version(&set_disks, bucket, object, &version_id, &opts).await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(info.user_tags.as_str(), "site=local", "the stale tagging category must keep local values");
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str),
Some("COMPLIANCE"),
"the newer retention category must be applied in the same write"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(), Some(T_NEW));
}
#[tokio::test]
async fn inbound_stale_legal_hold_keeps_local_value() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-legalhold-stale";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local: legal hold released (OFF) at T_LOCAL. (A local hold that is
// still ON already blocks the overwrite at the WORM gate; the
// LWW-reachable divergence is a stale inbound ON resurrecting a hold
// that was released more recently on this site.)
let mut local = HashMap::new();
local.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "OFF".to_string());
insert_str(&mut local, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_LOCAL.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER.to_string(), "ON".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, T_OLD.to_string());
let opts = ObjectOptions {
replication_request: true,
replication_legalhold_timestamp: Some(parse_ts(T_OLD)),
..versioned_opts(&version_id, inbound)
};
put_version(&set_disks, bucket, object, &version_id, &opts).await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
info.user_defined.get(AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER).map(String::as_str),
Some("OFF"),
"a stale inbound legal hold must not resurrect a hold released more recently"
);
assert_eq!(
get_str(&info.user_defined, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP).as_deref(),
Some(T_LOCAL)
);
}
/// Dual-key invariant under LWW: a MinIO-written destination version may
/// carry only the x-minio-internal timestamp key; when the local category
/// wins, the restored map must still hold BOTH compatibility keys.
#[test]
fn local_win_restores_both_internal_timestamp_keys_for_minio_only_metadata() {
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_TAGGING.to_string(), "site=remote".to_string());
insert_str(&mut inbound, SUFFIX_TAGGING_TIMESTAMP, T_OLD.to_string());
let existing = HashMap::from([
(AMZ_OBJECT_TAGGING.to_string(), "site=local".to_string()),
("X-Minio-Internal-Tagging-Timestamp".to_string(), T_LOCAL.to_string()),
]);
let opts = ObjectOptions {
replication_request: true,
replication_tagging_timestamp: Some(parse_ts(T_OLD)),
..Default::default()
};
assert!(merge_replication_metadata_lww(&mut inbound, &existing, &opts));
assert_eq!(inbound.get(AMZ_OBJECT_TAGGING).map(String::as_str), Some("site=local"));
assert_eq!(
inbound.get("x-rustfs-internal-tagging-timestamp").map(String::as_str),
Some(T_LOCAL),
"the RustFS twin key must be materialized even when the source version only had the MinIO key"
);
assert_eq!(inbound.get("x-minio-internal-tagging-timestamp").map(String::as_str), Some(T_LOCAL));
}
/// When the inbound category wins, the stored timestamp must be the
/// source-authored one: the PUT path's eval_metadata stamps the
/// object-lock timestamps with the receiver's clock
/// (`parse_object_lock_retention`), which would otherwise make this
/// replica's clock the LWW authority and wedge later convergence.
#[tokio::test]
async fn inbound_win_pins_stored_timestamp_to_source_authored_value() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-retention-ts-pinned";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local cleared retention at T_OLD.
let mut local = HashMap::new();
insert_str(&mut local, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_OLD.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
// Inbound newer retention: the source authored T_LOCAL, but the PUT
// path's eval_metadata stomped the metadata key with receiver-now
// (simulated by T_NEW here).
let mut inbound = HashMap::new();
inbound.insert(AMZ_OBJECT_LOCK_MODE_LOWER.to_string(), "GOVERNANCE".to_string());
inbound.insert(AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER.to_string(), "2028-01-01T00:00:00Z".to_string());
insert_str(&mut inbound, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, T_NEW.to_string());
let opts = ObjectOptions {
replication_request: true,
replication_retention_timestamp: Some(parse_ts(T_LOCAL)),
..versioned_opts(&version_id, inbound)
};
put_version(&set_disks, bucket, object, &version_id, &opts).await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert_eq!(
get_str(&info.user_defined, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP).as_deref(),
Some(T_LOCAL),
"the stored category timestamp must be the source-authored time, not the receiver's clock"
);
assert_eq!(info.user_defined.get(AMZ_OBJECT_LOCK_MODE_LOWER).map(String::as_str), Some("GOVERNANCE"));
}
#[tokio::test]
async fn newer_local_tag_deletion_survives_stale_inbound_tags() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "lww-tagging-deleted";
let object = "object";
let version_id = Uuid::new_v4().to_string();
make_bucket(&disk_stores, bucket).await;
// Local DeleteObjectTagging state: no tags, but a newer tagging timestamp.
let mut local = HashMap::new();
insert_str(&mut local, SUFFIX_TAGGING_TIMESTAMP, T_LOCAL.to_string());
put_version(&set_disks, bucket, object, &version_id, &versioned_opts(&version_id, local)).await;
put_version(
&set_disks,
bucket,
object,
&version_id,
&inbound_tagging_opts(&version_id, "site=remote", T_OLD),
)
.await;
let info = version_info(&set_disks, bucket, object, &version_id).await;
assert!(
info.user_tags.is_empty(),
"a newer local tag deletion must not be resurrected by older inbound tags"
);
assert_eq!(get_str(&info.user_defined, SUFFIX_TAGGING_TIMESTAMP).as_deref(), Some(T_LOCAL));
}
}
#[cfg(test)]
mod inline_put_commit_path_tests {
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
+54 -513
View File
@@ -13,10 +13,10 @@
// limitations under the License.
use crate::heal::{
progress::{HealProgress, add_bytes, increment_counter},
progress::HealProgress,
resume::{
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ReplacementTargetIdentity, ResumeManager,
ResumeUtils, compose_key, replacement_target_identities_match,
CheckpointManager, ReplacementTargetIdentity, ResumeManager, ResumeUtils, compose_key,
replacement_target_identities_match,
},
storage::{HealStorageAPI, next_heal_listing_token},
task::{demote_to_debug_when, is_missing_object_dir_heal_result, take_failure_log_sample},
@@ -410,9 +410,6 @@ impl ErasureSetHealer {
&& state.successful_objects == 0
&& state.failed_objects == 0
&& state.skipped_objects == 0
&& state.skipped_new_versions == 0
&& state.skipped_ilm_expired == 0
&& state.processed_bytes == 0
{
// schedule_retry persists the authoritative resume reset before
// resetting the checkpoint. Reapply the checkpoint reset after
@@ -477,23 +474,6 @@ impl ErasureSetHealer {
// 2. initialize progress
self.initialize_progress(buckets, &state).await;
let (baseline_known, baseline_count, baseline_size, baseline_generation) = {
let baseline = self.progress.read().await;
(
baseline.baseline_known,
baseline.objects_total_count,
baseline.objects_total_size,
baseline.baseline_generation,
)
};
if baseline_known {
resume_manager
.set_progress_baseline(baseline_count, baseline_size, baseline_generation)
.await?;
checkpoint_manager
.set_progress_baseline(baseline_count, baseline_size, baseline_generation)
.await?;
}
// 3. continue from checkpoint
let current_bucket_index = checkpoint.current_bucket_index;
@@ -503,66 +483,12 @@ impl ErasureSetHealer {
let mut successful_objects = state.successful_objects;
let mut failed_objects = state.failed_objects;
let mut skipped_objects = state.skipped_objects;
let checkpoint_has_progress = checkpoint.baseline_known
|| checkpoint.successful_objects > 0
|| checkpoint.failed_object_count > 0
|| checkpoint.skipped_object_count > 0
|| checkpoint.skipped_new_versions > 0
|| checkpoint.skipped_ilm_expired > 0
|| checkpoint.processed_bytes > 0
|| checkpoint.total_objects > 0
|| checkpoint.total_bytes > 0
|| checkpoint.baseline_generation.is_some()
|| checkpoint.counter_unknown;
let checkpoint_generation_mismatch = checkpoint.baseline_known && checkpoint.baseline_generation != baseline_generation;
let mut restored_counter_unknown = state.counter_unknown || checkpoint.counter_unknown;
if checkpoint_has_progress {
successful_objects = checkpoint.successful_objects;
failed_objects = checkpoint.failed_object_count;
skipped_objects = checkpoint.skipped_object_count;
let restored_processed_objects = successful_objects
.checked_add(failed_objects)
.and_then(|value| value.checked_add(skipped_objects))
.and_then(|value| value.checked_add(checkpoint.skipped_new_versions))
.and_then(|value| value.checked_add(checkpoint.skipped_ilm_expired));
let checkpoint_counter_overflow = restored_processed_objects.is_none();
restored_counter_unknown |= checkpoint_counter_overflow;
processed_objects = restored_processed_objects.unwrap_or(u64::MAX);
let mut progress = self.progress.write().await;
progress.objects_scanned = processed_objects;
progress.objects_healed = successful_objects;
progress.objects_failed = failed_objects;
progress.skipped_objects = skipped_objects;
progress.skipped_new_versions = checkpoint.skipped_new_versions;
progress.skipped_ilm_expired = checkpoint.skipped_ilm_expired;
if checkpoint.baseline_known && !checkpoint_generation_mismatch {
progress.objects_total_count = checkpoint.total_objects;
progress.objects_total_size = checkpoint.total_bytes;
progress.baseline_generation = checkpoint.baseline_generation;
progress.baseline_known = true;
}
progress.bytes_processed = checkpoint.processed_bytes;
progress.counter_unknown = state.counter_unknown || checkpoint.counter_unknown;
progress.refresh_progress_percentage();
if checkpoint_generation_mismatch || checkpoint_counter_overflow || progress.counter_unknown {
progress.mark_unknown();
}
}
if checkpoint_generation_mismatch {
restored_counter_unknown = true;
}
if restored_counter_unknown {
checkpoint_manager.mark_counter_unknown().await?;
resume_manager.mark_counter_unknown().await?;
}
let mut failed_buckets = 0u64;
// 4. process remaining buckets
for (bucket_idx, bucket) in buckets.iter().enumerate().skip(current_bucket_index) {
// check if completed
if state.completed_buckets.contains(bucket) {
checkpoint_manager.complete_bucket(bucket_idx.saturating_add(1)).await?;
current_object_index = 0;
continue;
}
@@ -590,42 +516,13 @@ impl ErasureSetHealer {
return bucket_result;
}
// update progress
let progress_snapshot = self.progress.read().await;
let bytes_processed = progress_snapshot.bytes_processed;
let skipped_new_versions = progress_snapshot.skipped_new_versions;
let skipped_ilm_expired = progress_snapshot.skipped_ilm_expired;
let counter_unknown = progress_snapshot.counter_unknown;
drop(progress_snapshot);
// The checkpoint is the recovery authority for object progress.
// Publish its counters and fence before the resume summary so a
// crash between the two stores cannot make recovery select newer
// summary bytes with an older checkpoint ledger.
if counter_unknown {
checkpoint_manager.mark_counter_unknown().await?;
}
checkpoint_manager
.update_progress(successful_objects, failed_objects, skipped_objects, bytes_processed)
.await?;
checkpoint_manager
.set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired)
.await?;
// update checkpoint position
checkpoint_manager.update_position(bucket_idx, current_object_index).await?;
// update progress
resume_manager
.update_progress_with_bytes(
processed_objects,
successful_objects,
failed_objects,
skipped_objects,
bytes_processed,
)
.update_progress(processed_objects, successful_objects, failed_objects, skipped_objects)
.await?;
resume_manager
.set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired)
.await?;
if counter_unknown {
resume_manager.mark_counter_unknown().await?;
}
// check cancel status
if self.cancel_token.is_cancelled() {
@@ -645,7 +542,6 @@ impl ErasureSetHealer {
match bucket_result {
Ok(_) => {
resume_manager.complete_bucket(bucket).await?;
checkpoint_manager.complete_bucket(bucket_idx.saturating_add(1)).await?;
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
@@ -671,9 +567,7 @@ impl ErasureSetHealer {
error = %e,
"Erasure set bucket heal failed"
);
// A single durable cursor and ledger cannot safely preserve
// this bucket while processing a later one.
break;
// continue to next bucket, do not interrupt the whole process
}
}
@@ -881,49 +775,20 @@ impl ErasureSetHealer {
// Per-version dedup identity — the single canonical key.
let key = compose_key(&item.name, item.version_id.as_deref());
if checkpoint.processed_objects.contains(&key)
|| checkpoint.failed_objects.contains(&key)
|| checkpoint.skipped_objects.contains(&key)
{
if checkpoint.processed_objects.contains(&key) || checkpoint.skipped_objects.contains(&key) {
continue;
}
if should_skip_new_version(item.mod_time_unix_nanos, started_at_secs) {
let counter_ok = increment_counter(processed_objects);
checkpoint_manager.add_processed_object(key).await?;
*processed_objects = processed_objects.saturating_add(1);
completed_in_page = completed_in_page.saturating_add(1);
counter!("rustfs_heal_skipped_new_versions_total").increment(1);
let (outcome_record, counter_unknown) = {
{
let mut progress = self.progress.write().await;
progress.record_skipped_new_version();
progress.set_current_object(Some(format!("skipped_new: {bucket}/{}", item.name)));
progress.update_object_progress(
*processed_objects,
*successful_objects,
*failed_objects,
*skipped_objects,
bytes_processed,
);
if !counter_ok {
progress.mark_unknown();
}
(
CheckpointObjectOutcomeRecord {
object: key,
outcome: CheckpointObjectOutcome::Processed,
successful: progress.objects_healed,
failed: progress.objects_failed,
skipped: progress.skipped_objects,
bytes: progress.bytes_processed,
skipped_new_versions: progress.skipped_new_versions,
skipped_ilm_expired: progress.skipped_ilm_expired,
counter_unknown: progress.counter_unknown,
},
progress.counter_unknown,
)
};
checkpoint_manager.record_object_outcome(outcome_record).await?;
if counter_unknown {
resume_manager.mark_counter_unknown().await?;
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
}
debug!(
target: "rustfs::heal::erasure_healer",
@@ -955,41 +820,15 @@ impl ErasureSetHealer {
)
.await?
{
let counter_ok = increment_counter(processed_objects);
checkpoint_manager.add_processed_object(key).await?;
*processed_objects = processed_objects.saturating_add(1);
completed_in_page = completed_in_page.saturating_add(1);
counter!("rustfs_heal_skipped_ilm_expired_total").increment(1);
let (outcome_record, counter_unknown) = {
{
let mut progress = self.progress.write().await;
progress.record_skipped_ilm_expired();
progress.set_current_object(Some(format!("skipped_ilm: {bucket}/{}", item.name)));
progress.update_object_progress(
*processed_objects,
*successful_objects,
*failed_objects,
*skipped_objects,
bytes_processed,
);
if !counter_ok {
progress.mark_unknown();
}
(
CheckpointObjectOutcomeRecord {
object: key,
outcome: CheckpointObjectOutcome::Processed,
successful: progress.objects_healed,
failed: progress.objects_failed,
skipped: progress.skipped_objects,
bytes: progress.bytes_processed,
skipped_new_versions: progress.skipped_new_versions,
skipped_ilm_expired: progress.skipped_ilm_expired,
counter_unknown: progress.counter_unknown,
},
progress.counter_unknown,
)
};
checkpoint_manager.record_object_outcome(outcome_record).await?;
if counter_unknown {
resume_manager.mark_counter_unknown().await?;
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
}
debug!(
target: "rustfs::heal::erasure_healer",
@@ -1115,11 +954,11 @@ impl ErasureSetHealer {
while let Some((key, object, version_id, result)) = page_tasks.next().await {
let (object_size, result) = result;
let mut telemetry_unknown = false;
let checkpoint_outcome = match result {
match result {
Ok(true) => {
telemetry_unknown |= !increment_counter(successful_objects);
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
*successful_objects += 1;
bytes_processed = bytes_processed.saturating_add(object_size);
checkpoint_manager.add_processed_object(key).await?;
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
@@ -1132,11 +971,11 @@ impl ErasureSetHealer {
state = "healed",
"Erasure set object healed"
);
CheckpointObjectOutcome::Processed
}
Ok(false) => {
telemetry_unknown |= !increment_counter(successful_objects);
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
checkpoint_manager.add_processed_object(key).await?;
*successful_objects += 1;
bytes_processed = bytes_processed.saturating_add(object_size);
debug!(
target: "rustfs::heal::erasure_healer",
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
@@ -1149,12 +988,12 @@ impl ErasureSetHealer {
state = "missing_treated_as_ok",
"Erasure set missing object treated as ok"
);
CheckpointObjectOutcome::Processed
}
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
Err(Error::TransientSkip { message }) => {
telemetry_unknown |= !increment_counter(skipped_objects);
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
*skipped_objects += 1;
bytes_processed = bytes_processed.saturating_add(object_size);
checkpoint_manager.add_skipped_object(key).await?;
demote_to_debug_when!(!take_failure_log_sample(&mut transient_skip_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -1167,11 +1006,11 @@ impl ErasureSetHealer {
error = %message,
"Erasure set object heal skipped due to transient error"
});
CheckpointObjectOutcome::Skipped
}
Err(err) => {
telemetry_unknown |= !increment_counter(failed_objects);
telemetry_unknown |= !add_bytes(&mut bytes_processed, object_size);
*failed_objects += 1;
bytes_processed = bytes_processed.saturating_add(object_size);
checkpoint_manager.add_failed_object(key).await?;
demote_to_debug_when!(!take_failure_log_sample(&mut failure_samples_logged), warn, target: "rustfs::heal::erasure_healer", {
event = EVENT_HEAL_ERASURE_OBJECT_STATE,
component = LOG_COMPONENT_HEAL,
@@ -1184,43 +1023,15 @@ impl ErasureSetHealer {
error = %err,
"Erasure set object heal failed"
});
CheckpointObjectOutcome::Failed
}
};
}
telemetry_unknown |= !increment_counter(processed_objects);
*processed_objects += 1;
completed_in_page += 1;
let (outcome_record, counter_unknown) = {
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("{bucket}/{object}")));
progress.update_object_progress(
*processed_objects,
*successful_objects,
*failed_objects,
*skipped_objects,
bytes_processed,
);
if telemetry_unknown {
progress.mark_unknown();
}
(
CheckpointObjectOutcomeRecord {
object: key,
outcome: checkpoint_outcome,
successful: progress.objects_healed,
failed: progress.objects_failed,
skipped: progress.skipped_objects,
bytes: progress.bytes_processed,
skipped_new_versions: progress.skipped_new_versions,
skipped_ilm_expired: progress.skipped_ilm_expired,
counter_unknown: progress.counter_unknown,
},
progress.counter_unknown,
)
};
checkpoint_manager.record_object_outcome(outcome_record).await?;
if counter_unknown {
resume_manager.mark_counter_unknown().await?;
progress.update_progress(*processed_objects, *successful_objects, *failed_objects, bytes_processed);
}
if completed_in_page.is_multiple_of(100) {
@@ -1230,22 +1041,16 @@ impl ErasureSetHealer {
*current_object_index = global_obj_idx;
// Persist the checkpoint ledger and page position before exposing
// the next resume cursor. A crash before cursor publication keeps
// the page identities available for exact-once replay.
checkpoint_manager.advance_page(bucket_index, *current_object_index).await?;
// Persist the authoritative cursor FIRST (points at the next page
// boundary), then prune the per-version dedup sets. Both are
// idempotent under crash: heal_object re-heals safely.
let next_cursor = if is_truncated { next_token.clone() } else { None };
resume_manager.set_resume_cursor(next_cursor.clone()).await?;
checkpoint_manager.complete_page(bucket_index, *current_object_index).await?;
// Check if there are more pages
if !is_truncated {
break;
}
continuation_token = next_heal_listing_token(bucket, "", next_token, is_truncated)?;
if continuation_token.is_none() {
// A truncated page without a continuation token is terminal.
// Retain its ledger until bucket completion is durable.
break;
}
resume_manager.set_resume_cursor(continuation_token.clone()).await?;
checkpoint_manager.prune_completed_page().await?;
// Anti-loop guard: an empty page reported as truncated cannot advance
// the cursor (there is no last identity to move past), so treat it as a
@@ -1264,6 +1069,12 @@ impl ErasureSetHealer {
)));
}
previous_page_last = page_last;
continuation_token = next_heal_listing_token(bucket, "", next_token, is_truncated)?;
if continuation_token.is_none() {
// Truncated but no continuation token: treat as end of listing.
break;
}
}
Ok(())
@@ -1272,66 +1083,10 @@ impl ErasureSetHealer {
/// initialize progress tracking
async fn initialize_progress(&self, _buckets: &[String], state: &crate::heal::resume::ResumeState) {
let mut progress = self.progress.write().await;
let existing_baseline = (
progress.objects_total_count,
progress.objects_total_size,
progress.baseline_generation,
progress.progress_state,
progress.baseline_known,
);
let baseline_generation_mismatch =
state.baseline_known && existing_baseline.4 && state.baseline_generation != existing_baseline.2;
let use_persisted_baseline = state.baseline_known && !baseline_generation_mismatch;
progress.objects_scanned = state.processed_objects;
progress.objects_scanned = state.total_objects;
progress.objects_healed = state.successful_objects;
progress.objects_failed = state.failed_objects;
progress.skipped_objects = state.skipped_objects;
progress.skipped_new_versions = state.skipped_new_versions;
progress.skipped_ilm_expired = state.skipped_ilm_expired;
progress.bytes_processed = state.processed_bytes;
progress.counter_unknown = state.counter_unknown;
if use_persisted_baseline
|| existing_baseline.0 > 0
|| existing_baseline.1 > 0
|| existing_baseline.2.is_some()
|| existing_baseline.4
{
progress.objects_total_count = if use_persisted_baseline {
state.total_objects
} else {
existing_baseline.0
};
progress.objects_total_size = if use_persisted_baseline {
state.total_bytes
} else {
existing_baseline.1
};
progress.baseline_generation = if use_persisted_baseline {
state.baseline_generation
} else {
existing_baseline.2
};
progress.baseline_known = use_persisted_baseline
|| existing_baseline.0 > 0
|| existing_baseline.1 > 0
|| existing_baseline.2.is_some()
|| existing_baseline.4;
}
progress.progress_state = if use_persisted_baseline
|| existing_baseline.0 > 0
|| existing_baseline.1 > 0
|| existing_baseline.2.is_some()
|| existing_baseline.4
{
crate::heal::progress::HealProgressState::Running
} else {
crate::heal::progress::HealProgressState::Indeterminate
};
if baseline_generation_mismatch || state.counter_unknown {
progress.mark_unknown();
}
progress.ledger_complete = false;
progress.refresh_progress_percentage();
progress.bytes_processed = 0; // Resume state tracks object counts, not byte counters.
progress.start_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.start_time));
progress.last_update_time = UNIX_EPOCH.checked_add(Duration::from_secs(state.last_update));
progress.set_current_object(state.current_object.clone());
@@ -1509,8 +1264,8 @@ mod resume_loop_tests {
};
use crate::heal::progress::HealProgress;
use crate::heal::resume::{
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, RESUME_CHECKPOINT_FILE,
ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils, compose_key,
CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils,
compose_key,
};
use crate::heal::storage::{HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI};
use crate::heal::storage_api::status::BucketInfo;
@@ -1650,7 +1405,6 @@ mod resume_loop_tests {
list_include_lifecycle_object_info: Mutex<Vec<bool>>,
replacement_target_identity_sequences: Mutex<VecDeque<Vec<ReplacementTargetIdentity>>>,
fail_listing: AtomicBool,
fail_listing_buckets: Mutex<HashSet<String>>,
}
impl FakeStorage {
@@ -1687,9 +1441,6 @@ mod resume_loop_tests {
fn fail_listing(&self) {
self.fail_listing.store(true, Ordering::SeqCst);
}
fn fail_bucket_listing(&self, bucket: &str) {
self.fail_listing_buckets.lock().unwrap().insert(bucket.to_string());
}
}
#[async_trait::async_trait]
@@ -1780,7 +1531,7 @@ mod resume_loop_tests {
}
async fn list_objects_for_heal_page(
&self,
bucket: &str,
_bucket: &str,
_prefix: &str,
continuation_token: Option<&str>,
include_lifecycle_object_info: bool,
@@ -1789,7 +1540,7 @@ mod resume_loop_tests {
.lock()
.unwrap()
.push(include_lifecycle_object_info);
if self.fail_listing.load(Ordering::SeqCst) || self.fail_listing_buckets.lock().unwrap().contains(bucket) {
if self.fail_listing.load(Ordering::SeqCst) {
return Err(Error::other("injected listing failure"));
}
let key = continuation_token.map(str::to_string);
@@ -2167,49 +1918,6 @@ mod resume_loop_tests {
assert!(state.completed_buckets.is_empty(), "the failed bucket must remain resumable");
}
#[tokio::test]
async fn bucket_failure_stops_before_a_later_bucket_checkpoint() {
let env = make_env().await;
let task_id = ResumeUtils::generate_task_id();
let buckets = vec!["a".to_string(), "b".to_string()];
let resume = ResumeManager::new(
env.healer.disk.clone(),
task_id.clone(),
"erasure_set".to_string(),
"pool_0_set_0".to_string(),
buckets.clone(),
)
.await
.unwrap();
let checkpoint = CheckpointManager::new(env.healer.disk.clone(), task_id.clone())
.await
.unwrap();
env.storage.fail_bucket_listing("a");
for _ in 0..3 {
assert!(resume.schedule_retry().await.unwrap());
}
env.healer
.execute_heal_with_resume(&buckets, "pool_0_set_0", &resume, &checkpoint)
.await
.expect_err("the first bucket failure must keep the pass incomplete");
let persisted = checkpoint.get_checkpoint().await;
assert_eq!(persisted.current_bucket_index, 0);
assert!(resume.get_state().await.completed_buckets.is_empty());
let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &task_id)
.await
.unwrap();
let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &task_id)
.await
.unwrap();
env.healer
.execute_heal_with_resume(&buckets, "pool_0_set_0", &resumed, &checkpoint)
.await
.expect_err("recovery must retry the earlier failed bucket");
assert!(!resumed.get_state().await.completed);
}
#[tokio::test]
async fn completed_resume_state_is_not_selected_for_a_new_heal() {
let env = make_env().await;
@@ -2399,175 +2107,8 @@ mod resume_loop_tests {
let mut names: Vec<String> = env.storage.calls().into_iter().map(|(n, _)| n).collect();
names.sort();
assert_eq!(names, vec!["a", "b", "c", "d"], "every object exactly once, none dropped/doubled");
// Keep the final page cursor until the outer loop durably completes the
// bucket, so a crash can replay only this page against its identities.
assert_eq!(env.resume.resume_cursor().await, Some("t1".to_string()));
}
#[tokio::test]
async fn persisted_failure_waits_for_the_bounded_retry_after_page_replay() {
let env = make_env().await;
env.storage.set_page(
None,
Page {
items: vec![item("object", Some("v1"), false)],
next: None,
truncated: false,
},
);
env.checkpoint
.record_object_outcome(CheckpointObjectOutcomeRecord {
object: compose_key("object", Some("v1")),
outcome: CheckpointObjectOutcome::Failed,
successful: 0,
failed: 1,
skipped: 0,
bytes: 0,
skipped_new_versions: 0,
skipped_ilm_expired: 0,
counter_unknown: false,
})
.await
.unwrap();
env.checkpoint.advance_page(0, 1).await.unwrap();
let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
env.healer
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
.await
.expect_err("the persisted failure must schedule a bounded retry");
assert!(
env.storage.calls().is_empty(),
"the failed identity must not be repeated in the same pass"
);
env.healer
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
.await
.expect("the bounded retry must heal the object");
assert_eq!(env.storage.calls(), vec![("object".to_string(), Some("v1".to_string()))]);
let state = resumed.get_state().await;
assert_eq!(state.successful_objects, 1);
assert_eq!(state.failed_objects, 0);
}
#[tokio::test]
async fn final_page_crash_replays_only_the_retained_page_identities() {
let env = make_env().await;
env.storage.set_page(
None,
Page {
items: vec![item("first", Some("v1"), false)],
next: Some("final-page".to_string()),
truncated: true,
},
);
env.storage.set_page(
Some("final-page"),
Page {
items: vec![item("last", Some("v1"), false)],
next: None,
truncated: false,
},
);
let (processed, successful, failed, skipped, result) = run(&env).await;
result.expect("the bucket pass must finish before the simulated crash");
assert_eq!((processed, successful, failed, skipped), (2, 2, 0, 0));
let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
env.healer
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
.await
.expect("the retained final-page ledger must make recovery exact");
assert_eq!(
env.storage.calls(),
vec![
("first".to_string(), Some("v1".to_string())),
("last".to_string(), Some("v1".to_string()))
]
);
let state = resumed.get_state().await;
assert_eq!(state.successful_objects, 2);
assert_eq!(state.processed_objects, 2);
}
#[tokio::test]
async fn truncated_page_without_token_retains_its_replay_ledger() {
let env = make_env().await;
env.storage.set_page(
None,
Page {
items: vec![item("object", Some("v1"), false)],
next: None,
truncated: true,
},
);
let (processed, successful, failed, skipped, result) = run(&env).await;
result.expect("the tokenless truncated page is a terminal page");
assert_eq!((processed, successful, failed, skipped), (1, 1, 0, 0));
let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
env.healer
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
.await
.expect("terminal-page recovery must not replay a durable identity");
assert_eq!(env.storage.calls(), vec![("object".to_string(), Some("v1".to_string()))]);
let state = resumed.get_state().await;
assert_eq!(state.successful_objects, 1);
assert_eq!(state.processed_objects, 1);
}
#[tokio::test]
async fn completed_bucket_reconciles_its_final_page_checkpoint_after_crash() {
let env = make_env().await;
env.storage.set_page(
None,
Page {
items: vec![item("object", Some("v1"), false)],
next: None,
truncated: false,
},
);
let (_, _, _, _, result) = run(&env).await;
result.expect("the bucket pass must finish before the simulated crash");
env.resume.complete_bucket("b").await.unwrap();
let resumed = ResumeManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
let checkpoint = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
.await
.unwrap();
env.healer
.execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &resumed, &checkpoint)
.await
.expect("recovery must finish the checkpoint transition without replaying the bucket");
assert_eq!(env.storage.calls(), vec![("object".to_string(), Some("v1".to_string()))]);
let checkpoint = checkpoint.get_checkpoint().await;
assert_eq!(checkpoint.current_bucket_index, 1);
assert!(checkpoint.processed_objects.is_empty());
// Final page not truncated => cursor cleared.
assert_eq!(env.resume.resume_cursor().await, None);
}
#[tokio::test]
+26 -3
View File
@@ -2011,11 +2011,34 @@ impl HealManager {
return None;
}
let mut progresses = Vec::with_capacity(active_tasks.len());
let mut snapshot = HealProgress::default();
for task in active_tasks {
progresses.push(task.get_progress().await);
let progress = task.get_progress().await;
snapshot.objects_scanned = snapshot.objects_scanned.saturating_add(progress.objects_scanned);
snapshot.objects_healed = snapshot.objects_healed.saturating_add(progress.objects_healed);
snapshot.objects_failed = snapshot.objects_failed.saturating_add(progress.objects_failed);
snapshot.skipped_new_versions = snapshot.skipped_new_versions.saturating_add(progress.skipped_new_versions);
snapshot.skipped_ilm_expired = snapshot.skipped_ilm_expired.saturating_add(progress.skipped_ilm_expired);
snapshot.objects_total_count = snapshot.objects_total_count.saturating_add(progress.objects_total_count);
snapshot.objects_total_size = snapshot.objects_total_size.saturating_add(progress.objects_total_size);
snapshot.bytes_processed = snapshot.bytes_processed.saturating_add(progress.bytes_processed);
snapshot.start_time = match (snapshot.start_time, progress.start_time) {
(Some(current), Some(next)) => Some(current.min(next)),
(None, next) => next,
(current, None) => current,
};
snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) {
(Some(current), Some(next)) => Some(current.max(next)),
(None, next) => next,
(current, None) => current,
};
if progress.current_object.is_some() {
snapshot.current_object = progress.current_object;
}
}
crate::heal::progress::aggregate_heal_progress(progresses)
snapshot.refresh_progress_percentage();
snapshot.refresh_estimated_completion_time();
Some(snapshot)
}
}
+34 -475
View File
@@ -15,91 +15,15 @@
use serde::{Deserialize, Serialize};
use std::time::{Duration, SystemTime};
pub(crate) fn stable_generation(parts: &[&[u8]]) -> u64 {
let mut hash = 0xcbf29ce484222325u64;
for part in parts {
for byte in (part.len() as u64).to_be_bytes().into_iter().chain(part.iter().copied()) {
hash ^= u64::from(byte);
hash = hash.wrapping_mul(0x100000001b3);
}
}
hash
}
#[cfg(test)]
mod stable_generation_tests {
use super::stable_generation;
#[test]
fn stable_generation_has_a_fixed_vector() {
assert_eq!(stable_generation(&[b"rustfs", b"heal", b"42"]), 11_007_672_338_488_385_056);
}
}
pub(crate) fn increment_counter(counter: &mut u64) -> bool {
match counter.checked_add(1) {
Some(next) => {
*counter = next;
true
}
None => {
*counter = u64::MAX;
false
}
}
}
pub(crate) fn add_bytes(total: &mut u64, amount: u64) -> bool {
match total.checked_add(amount) {
Some(next) => {
*total = next;
true
}
None => {
*total = u64::MAX;
false
}
}
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum HealProgressKind {
#[default]
Unknown,
Stage,
ObjectSweep,
}
/// Whether the object ledger can produce a meaningful percentage.
///
/// A zero-valued baseline is not a completed scan: it means that no complete
/// usage snapshot was available. Keep this state explicit so callers do not
/// mistake the legacy `0.0` wire value for a measured zero-percent result.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum HealProgressState {
#[default]
Unknown,
Indeterminate,
Running,
Completed,
}
#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default, rename_all = "camelCase")]
pub struct HealProgress {
#[serde(default)]
pub kind: HealProgressKind,
/// Objects scanned
pub objects_scanned: u64,
/// Objects healed
pub objects_healed: u64,
/// Objects failed
pub objects_failed: u64,
/// Versions deferred for a later retry pass.
#[serde(default)]
pub skipped_objects: u64,
/// Versions skipped because they were written after this heal started
pub skipped_new_versions: u64,
/// Versions skipped because lifecycle already selected them for expiry
@@ -120,38 +44,11 @@ pub struct HealProgress {
pub last_update_time: Option<SystemTime>,
/// Estimated completion time
pub estimated_completion_time: Option<SystemTime>,
/// Current stage number. Stage updates are intentionally independent from
/// the object ledger below.
#[serde(default)]
pub stage_current: u64,
/// Number of stages in the current task.
#[serde(default)]
pub stage_total: u64,
/// Explicitly distinguishes a missing usage baseline from measured 0%.
#[serde(default)]
pub progress_state: HealProgressState,
/// True only after the task's durable completion ledger was committed.
#[serde(default)]
pub ledger_complete: bool,
/// Generation of the usage snapshot used for the baseline, if available.
#[serde(default)]
pub baseline_generation: Option<u64>,
/// Whether the baseline was explicitly observed. This is separate from
/// the counters so a known empty scope (0 objects, 0 bytes) is not
/// confused with a legacy snapshot that omitted the baseline fields.
#[serde(default)]
pub baseline_known: bool,
/// Internal telemetry fence set when an aggregate counter overflows or
/// becomes inconsistent. It prevents a later refresh from fabricating a
/// percentage from the poisoned values.
#[serde(default)]
pub counter_unknown: bool,
}
impl HealProgress {
pub fn new() -> Self {
Self {
kind: HealProgressKind::Unknown,
start_time: Some(SystemTime::now()),
last_update_time: Some(SystemTime::now()),
..Default::default()
@@ -159,87 +56,12 @@ impl HealProgress {
}
pub fn update_progress(&mut self, scanned: u64, healed: u64, failed: u64, bytes: u64) {
self.update_object_sweep_progress(scanned, healed, failed, bytes);
}
pub fn update_object_sweep_progress(&mut self, scanned: u64, healed: u64, failed: u64, bytes: u64) {
self.kind = HealProgressKind::ObjectSweep;
self.objects_scanned = scanned;
self.objects_healed = healed;
self.objects_failed = failed;
self.bytes_processed = bytes;
self.last_update_time = Some(SystemTime::now());
let explicit_skipped = match self.skipped_new_versions.checked_add(self.skipped_ilm_expired) {
Some(value) => value,
None => {
self.mark_unknown();
0
}
};
let skipped = healed
.checked_add(failed)
.and_then(|value| value.checked_add(explicit_skipped))
.and_then(|value| scanned.checked_sub(value))
.unwrap_or(0);
self.update_object_progress(scanned, healed, failed, skipped, bytes);
}
/// Update task stage progress without modifying object counters.
pub fn update_stage(&mut self, current: u64, total: u64) {
let object_sweep_active = matches!(self.kind, HealProgressKind::ObjectSweep);
if !object_sweep_active {
self.kind = HealProgressKind::Stage;
}
self.ledger_complete = false;
self.stage_current = current.min(total);
self.stage_total = total;
if object_sweep_active {
self.last_update_time = Some(SystemTime::now());
self.refresh_progress_percentage();
return;
}
self.progress_state = if total == 0 {
HealProgressState::Indeterminate
} else {
HealProgressState::Running
};
self.progress_percentage = if total == 0 {
0.0
} else {
(current as f64 / total as f64 * 100.0).min(100.0)
};
self.last_update_time = Some(SystemTime::now());
}
/// Update the disjoint object ledger. `scanned` is the number of terminal
/// object outcomes and must equal healed + failed + deferred skipped plus
/// the two terminal skip classes. Overflow is a corrupt/unknown counter
/// state, not a reason to abort a completed heal.
pub fn update_object_progress(&mut self, scanned: u64, healed: u64, failed: u64, skipped: u64, bytes: u64) {
self.kind = HealProgressKind::ObjectSweep;
// `skipped` is the transient/deferred class. The two explicit skip
// counters are terminal classifications too, so include them in the
// same ledger without making callers maintain a second aggregate.
let outcomes = healed
.checked_add(failed)
.and_then(|value| value.checked_add(skipped))
.and_then(|value| value.checked_add(self.skipped_new_versions))
.and_then(|value| value.checked_add(self.skipped_ilm_expired));
self.objects_scanned = scanned;
self.objects_healed = healed;
self.objects_failed = failed;
self.skipped_objects = skipped;
self.bytes_processed = bytes;
self.last_update_time = Some(SystemTime::now());
self.ledger_complete = false;
if outcomes != Some(scanned) {
// Telemetry corruption must not abort a heal. Preserve the
// counters for diagnostics, but do not derive a percentage from a
// double-counted or overflowing ledger.
self.mark_unknown();
return;
}
self.refresh_progress_percentage();
self.refresh_estimated_completion_time();
}
@@ -247,88 +69,50 @@ impl HealProgress {
pub fn set_total_baseline(&mut self, objects_total_count: u64, objects_total_size: u64) {
self.objects_total_count = objects_total_count;
self.objects_total_size = objects_total_size;
self.baseline_known = true;
self.last_update_time = Some(SystemTime::now());
self.refresh_progress_percentage();
self.refresh_estimated_completion_time();
}
pub fn set_total_baseline_with_generation(&mut self, objects_total_count: u64, objects_total_size: u64, generation: u64) {
self.baseline_generation = Some(generation);
self.set_total_baseline(objects_total_count, objects_total_size);
}
pub fn record_skipped_new_version(&mut self) {
let Some(next) = self.skipped_new_versions.checked_add(1) else {
self.mark_unknown();
return;
};
self.skipped_new_versions = next;
self.skipped_new_versions = self.skipped_new_versions.saturating_add(1);
self.last_update_time = Some(SystemTime::now());
self.refresh_progress_percentage();
self.refresh_estimated_completion_time();
}
pub fn record_skipped_ilm_expired(&mut self) {
let Some(next) = self.skipped_ilm_expired.checked_add(1) else {
self.mark_unknown();
return;
};
self.skipped_ilm_expired = next;
self.skipped_ilm_expired = self.skipped_ilm_expired.saturating_add(1);
self.last_update_time = Some(SystemTime::now());
self.refresh_progress_percentage();
self.refresh_estimated_completion_time();
}
fn completed_for_baseline(&self) -> Option<u64> {
fn completed_for_baseline(&self) -> u64 {
self.objects_healed
.checked_add(self.objects_failed)?
.checked_add(self.skipped_objects)?
.checked_add(self.skipped_new_versions)?
.checked_add(self.skipped_ilm_expired)
.saturating_add(self.objects_failed)
.saturating_add(self.skipped_new_versions)
.saturating_add(self.skipped_ilm_expired)
}
pub(crate) fn refresh_progress_percentage(&mut self) {
if self.ledger_complete {
self.progress_state = HealProgressState::Completed;
self.progress_percentage = 100.0;
return;
}
if self.counter_unknown {
self.progress_state = HealProgressState::Unknown;
self.progress_percentage = 0.0;
return;
}
if !self.baseline_known {
self.progress_state = HealProgressState::Indeterminate;
self.progress_percentage = 0.0;
self.estimated_completion_time = None;
return;
}
if self.objects_total_size > 0 {
self.progress_percentage = ((self.bytes_processed as f64 / self.objects_total_size as f64) * 100.0).min(100.0);
self.progress_percentage = self.progress_percentage.min(99.999);
self.progress_state = HealProgressState::Running;
return;
}
if self.objects_total_count > 0 {
let Some(completed) = self.completed_for_baseline() else {
self.progress_state = HealProgressState::Unknown;
self.progress_percentage = 0.0;
return;
};
let completed = self.completed_for_baseline();
self.progress_percentage = ((completed as f64 / self.objects_total_count as f64) * 100.0).min(100.0);
self.progress_percentage = self.progress_percentage.min(99.999);
self.progress_state = HealProgressState::Running;
return;
}
if self.baseline_known {
self.progress_state = HealProgressState::Running;
self.progress_percentage = 0.0;
return;
let total = self
.objects_scanned
.saturating_add(self.objects_healed)
.saturating_add(self.objects_failed);
if total > 0 {
self.progress_percentage = (self.objects_healed as f64 / total as f64) * 100.0;
}
self.progress_state = HealProgressState::Indeterminate;
self.progress_percentage = 0.0;
}
pub fn set_current_object(&mut self, object: Option<String>) {
@@ -341,11 +125,7 @@ impl HealProgress {
self.estimated_completion_time = None;
return;
};
if self.is_completed()
|| self.progress_percentage <= 0.0
|| self.progress_percentage >= 100.0
|| self.bytes_processed == 0
{
if self.is_completed() || !(0.0..100.0).contains(&self.progress_percentage) || self.bytes_processed == 0 {
self.estimated_completion_time = None;
return;
}
@@ -362,39 +142,18 @@ impl HealProgress {
}
pub fn is_completed(&self) -> bool {
self.ledger_complete
}
/// Mark telemetry unknown while allowing the underlying heal operation to
/// continue. This is used for corrupt/overflowing counters at the
/// observability boundary; it must never turn a successful heal into an
/// execution error.
pub fn mark_unknown(&mut self) {
self.counter_unknown = true;
self.progress_state = HealProgressState::Unknown;
self.ledger_complete = false;
self.progress_percentage = 0.0;
self.estimated_completion_time = None;
self.last_update_time = Some(SystemTime::now());
}
/// Mark the object ledger terminal only after the enclosing task has
/// committed all durable resume state and cleanup fences.
pub fn mark_completed(&mut self) {
let telemetry_unknown = self.counter_unknown || self.progress_state == HealProgressState::Unknown;
self.ledger_complete = true;
if !telemetry_unknown {
self.progress_state = HealProgressState::Completed;
if self.progress_percentage >= 100.0 {
return true;
}
self.progress_percentage = 100.0;
self.last_update_time = Some(SystemTime::now());
self.estimated_completion_time = None;
if self.objects_total_count > 0 || self.objects_total_size > 0 {
return false;
}
self.objects_scanned > 0 && self.objects_healed.saturating_add(self.objects_failed) >= self.objects_scanned
}
pub fn get_success_rate(&self) -> f64 {
let Some(total) = self.objects_healed.checked_add(self.objects_failed) else {
return 0.0;
};
let total = self.objects_healed + self.objects_failed;
if total > 0 {
(self.objects_healed as f64 / total as f64) * 100.0
} else {
@@ -403,101 +162,6 @@ impl HealProgress {
}
}
pub fn aggregate_heal_progress(progresses: impl IntoIterator<Item = HealProgress>) -> Option<HealProgress> {
let mut snapshot = HealProgress::default();
let mut found = false;
let mut has_object_sweep = false;
let mut all_object_baselines_known = true;
let mut baseline_generation = None;
let mut baseline_generation_consistent = true;
let mut all_ledgers_complete = true;
let mut counter_overflow = false;
for progress in progresses {
found = true;
let object_sweep = matches!(progress.kind, HealProgressKind::ObjectSweep);
has_object_sweep |= object_sweep;
all_ledgers_complete &= progress.ledger_complete;
if object_sweep {
all_object_baselines_known &= progress.baseline_known;
match baseline_generation {
None => baseline_generation = Some(progress.baseline_generation),
Some(generation) => baseline_generation_consistent &= generation == progress.baseline_generation,
}
}
counter_overflow |= progress.counter_unknown || matches!(progress.progress_state, HealProgressState::Unknown);
for (target, value) in [
(&mut snapshot.objects_scanned, progress.objects_scanned),
(&mut snapshot.objects_healed, progress.objects_healed),
(&mut snapshot.objects_failed, progress.objects_failed),
(&mut snapshot.skipped_objects, progress.skipped_objects),
(&mut snapshot.skipped_new_versions, progress.skipped_new_versions),
(&mut snapshot.skipped_ilm_expired, progress.skipped_ilm_expired),
(&mut snapshot.objects_total_count, progress.objects_total_count),
(&mut snapshot.objects_total_size, progress.objects_total_size),
(&mut snapshot.bytes_processed, progress.bytes_processed),
(&mut snapshot.stage_current, progress.stage_current),
(&mut snapshot.stage_total, progress.stage_total),
] {
match target.checked_add(value) {
Some(sum) => *target = sum,
None => {
*target = u64::MAX;
counter_overflow = true;
}
}
}
snapshot.start_time = match (snapshot.start_time, progress.start_time) {
(Some(current), Some(next)) => Some(current.min(next)),
(None, next) => next,
(current, None) => current,
};
snapshot.last_update_time = match (snapshot.last_update_time, progress.last_update_time) {
(Some(current), Some(next)) => Some(current.max(next)),
(None, next) => next,
(current, None) => current,
};
if progress.current_object.is_some() {
snapshot.current_object = progress.current_object;
}
}
if !found {
return None;
}
snapshot.kind = if has_object_sweep {
HealProgressKind::ObjectSweep
} else {
HealProgressKind::Stage
};
snapshot.baseline_known = has_object_sweep && all_object_baselines_known && baseline_generation_consistent;
snapshot.baseline_generation = if snapshot.baseline_known && baseline_generation_consistent {
baseline_generation.flatten()
} else {
None
};
snapshot.ledger_complete = all_ledgers_complete;
snapshot.counter_unknown = counter_overflow;
if counter_overflow {
snapshot.progress_state = HealProgressState::Unknown;
snapshot.progress_percentage = if snapshot.ledger_complete { 100.0 } else { 0.0 };
} else if snapshot.ledger_complete {
snapshot.progress_state = HealProgressState::Completed;
snapshot.progress_percentage = 100.0;
} else if has_object_sweep {
snapshot.refresh_progress_percentage();
} else if snapshot.stage_total == 0 {
snapshot.progress_state = HealProgressState::Indeterminate;
snapshot.progress_percentage = 0.0;
} else {
snapshot.progress_state = HealProgressState::Running;
snapshot.progress_percentage = ((snapshot.stage_current as f64 / snapshot.stage_total as f64) * 100.0).min(99.999);
}
snapshot.refresh_estimated_completion_time();
Some(snapshot)
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HealStatistics {
/// Total heal tasks
@@ -566,7 +230,6 @@ mod tests {
assert_eq!(progress.objects_scanned, 0);
assert_eq!(progress.objects_healed, 0);
assert_eq!(progress.objects_failed, 0);
assert_eq!(progress.skipped_objects, 0);
assert_eq!(progress.skipped_new_versions, 0);
assert_eq!(progress.skipped_ilm_expired, 0);
assert_eq!(progress.objects_total_count, 0);
@@ -587,8 +250,10 @@ mod tests {
assert_eq!(progress.objects_healed, 8);
assert_eq!(progress.objects_failed, 2);
assert_eq!(progress.bytes_processed, 1024);
assert_eq!(progress.progress_state, HealProgressState::Indeterminate);
assert_eq!(progress.progress_percentage, 0.0);
// Progress percentage should be calculated based on healed/total
// total = scanned + healed + failed = 10 + 8 + 2 = 20
// healed/total = 8/20 = 0.4 = 40%
assert!((progress.progress_percentage - 40.0).abs() < 0.001);
assert!(progress.last_update_time.is_some());
}
@@ -597,8 +262,7 @@ mod tests {
let mut progress = HealProgress::new();
progress.start_time = Some(SystemTime::now() - Duration::from_secs(10));
progress.set_total_baseline(100, 16384);
progress.update_progress(25, 25, 0, 4096);
progress.update_progress(100, 25, 0, 4096);
let eta = progress
.estimated_completion_time
@@ -611,7 +275,7 @@ mod tests {
let mut progress = HealProgress::new();
progress.set_total_baseline(10, 8192);
progress.update_progress(25, 25, 0, 4096);
progress.update_progress(100, 25, 0, 4096);
assert!((progress.progress_percentage - 50.0).abs() < 0.001);
}
@@ -621,7 +285,7 @@ mod tests {
let mut progress = HealProgress::new();
progress.set_total_baseline(10, 0);
progress.update_progress(5, 3, 2, 0);
progress.update_progress(100, 3, 2, 0);
assert!((progress.progress_percentage - 50.0).abs() < 0.001);
}
@@ -631,7 +295,7 @@ mod tests {
let mut progress = HealProgress::new();
progress.set_total_baseline(10, 0);
progress.update_progress(5, 3, 2, 0);
progress.update_progress(100, 3, 2, 0);
progress.record_skipped_new_version();
assert_eq!(progress.skipped_new_versions, 1);
@@ -672,8 +336,7 @@ mod tests {
fn test_heal_progress_update_progress_all_healed() {
let mut progress = HealProgress::new();
// When scanned=0, healed=10, failed=0: total=10, progress = 10/10 = 100%
progress.update_progress(10, 10, 0, 2048);
progress.mark_completed();
progress.update_progress(0, 10, 0, 2048);
// All healed, should be 100%
assert!((progress.progress_percentage - 100.0).abs() < 0.001);
@@ -731,7 +394,6 @@ mod tests {
assert_eq!(json["objectsScanned"], 10);
assert_eq!(json["objectsHealed"], 8);
assert_eq!(json["objectsFailed"], 2);
assert_eq!(json["skippedObjects"], 0);
assert_eq!(json["skippedNewVersions"], 0);
assert_eq!(json["skippedIlmExpired"], 0);
assert_eq!(json["bytesProcessed"], 1024);
@@ -743,7 +405,6 @@ mod tests {
fn test_heal_progress_is_completed_by_percentage() {
let mut progress = HealProgress::new();
progress.update_progress(10, 10, 0, 1024);
progress.mark_completed();
assert!(progress.is_completed());
}
@@ -754,7 +415,7 @@ mod tests {
progress.objects_scanned = 10;
progress.objects_healed = 8;
progress.objects_failed = 2;
progress.mark_completed();
// healed + failed = 8 + 2 = 10 >= scanned = 10
assert!(progress.is_completed());
}
@@ -794,108 +455,6 @@ mod tests {
assert!((progress.get_success_rate() - 100.0).abs() < 0.001);
}
#[test]
fn single_object_progress_reaches_terminal_100() {
let mut progress = HealProgress::new();
progress.update_object_progress(1, 1, 0, 0, 128);
assert!(!progress.is_completed());
progress.mark_completed();
assert!(progress.is_completed());
assert_eq!(progress.progress_percentage, 100.0);
}
#[test]
fn progress_without_baseline_is_indeterminate() {
let mut progress = HealProgress::new();
progress.update_object_progress(1, 1, 0, 0, 128);
assert_eq!(progress.progress_state, HealProgressState::Indeterminate);
assert_eq!(progress.progress_percentage, 0.0);
assert!(progress.estimated_completion_time.is_none());
}
#[test]
fn progress_retry_is_exactly_once() {
let mut progress = HealProgress::new();
progress.set_total_baseline(1, 128);
progress.update_object_progress(1, 1, 0, 0, 128);
progress.update_object_progress(1, 1, 0, 0, 128);
assert_eq!(progress.objects_scanned, 1);
assert_eq!(progress.objects_healed, 1);
assert_eq!(progress.bytes_processed, 128);
}
#[test]
fn progress_never_triggers_cleanup_before_terminal_ledger_empty() {
let mut progress = HealProgress::new();
progress.progress_percentage = 100.0;
assert!(!progress.is_completed());
progress.mark_completed();
assert!(progress.is_completed());
}
#[test]
fn progress_counter_overflow_is_marked_unknown_without_aborting_completed_heal() {
let mut progress = HealProgress::new();
progress.update_object_progress(u64::MAX, u64::MAX, 1, 0, 0);
assert_eq!(progress.progress_state, HealProgressState::Unknown);
progress.mark_completed();
assert!(progress.is_completed());
assert_eq!(progress.progress_state, HealProgressState::Unknown);
let aggregate = aggregate_heal_progress([progress]).expect("progress should aggregate");
assert!(aggregate.ledger_complete);
assert!(aggregate.counter_unknown);
assert_eq!(aggregate.progress_state, HealProgressState::Unknown);
assert_eq!(aggregate.progress_percentage, 100.0);
}
#[test]
fn aggregate_rejects_mixed_baseline_generations() {
let progress = |generation| HealProgress {
kind: HealProgressKind::ObjectSweep,
objects_scanned: 5,
objects_total_count: 10,
progress_state: HealProgressState::Running,
baseline_generation: Some(generation),
baseline_known: true,
..Default::default()
};
let aggregate = aggregate_heal_progress([progress(1), progress(2)]).expect("progress should aggregate");
assert!(!aggregate.baseline_known);
assert_eq!(aggregate.baseline_generation, None);
assert_eq!(aggregate.progress_state, HealProgressState::Indeterminate);
assert_eq!(aggregate.progress_percentage, 0.0);
}
#[test]
fn aggregate_accepts_multiple_sets_from_one_snapshot_generation() {
let progress = |objects_scanned| HealProgress {
kind: HealProgressKind::ObjectSweep,
objects_scanned,
objects_total_count: 10,
progress_state: HealProgressState::Running,
baseline_generation: Some(7),
baseline_known: true,
..Default::default()
};
let aggregate = aggregate_heal_progress([progress(5), progress(3)]).expect("progress should aggregate");
assert!(aggregate.baseline_known);
assert_eq!(aggregate.baseline_generation, Some(7));
}
#[test]
fn stage_updates_do_not_double_count_object_outcomes() {
let mut progress = HealProgress::new();
progress.update_object_progress(2, 1, 0, 1, 256);
progress.update_stage(3, 4);
assert_eq!(progress.kind, HealProgressKind::ObjectSweep);
assert_eq!(progress.objects_scanned, 2);
assert_eq!(progress.objects_healed, 1);
assert_eq!(progress.skipped_objects, 1);
}
#[test]
fn test_heal_statistics_new() {
let stats = HealStatistics::new();
+4 -130
View File
@@ -31,7 +31,7 @@ mod checkpoint;
mod replacement;
mod utils;
pub use checkpoint::{CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ResumeCheckpoint};
pub use checkpoint::{CheckpointManager, ResumeCheckpoint};
pub(crate) use replacement::replacement_target_identities_match;
use replacement::replacement_targets_match_identities;
pub use replacement::{
@@ -340,12 +340,6 @@ pub struct ResumeState {
pub failed_objects: u64,
/// skipped objects
pub skipped_objects: u64,
/// Terminal versions skipped because they were newer than the heal start.
#[serde(default)]
pub skipped_new_versions: u64,
/// Terminal versions handed to lifecycle expiry.
#[serde(default)]
pub skipped_ilm_expired: u64,
/// current bucket
pub current_bucket: Option<String>,
/// current object
@@ -360,24 +354,6 @@ pub struct ResumeState {
pub retry_count: u32,
/// max retries
pub max_retries: u32,
/// Bytes accounted by the object ledger; additive for old snapshots.
#[serde(default)]
pub processed_bytes: u64,
/// Total bytes from a complete usage snapshot, when available.
#[serde(default)]
pub total_bytes: u64,
/// Generation of the usage snapshot used for the baseline.
#[serde(default)]
pub baseline_generation: Option<u64>,
/// Whether the usage baseline is known. Missing in old snapshots means
/// indeterminate rather than a measured zero baseline.
#[serde(default)]
pub baseline_known: bool,
/// Persistent telemetry fence for counter/byte overflow or corruption.
/// It must survive a restart so a saturated snapshot is never presented as
/// a measured percentage on the next resume.
#[serde(default)]
pub counter_unknown: bool,
}
impl ResumeState {
@@ -401,8 +377,6 @@ impl ResumeState {
successful_objects: 0,
failed_objects: 0,
skipped_objects: 0,
skipped_new_versions: 0,
skipped_ilm_expired: 0,
current_bucket: None,
current_object: None,
completed_buckets: Vec::new(),
@@ -410,11 +384,6 @@ impl ResumeState {
error_message: None,
retry_count: 0,
max_retries: 3,
processed_bytes: 0,
total_bytes: 0,
baseline_generation: None,
baseline_known: false,
counter_unknown: false,
}
}
@@ -443,39 +412,6 @@ impl ResumeState {
self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
}
pub fn update_progress_with_bytes(
&mut self,
processed: u64,
successful: u64,
failed: u64,
skipped: u64,
processed_bytes: u64,
) {
self.update_progress(processed, successful, failed, skipped);
self.processed_bytes = processed_bytes;
}
pub fn set_skipped_version_counts(&mut self, new_versions: u64, ilm_expired: u64) {
self.skipped_new_versions = new_versions;
self.skipped_ilm_expired = ilm_expired;
self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
}
pub fn set_progress_baseline(&mut self, total_objects: u64, total_bytes: u64, generation: Option<u64>) {
self.total_objects = total_objects;
self.total_bytes = total_bytes;
self.baseline_generation = generation;
// This method is called only after a complete usage snapshot has been
// validated. A complete but empty snapshot is still a known baseline.
self.baseline_known = true;
self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
}
pub fn mark_counter_unknown(&mut self) {
self.counter_unknown = true;
self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
}
pub fn set_current_item(&mut self, bucket: Option<String>, object: Option<String>) {
self.current_bucket = bucket;
self.current_object = object;
@@ -501,7 +437,6 @@ impl ResumeState {
if let Some(pos) = self.pending_buckets.iter().position(|b| b == bucket) {
self.pending_buckets.remove(pos);
}
self.resume_cursor = None;
self.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
}
@@ -519,10 +454,6 @@ impl ResumeState {
self.successful_objects = 0;
self.failed_objects = 0;
self.skipped_objects = 0;
self.skipped_new_versions = 0;
self.skipped_ilm_expired = 0;
self.processed_bytes = 0;
self.counter_unknown = false;
self.completed = false;
// A retry re-scans every bucket from the beginning, so the version
// cursor must be cleared too — otherwise the retry would resume mid-scan.
@@ -545,28 +476,14 @@ impl ResumeState {
}
pub fn get_progress_percentage(&self) -> f64 {
if self.completed {
return 100.0;
}
if self.counter_unknown {
return 0.0;
}
if !self.baseline_known {
return 0.0;
}
if self.total_bytes > 0 {
return ((self.processed_bytes as f64 / self.total_bytes as f64) * 100.0).min(99.999);
}
if self.total_objects == 0 {
return 0.0;
}
((self.processed_objects as f64 / self.total_objects as f64) * 100.0).min(99.999)
(self.processed_objects as f64 / self.total_objects as f64) * 100.0
}
pub fn get_success_rate(&self) -> f64 {
let Some(total) = self.successful_objects.checked_add(self.failed_objects) else {
return 0.0;
};
let total = self.successful_objects + self.failed_objects;
if total == 0 {
return 0.0;
}
@@ -837,14 +754,6 @@ impl ResumeManager {
state.successful_objects = 0;
state.failed_objects = 0;
state.skipped_objects = 0;
state.skipped_new_versions = 0;
state.skipped_ilm_expired = 0;
state.processed_bytes = 0;
state.total_objects = 0;
state.total_bytes = 0;
state.baseline_generation = None;
state.baseline_known = false;
state.counter_unknown = false;
state.completed = false;
state.completed_buckets.clear();
state.schema_version = CURRENT_RESUME_SCHEMA;
@@ -929,41 +838,6 @@ impl ResumeManager {
self.save_state_throttled().await
}
pub async fn update_progress_with_bytes(
&self,
processed: u64,
successful: u64,
failed: u64,
skipped: u64,
processed_bytes: u64,
) -> Result<()> {
let mut state = self.state.write().await;
state.update_progress_with_bytes(processed, successful, failed, skipped, processed_bytes);
drop(state);
self.save_state_throttled().await
}
pub async fn set_progress_baseline(&self, total_objects: u64, total_bytes: u64, generation: Option<u64>) -> Result<()> {
let mut state = self.state.write().await;
state.set_progress_baseline(total_objects, total_bytes, generation);
drop(state);
self.save_state_throttled().await
}
pub async fn mark_counter_unknown(&self) -> Result<()> {
let mut state = self.state.write().await;
state.mark_counter_unknown();
drop(state);
self.save_state().await
}
pub async fn set_skipped_version_counts(&self, new_versions: u64, ilm_expired: u64) -> Result<()> {
let mut state = self.state.write().await;
state.set_skipped_version_counts(new_versions, ilm_expired);
drop(state);
self.save_state_throttled().await
}
/// Set current item. Called once per healed object, so persistence is
/// throttled: the in-memory state always updates, but the snapshot is only
/// written every `PERSIST_EVERY_MUTATIONS` calls or `PERSIST_INTERVAL`.
@@ -1008,7 +882,7 @@ impl ResumeManager {
let mut state = self.state.write().await;
state.complete_bucket(bucket);
drop(state);
self.save_state().await
self.save_state_throttled().await
}
/// mark task completed
+9 -198
View File
@@ -29,30 +29,10 @@ use super::{
const EVENT_HEAL_CHECKPOINT_STATE: &str = "heal_checkpoint_state";
/// Current on-disk schema version for `ResumeCheckpoint`. Schema 5 could
/// persist dedup identities without the aggregate counters needed to restore
/// them safely, so stale checkpoints are discarded and replayed.
pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 6;
#[derive(Debug, Clone, Copy)]
pub enum CheckpointObjectOutcome {
Processed,
Failed,
Skipped,
}
#[derive(Debug)]
pub struct CheckpointObjectOutcomeRecord {
pub object: String,
pub outcome: CheckpointObjectOutcome,
pub successful: u64,
pub failed: u64,
pub skipped: u64,
pub bytes: u64,
pub skipped_new_versions: u64,
pub skipped_ilm_expired: u64,
pub counter_unknown: bool,
}
/// Current on-disk schema version for `ResumeCheckpoint`. Same rationale as
/// `CURRENT_RESUME_SCHEMA`: pre-per-version dedup identities are not comparable
/// to the new `compose_key` identities, so a stale checkpoint is discarded.
pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 5;
/// resume checkpoint
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -77,30 +57,6 @@ pub struct ResumeCheckpoint {
pub failed_objects: HashSet<String>,
/// skipped objects
pub skipped_objects: HashSet<String>,
/// Aggregate object ledger counters restored alongside the dedup sets.
#[serde(default)]
pub successful_objects: u64,
#[serde(default)]
pub failed_object_count: u64,
#[serde(default)]
pub skipped_object_count: u64,
#[serde(default)]
pub skipped_new_versions: u64,
#[serde(default)]
pub skipped_ilm_expired: u64,
#[serde(default)]
pub processed_bytes: u64,
#[serde(default)]
pub total_objects: u64,
#[serde(default)]
pub total_bytes: u64,
#[serde(default)]
pub baseline_generation: Option<u64>,
#[serde(default)]
pub baseline_known: bool,
/// Persistent telemetry fence for counter/byte overflow or corruption.
#[serde(default)]
pub counter_unknown: bool,
}
impl ResumeCheckpoint {
@@ -114,17 +70,6 @@ impl ResumeCheckpoint {
processed_objects: HashSet::new(),
failed_objects: HashSet::new(),
skipped_objects: HashSet::new(),
successful_objects: 0,
failed_object_count: 0,
skipped_object_count: 0,
skipped_new_versions: 0,
skipped_ilm_expired: 0,
processed_bytes: 0,
total_objects: 0,
total_bytes: 0,
baseline_generation: None,
baseline_known: false,
counter_unknown: false,
}
}
@@ -146,34 +91,6 @@ impl ResumeCheckpoint {
self.skipped_objects.insert(object);
}
pub fn update_progress(&mut self, successful: u64, failed: u64, skipped: u64, bytes: u64) {
self.successful_objects = successful;
self.failed_object_count = failed;
self.skipped_object_count = skipped;
self.processed_bytes = bytes;
}
pub fn set_progress_baseline(&mut self, total_objects: u64, total_bytes: u64, generation: Option<u64>) {
self.total_objects = total_objects;
self.total_bytes = total_bytes;
self.baseline_generation = generation;
// The caller has already validated that this is a complete snapshot;
// preserve the distinction between a known empty scope and an old
// checkpoint that omitted all baseline fields.
self.baseline_known = true;
}
pub fn mark_counter_unknown(&mut self) {
self.counter_unknown = true;
self.checkpoint_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
}
pub fn set_skipped_version_counts(&mut self, new_versions: u64, ilm_expired: u64) {
self.skipped_new_versions = new_versions;
self.skipped_ilm_expired = ilm_expired;
self.checkpoint_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
}
/// Advance past a fully-processed page: objects below `object_index` are
/// skipped by position on resume, so the per-object sets no longer need
/// their entries and would otherwise grow with the whole bucket.
@@ -190,17 +107,6 @@ impl ResumeCheckpoint {
self.update_position(0, 0);
self.processed_objects.clear();
self.skipped_objects.clear();
self.successful_objects = 0;
self.failed_object_count = 0;
self.skipped_object_count = 0;
self.skipped_new_versions = 0;
self.skipped_ilm_expired = 0;
self.processed_bytes = 0;
self.total_objects = 0;
self.total_bytes = 0;
self.baseline_generation = None;
self.baseline_known = false;
self.counter_unknown = false;
self.failed_objects.clear();
}
}
@@ -252,9 +158,10 @@ impl CheckpointManager {
});
}
// Older checkpoints can contain identities that are not comparable to
// the current keys or lack their corresponding aggregate counters.
// Discard the stale sets and position so the scan restarts cleanly.
// A checkpoint from an older schema stored latest-only dedup identities
// that are not comparable to the new per-version `compose_key`
// identities. Discard the stale sets and position, then stamp the
// current schema so the scan restarts cleanly.
if checkpoint.schema_version > CURRENT_CHECKPOINT_SCHEMA {
return Err(Error::TaskExecutionFailed {
message: format!(
@@ -278,17 +185,6 @@ impl CheckpointManager {
checkpoint.processed_objects.clear();
checkpoint.failed_objects.clear();
checkpoint.skipped_objects.clear();
checkpoint.successful_objects = 0;
checkpoint.failed_object_count = 0;
checkpoint.skipped_object_count = 0;
checkpoint.skipped_new_versions = 0;
checkpoint.skipped_ilm_expired = 0;
checkpoint.processed_bytes = 0;
checkpoint.total_objects = 0;
checkpoint.total_bytes = 0;
checkpoint.baseline_generation = None;
checkpoint.baseline_known = false;
checkpoint.counter_unknown = false;
checkpoint.current_bucket_index = 0;
checkpoint.current_object_index = 0;
checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA;
@@ -329,7 +225,7 @@ impl CheckpointManager {
self.save_checkpoint_throttled().await
}
/// Persist a completed page position while retaining its identities.
/// Advance past a completed page and prune the per-object sets, then persist.
pub async fn complete_page(&self, bucket_index: usize, object_index: usize) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.complete_page(bucket_index, object_index);
@@ -337,35 +233,6 @@ impl CheckpointManager {
self.save_checkpoint_throttled().await
}
/// Persist the page position while retaining identities until the resume
/// cursor is durable.
pub async fn advance_page(&self, bucket_index: usize, object_index: usize) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.update_position(bucket_index, object_index);
drop(checkpoint);
self.save_checkpoint().await
}
/// Remove the previous page's dedup identities only after its resume cursor
/// has been durably exposed.
pub async fn prune_completed_page(&self) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.processed_objects.clear();
checkpoint.skipped_objects.clear();
checkpoint.failed_objects.clear();
drop(checkpoint);
self.save_checkpoint().await
}
/// Advance to the next bucket and clear the final page identities after the
/// resume state has durably recorded the completed bucket.
pub async fn complete_bucket(&self, next_bucket_index: usize) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.complete_page(next_bucket_index, 0);
drop(checkpoint);
self.save_checkpoint().await
}
/// Reset the checkpoint to the start of the scan for a retry, then persist.
pub async fn reset_for_retry(&self) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
@@ -400,62 +267,6 @@ impl CheckpointManager {
self.save_checkpoint_if_due().await
}
/// Atomically persist an object's dedup identity with its aggregate result.
pub async fn record_object_outcome(&self, record: CheckpointObjectOutcomeRecord) -> Result<()> {
let CheckpointObjectOutcomeRecord {
object,
outcome,
successful,
failed,
skipped,
bytes,
skipped_new_versions,
skipped_ilm_expired,
counter_unknown,
} = record;
let mut checkpoint = self.checkpoint.write().await;
match outcome {
CheckpointObjectOutcome::Processed => checkpoint.add_processed_object(object),
CheckpointObjectOutcome::Failed => checkpoint.add_failed_object(object),
CheckpointObjectOutcome::Skipped => checkpoint.add_skipped_object(object),
}
checkpoint.update_progress(successful, failed, skipped, bytes);
checkpoint.set_skipped_version_counts(skipped_new_versions, skipped_ilm_expired);
if counter_unknown {
checkpoint.mark_counter_unknown();
}
drop(checkpoint);
self.save_checkpoint_if_due().await
}
pub async fn update_progress(&self, successful: u64, failed: u64, skipped: u64, bytes: u64) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.update_progress(successful, failed, skipped, bytes);
drop(checkpoint);
self.save_checkpoint_if_due().await
}
pub async fn set_progress_baseline(&self, total_objects: u64, total_bytes: u64, generation: Option<u64>) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.set_progress_baseline(total_objects, total_bytes, generation);
drop(checkpoint);
self.save_checkpoint_throttled().await
}
pub async fn mark_counter_unknown(&self) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.mark_counter_unknown();
drop(checkpoint);
self.save_checkpoint().await
}
pub async fn set_skipped_version_counts(&self, new_versions: u64, ilm_expired: u64) -> Result<()> {
let mut checkpoint = self.checkpoint.write().await;
checkpoint.set_skipped_version_counts(new_versions, ilm_expired);
drop(checkpoint);
self.save_checkpoint_throttled().await
}
async fn save_checkpoint_if_due(&self) -> Result<()> {
let should_save = self.throttle.lock().map(|mut throttle| throttle.record()).unwrap_or(true);
if !should_save {
+4 -153
View File
@@ -1296,7 +1296,6 @@ async fn test_resume_state_progress() {
assert_eq!(progress, 0.0); // total_objects is 0
state.total_objects = 100;
state.baseline_known = true;
let progress = state.get_progress_percentage();
assert_eq!(progress, 10.0);
}
@@ -1476,40 +1475,6 @@ fn test_checkpoint_object_sets_dedupe_and_prune() {
assert!(checkpoint.failed_objects.is_empty());
}
#[tokio::test]
async fn checkpoint_page_commit_keeps_ledger_until_cursor_is_durable() {
let (_temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let checkpoint = CheckpointManager::new(disk.clone(), task_id.clone()).await.unwrap();
checkpoint
.record_object_outcome(CheckpointObjectOutcomeRecord {
object: "bucket/object:v1".to_string(),
outcome: CheckpointObjectOutcome::Processed,
successful: 1,
failed: 0,
skipped: 0,
bytes: 128,
skipped_new_versions: 0,
skipped_ilm_expired: 0,
counter_unknown: false,
})
.await
.unwrap();
checkpoint.advance_page(0, 1).await.unwrap();
let reloaded = CheckpointManager::load_from_disk(disk.clone(), &task_id).await.unwrap();
let snapshot = reloaded.get_checkpoint().await;
assert_eq!(snapshot.current_object_index, 1);
assert_eq!(snapshot.successful_objects, 1);
assert_eq!(snapshot.processed_bytes, 128);
assert!(snapshot.processed_objects.contains("bucket/object:v1"));
checkpoint.prune_completed_page().await.unwrap();
let reloaded = CheckpointManager::load_from_disk(disk, &task_id).await.unwrap();
assert!(reloaded.get_checkpoint().await.processed_objects.is_empty());
}
#[test]
fn test_checkpoint_loads_legacy_vec_format() {
// Checkpoints written before the HashSet migration stored the object
@@ -1603,14 +1568,14 @@ async fn test_resumestate_schema_v0_discarded_on_load() {
}
#[tokio::test]
async fn test_checkpoint_schema_v5_discarded_on_load() {
async fn test_checkpoint_schema_v4_discarded_on_load() {
let (temp_dir, disk) = schema_test_disk().await;
// Schema v5 can persist failed identities without the aggregate counters
// that make those identities safe to deduplicate after an upgrade.
// The previous checkpoint schema is unsafe once its paired resume
// state is discarded: retaining either position would skip work.
let task_id = "00000000-0000-4000-8000-000000000002";
let legacy = r#"{
"schema_version": 5,
"schema_version": 4,
"task_id": "00000000-0000-4000-8000-000000000002",
"checkpoint_time": 1700000000,
"current_bucket_index": 2,
@@ -1674,120 +1639,6 @@ async fn current_normal_resume_schema_preserves_progress() {
temp_dir.close().expect("remove schema test directory");
}
#[test]
fn progress_checkpoint_restores_bytes_and_generation() {
let mut checkpoint = ResumeCheckpoint::new("progress-checkpoint".to_string());
checkpoint.set_progress_baseline(9, 4096, Some(77));
checkpoint.update_progress(4, 1, 2, 2048);
checkpoint.set_skipped_version_counts(3, 1);
checkpoint.mark_counter_unknown();
let restored: ResumeCheckpoint =
serde_json::from_slice(&serde_json::to_vec(&checkpoint).expect("serialize checkpoint")).expect("deserialize checkpoint");
assert_eq!(restored.processed_bytes, 2048);
assert_eq!(restored.total_objects, 9);
assert_eq!(restored.total_bytes, 4096);
assert_eq!(restored.baseline_generation, Some(77));
assert!(restored.baseline_known);
assert_eq!(restored.skipped_new_versions, 3);
assert_eq!(restored.skipped_ilm_expired, 1);
assert!(restored.counter_unknown);
}
#[test]
fn old_progress_schema_migrates_missing_fields_to_unknown() {
let state = ResumeState::new(
"legacy-progress".to_string(),
"erasure_set".to_string(),
"pool_0_set_0".to_string(),
Vec::new(),
);
let mut value = serde_json::to_value(state).expect("serialize legacy-compatible state");
let object = value.as_object_mut().expect("state must be an object");
for field in [
"processed_bytes",
"total_bytes",
"baseline_generation",
"baseline_known",
"skipped_new_versions",
"skipped_ilm_expired",
] {
object.remove(field);
}
object.insert("total_objects".to_string(), serde_json::json!(10));
object.insert("processed_objects".to_string(), serde_json::json!(5));
let restored: ResumeState = serde_json::from_value(value).expect("deserialize old progress state");
assert_eq!(restored.processed_bytes, 0);
assert_eq!(restored.total_bytes, 0);
assert_eq!(restored.baseline_generation, None);
assert!(!restored.baseline_known, "missing baseline must remain unknown");
assert_eq!(restored.get_progress_percentage(), 0.0);
assert_eq!(restored.skipped_new_versions, 0);
assert_eq!(restored.skipped_ilm_expired, 0);
}
#[test]
fn progress_counter_unknown_survives_resume_round_trip() {
let mut state = ResumeState::new(
"overflow-progress".to_string(),
"erasure_set".to_string(),
"pool_0_set_0".to_string(),
Vec::new(),
);
state.mark_counter_unknown();
let restored: ResumeState =
serde_json::from_slice(&serde_json::to_vec(&state).expect("serialize resume state")).expect("deserialize resume state");
assert!(restored.counter_unknown);
}
#[tokio::test]
async fn checkpoint_progress_survives_a_torn_resume_summary_write() {
let (_temp_dir, disk) = schema_test_disk().await;
let task_id = ResumeUtils::generate_task_id();
let _resume = ResumeManager::new(
disk.clone(),
task_id.clone(),
"erasure_set".to_string(),
"pool_0_set_0".to_string(),
vec!["bucket".to_string()],
)
.await
.expect("resume state should persist");
let checkpoint = CheckpointManager::new(disk.clone(), task_id.clone())
.await
.expect("checkpoint should persist");
// This is the ordering used by the erasure-set loop: the checkpoint is
// durable before the summary write. Stop here to model a crash in the
// inter-store window and verify that the recovery authority retains the
// telemetry fence and bytes.
checkpoint
.update_progress(3, 0, 0, 1024)
.await
.expect("checkpoint progress should persist");
checkpoint.mark_counter_unknown().await.expect("unknown fence should persist");
checkpoint
.update_position(0, 3)
.await
.expect("checkpoint position should persist");
let restored_checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task_id)
.await
.expect("checkpoint should reload")
.get_checkpoint()
.await;
let restored_resume = ResumeManager::load_from_disk(disk, &task_id)
.await
.expect("resume summary should reload")
.get_state()
.await;
assert!(restored_checkpoint.counter_unknown);
assert_eq!(restored_checkpoint.processed_bytes, 1024);
assert_eq!(restored_checkpoint.current_object_index, 3);
assert!(!restored_resume.counter_unknown, "summary is intentionally the torn/older store");
}
#[tokio::test]
async fn future_resume_and_checkpoint_schemas_are_rejected() {
let (temp_dir, disk) = schema_test_disk().await;
+2 -47
View File
@@ -22,7 +22,6 @@ use serde::{Deserialize, Serialize};
use std::sync::Arc;
use tracing::{debug, error, warn};
use super::progress::stable_generation;
use super::storage_api::owner::{EcstoreHealLifecycleExpiryContext, ecstore_load_admin_data_usage_from_backend_cached};
use super::storage_api::storage::{
BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _,
@@ -35,9 +34,6 @@ pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
pub struct HealBucketUsageBaseline {
pub objects_count: u64,
pub bytes: u64,
/// Stable identity of the validated usage snapshot and selected scope.
/// `None` is retained for test/legacy providers that cannot expose one.
pub generation: Option<u64>,
}
pub struct HealLifecycleExpiryContext {
@@ -789,52 +785,11 @@ impl HealStorageAPI for ECStoreHealStorage {
let mut baseline = HealBucketUsageBaseline::default();
for bucket in buckets {
if let Some(usage) = info.buckets_usage.get(bucket) {
baseline.objects_count = match baseline.objects_count.checked_add(usage.objects_count) {
Some(total) => total,
// A corrupt/overflowing usage snapshot is not a usable
// denominator. Leave progress indeterminate instead of
// turning saturation into a plausible percentage.
None => return Ok(None),
};
baseline.bytes = match baseline.bytes.checked_add(usage.size) {
Some(total) => total,
None => return Ok(None),
};
baseline.objects_count = baseline.objects_count.saturating_add(usage.objects_count);
baseline.bytes = baseline.bytes.saturating_add(usage.size);
}
}
let identity = info.snapshot_identity();
let mut canonical = Vec::new();
match identity.last_update {
Some(last_update) => {
canonical.push(1);
canonical.extend_from_slice(
&last_update
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos()
.to_be_bytes(),
);
}
None => canonical.push(0),
}
for value in [identity.scanner_cycle, identity.scanner_epoch] {
match value {
Some(value) => {
canonical.push(1);
canonical.extend_from_slice(&value.to_be_bytes());
}
None => canonical.push(0),
}
}
let mut scope = buckets.to_vec();
scope.sort_unstable();
for bucket in scope {
canonical.extend_from_slice(&(bucket.len() as u64).to_be_bytes());
canonical.extend_from_slice(bucket.as_bytes());
}
baseline.generation = Some(stable_generation(&[&canonical]));
Ok(Some(baseline))
}
+3 -7
View File
@@ -649,7 +649,7 @@ impl HealTask {
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("skipped: {bucket}/{object}")));
progress.update_stage(1, 1);
progress.update_progress(0, 1, 0, 0);
Ok(())
}
@@ -733,7 +733,7 @@ impl HealTask {
"Heal object skipped for data usage cache after transient error"
);
let mut progress = self.progress.write().await;
progress.update_stage(3, 3);
progress.update_progress(3, 3, 0, 0);
true
}
@@ -757,7 +757,7 @@ impl HealTask {
);
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("skipped: {bucket}/{object}")));
progress.update_stage(4, 4);
progress.update_progress(4, 4, 0, 0);
true
}
@@ -831,10 +831,6 @@ impl HealTask {
match &result {
Ok(_) => {
// A stage can reach its final step before the durable resume
// ledger and cleanup fences commit. Publish terminal 100 only
// after the enclosing operation has returned success.
self.progress.write().await.mark_completed();
let mut status = self.status.write().await;
*status = HealTaskStatus::Completed;
demote_to_debug_when!(self.heal_type.is_per_object(), info, target: "rustfs::heal::task", {
+16 -41
View File
@@ -13,7 +13,6 @@
// limitations under the License.
/// bucket/cluster/prefix heal: the recursive bucket-objects sweep and the erasure-set usage baseline
use super::*;
use crate::heal::progress::{add_bytes, increment_counter, stable_generation};
impl HealTask {
pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> {
@@ -33,7 +32,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("bucket: {bucket}")));
progress.update_stage(0, 3);
progress.update_progress(0, 3, 0, 0);
}
// Step 1: Check if bucket exists
@@ -67,7 +66,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_stage(1, 3);
progress.update_progress(1, 3, 0, 0);
}
// Step 2: Perform bucket heal using ecstore
@@ -123,7 +122,7 @@ impl HealTask {
if !self.options.recursive {
let mut progress = self.progress.write().await;
progress.update_stage(3, 3);
progress.update_progress(3, 3, 0, 0);
}
Ok(())
}
@@ -143,7 +142,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_stage(3, 3);
progress.update_progress(3, 3, 0, 0);
}
Err(Error::TaskExecutionFailed {
message: format!("Failed to heal bucket {bucket}: {e}"),
@@ -246,7 +245,6 @@ impl HealTask {
let mut scanned = 0u64;
let mut healed = 0u64;
let mut failed = 0u64;
let mut skipped = 0u64;
let mut retryable_failed = 0u64;
let mut permanent_failed = 0u64;
let mut bytes = 0u64;
@@ -288,14 +286,16 @@ impl HealTask {
let mut retry = Vec::with_capacity(pending.len());
for item in pending {
self.check_control_flags().await?;
let mut telemetry_unknown = false;
let object = item.name.as_str();
if retry_attempt == 0 {
scanned = scanned.saturating_add(1);
}
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("{bucket}/{object}")));
progress.update_progress(scanned, healed, failed, bytes);
}
let mut terminal_outcome = true;
let error = match self
.await_with_control(
self.storage
@@ -304,13 +304,13 @@ impl HealTask {
.await
{
Ok((result, None)) => {
telemetry_unknown |= !increment_counter(&mut healed);
telemetry_unknown |= !add_bytes(&mut bytes, u64::try_from(result.object_size).unwrap_or(u64::MAX));
healed = healed.saturating_add(1);
bytes = bytes.saturating_add(u64::try_from(result.object_size).unwrap_or_default());
self.record_result_item(result).await;
None
}
Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => {
telemetry_unknown |= !increment_counter(&mut healed);
healed = healed.saturating_add(1);
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
@@ -329,7 +329,6 @@ impl HealTask {
if let Some(err) = error {
if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
telemetry_unknown |= !increment_counter(&mut skipped);
warn!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
@@ -343,7 +342,6 @@ impl HealTask {
"Heal bucket object repair skipped due to transient metadata error"
);
} else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
terminal_outcome = false;
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_BUCKET_RESULT,
@@ -359,7 +357,7 @@ impl HealTask {
);
retry.push(item);
} else {
telemetry_unknown |= !increment_counter(&mut failed);
failed = failed.saturating_add(1);
if err.is_recoverable_heal() {
retryable_failed = retryable_failed.saturating_add(1);
} else {
@@ -385,19 +383,8 @@ impl HealTask {
}
}
if terminal_outcome {
telemetry_unknown |= !increment_counter(&mut scanned);
}
if !terminal_outcome {
continue;
}
let mut progress = self.progress.write().await;
progress.update_object_progress(scanned, healed, failed, skipped, bytes);
if telemetry_unknown {
progress.mark_unknown();
}
progress.update_progress(scanned, healed, failed, bytes);
}
pending = retry;
retry_attempt = retry_attempt.saturating_add(1);
@@ -444,10 +431,7 @@ impl HealTask {
Ok(())
}
pub(super) async fn apply_erasure_set_usage_baseline(&self, buckets: &[String], _set_disk_id: &str) -> Result<()> {
if matches!(self.options.scan_mode, HealScanMode::Deep) || matches!(self.source, HealRequestSource::AutoHeal) {
return Ok(());
}
pub(super) async fn apply_erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<()> {
let baseline = match self
.await_with_control(self.storage.erasure_set_usage_baseline(buckets))
.await
@@ -458,18 +442,9 @@ impl HealTask {
Err(_) => return Ok(()),
};
let HealBucketUsageBaseline {
objects_count,
bytes,
generation,
} = baseline;
let generation = generation.map(|snapshot_generation| stable_generation(&[&snapshot_generation.to_be_bytes()]));
let HealBucketUsageBaseline { objects_count, bytes } = baseline;
let mut progress = self.progress.write().await;
if let Some(generation) = generation {
progress.set_total_baseline_with_generation(objects_count, bytes, generation);
} else {
progress.set_total_baseline(objects_count, bytes);
}
progress.set_total_baseline(objects_count, bytes);
Ok(())
}
}
+10 -8
View File
@@ -32,7 +32,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("erasure_set: {} ({} buckets)", set_disk_id, buckets.len())));
progress.update_stage(0, 4);
progress.update_progress(0, 4, 0, 0);
}
let is_auto_replacement = matches!(self.source, HealRequestSource::AutoHeal) && !self.heal_endpoints.is_empty();
@@ -158,7 +158,7 @@ impl HealTask {
None
};
self.apply_erasure_set_usage_baseline(&buckets, &set_disk_id).await?;
self.apply_erasure_set_usage_baseline(&buckets).await?;
let healing_marker = format!("{set_disk_id}:{}", self.id);
if let Some((disk, resume_manager, _)) = replacement_resume.as_ref() {
@@ -248,7 +248,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_stage(4, 4);
progress.update_progress(4, 4, 0, 0);
}
return Err(Error::TaskExecutionFailed {
message: format!("Failed to heal disk format for {set_disk_id}: {error}"),
@@ -304,7 +304,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_stage(4, 4);
progress.update_progress(4, 4, 0, 0);
}
return Err(Error::TaskExecutionFailed {
message: format!("Failed to heal disk format for {set_disk_id}: {e}"),
@@ -314,7 +314,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_stage(1, 4);
progress.update_progress(1, 4, 0, 0);
}
// The rebuilt disks are formatted now: mark them as healing so
@@ -343,7 +343,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_stage(2, 4);
progress.update_progress(2, 4, 0, 0);
}
// Step 3: Heal bucket structure
@@ -427,7 +427,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_stage(3, 4);
progress.update_progress(3, 4, 0, 0);
}
// Step 4: Execute erasure set heal with resume
@@ -470,7 +470,9 @@ impl HealTask {
};
{
self.progress.write().await.update_stage(4, 4);
let mut progress = self.progress.write().await;
let bytes_processed = progress.bytes_processed;
progress.update_progress(4, 4, 0, bytes_processed);
}
match result {
+10 -10
View File
@@ -32,7 +32,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("metadata: {bucket}/{object}")));
progress.update_stage(0, 3);
progress.update_progress(0, 3, 0, 0);
}
// Step 1: Check if object exists
@@ -74,7 +74,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_stage(1, 3);
progress.update_progress(1, 3, 0, 0);
}
// Step 2: Perform metadata heal using ecstore
@@ -122,7 +122,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_stage(3, 3);
progress.update_progress(3, 3, 0, 0);
}
return Err(Error::TaskExecutionFailed {
message: format!("Failed to heal metadata {bucket}/{object}: {e}"),
@@ -145,7 +145,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_stage(3, 3);
progress.update_progress(3, 3, 0, 0);
}
self.record_result_item(result).await;
Ok(())
@@ -167,7 +167,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_stage(3, 3);
progress.update_progress(3, 3, 0, 0);
}
Err(Error::TaskExecutionFailed {
message: format!("Failed to heal metadata {bucket}/{object}: {e}"),
@@ -194,7 +194,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("ec_decode: {bucket}/{object}")));
progress.update_stage(0, 3);
progress.update_progress(0, 3, 0, 0);
}
// Step 1: Check if object exists
@@ -236,7 +236,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_stage(1, 3);
progress.update_progress(1, 3, 0, 0);
}
// Step 2: Perform EC decode heal using ecstore
@@ -284,7 +284,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_stage(3, 3);
progress.update_progress(3, 3, 0, 0);
}
return Err(Error::TaskExecutionFailed {
message: format!("Failed to heal EC decode {bucket}/{object}: {e}"),
@@ -309,7 +309,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_object_progress(1, 1, 0, 0, object_size);
progress.update_progress(3, 3, 0, object_size);
}
self.record_result_item(result).await;
Ok(())
@@ -331,7 +331,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_stage(3, 3);
progress.update_progress(3, 3, 0, 0);
}
Err(Error::TaskExecutionFailed {
message: format!("Failed to heal EC decode {bucket}/{object}: {e}"),
+8 -8
View File
@@ -36,7 +36,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.set_current_object(Some(format!("{bucket}/{object}")));
progress.update_stage(0, 4);
progress.update_progress(0, 4, 0, 0);
}
// Step 1: Check if object exists and get metadata
@@ -132,7 +132,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_stage(1, 3);
progress.update_progress(1, 3, 0, 0);
}
// Step 2: directly call ecstore to perform heal
@@ -187,7 +187,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_stage(3, 3);
progress.update_progress(3, 3, 0, 0);
}
return Ok(());
}
@@ -207,7 +207,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_stage(3, 3);
progress.update_progress(3, 3, 0, 0);
}
if Self::should_return_typed_heal_error(&e) {
@@ -249,7 +249,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_object_progress(1, 1, 0, 0, object_size);
progress.update_progress(3, 3, 0, object_size);
}
self.record_result_item(result).await;
Ok(())
@@ -275,7 +275,7 @@ impl HealTask {
);
{
let mut progress = self.progress.write().await;
progress.update_stage(3, 3);
progress.update_progress(3, 3, 0, 0);
}
return Ok(());
}
@@ -295,7 +295,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_stage(3, 3);
progress.update_progress(3, 3, 0, 0);
}
if Self::should_return_typed_heal_error(&e) {
@@ -414,7 +414,7 @@ impl HealTask {
{
let mut progress = self.progress.write().await;
progress.update_object_progress(1, 1, 0, 0, object_size);
progress.update_progress(4, 4, 0, object_size);
}
self.record_result_item(result).await;
Ok(())
-47
View File
@@ -22,7 +22,6 @@ use std::sync::Mutex;
use tempfile::TempDir;
use super::super::storage_api::status::BucketInfo;
use crate::heal::progress::HealProgressState;
#[tokio::test]
async fn retry_request_carries_remaining_timeout_budget() {
@@ -2125,7 +2124,6 @@ async fn erasure_set_heal_applies_usage_baseline_to_progress() {
usage_baseline: Mutex::new(Some(HealBucketUsageBaseline {
objects_count: 10,
bytes: 8,
generation: Some(1),
})),
..Default::default()
});
@@ -2149,55 +2147,10 @@ async fn erasure_set_heal_applies_usage_baseline_to_progress() {
let progress = task.get_progress().await;
assert_eq!(progress.objects_total_count, 10);
assert_eq!(progress.objects_total_size, 8);
assert!(progress.baseline_generation.is_some());
assert!(progress.baseline_known);
assert_eq!(progress.bytes_processed, 2);
assert!((progress.progress_percentage - 25.0).abs() < 0.001);
}
#[tokio::test]
async fn erasure_set_disk_walk_keeps_cluster_usage_baseline_indeterminate() {
for (scan_mode, source) in [
(HealScanMode::Deep, HealRequestSource::Admin),
(HealScanMode::Normal, HealRequestSource::AutoHeal),
] {
let temp = TempDir::new().expect("temporary directory should be created");
let disk = make_resume_disk(&temp).await;
let storage = Arc::new(MockStorage {
resume_disk: Mutex::new(Some(disk)),
usage_baseline: Mutex::new(Some(HealBucketUsageBaseline {
objects_count: 10,
bytes: 8,
generation: Some(1),
})),
..Default::default()
});
let mut request = HealRequest::new(
HealType::ErasureSet {
buckets: vec!["bucket-a".to_string()],
set_disk_id: "pool_0_set_0".to_string(),
},
HealOptions {
scan_mode,
timeout: None,
..Default::default()
},
HealPriority::Normal,
);
request.source = source;
let task = HealTask::from_request(request, storage);
task.heal_erasure_set(vec!["bucket-a".to_string()], "pool_0_set_0".to_string())
.await
.expect("erasure set heal should complete");
let progress = task.get_progress().await;
assert!(!progress.baseline_known);
assert_eq!(progress.baseline_generation, None);
assert_eq!(progress.progress_state, HealProgressState::Indeterminate);
}
}
#[tokio::test]
async fn erasure_set_heal_ignores_usage_baseline_errors() {
let temp = TempDir::new().expect("temporary directory should be created");
+1 -1
View File
@@ -19,7 +19,7 @@ pub use error::{Error, Result};
pub use heal::{
HealManager, HealOperationsSnapshot, HealOptions, HealPriority, HealPriorityCounts, HealRequest, HealSourceCounts, HealType,
channel::HealChannelProcessor,
progress::{HealProgress, aggregate_heal_progress},
progress::HealProgress,
resume::{ReplacementRecoveryRecord, ReplacementRecoveryState, ResumeUtils},
};
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
@@ -1215,10 +1215,7 @@ pub struct ScannerActivityResponse {
pub dirty_usage_pending: bool,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct BackgroundHealStatusRequest {
#[prost(uint32, tag = "1")]
pub protocol_version: u32,
}
pub struct BackgroundHealStatusRequest {}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct BackgroundHealStatusResponse {
#[prost(bool, tag = "1")]
-19
View File
@@ -170,7 +170,6 @@ pub fn internode_rpc_max_message_size() -> usize {
pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SIZE + 1024;
pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 3;
pub const DYNAMIC_CONFIG_PROTOCOL_VERSION: u32 = 1;
pub const BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION: u32 = 2;
pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v3\0";
pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-remote-version-state-capability-v1\0";
pub const CROSS_POOL_FENCE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-cross-pool-fence-capability-v1\0";
@@ -2347,28 +2346,10 @@ pub async fn evict_failed_connection_with_log_level(addr: &str, log_level: Conne
#[cfg(test)]
mod tests {
use super::*;
use prost::Message as _;
use std::sync::Mutex;
static INTERNODE_RPC_MSGPACK_ONLY_ENV_LOCK: Mutex<()> = Mutex::new(());
#[derive(Clone, PartialEq, prost::Message)]
struct BackgroundHealStatusRequestV1 {}
#[test]
fn background_heal_status_request_remains_rolling_upgrade_compatible() {
let current = proto_gen::node_service::BackgroundHealStatusRequest {
protocol_version: BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION,
};
let encoded = current.encode_to_vec();
BackgroundHealStatusRequestV1::decode(encoded.as_slice()).expect("v1 server should ignore the version field");
let encoded = BackgroundHealStatusRequestV1 {}.encode_to_vec();
let decoded = proto_gen::node_service::BackgroundHealStatusRequest::decode(encoded.as_slice())
.expect("v2 server should accept a v1 request");
assert_eq!(decoded.protocol_version, 0);
}
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
struct CompatPayloadField {
message: &'static str,
+1 -3
View File
@@ -846,9 +846,7 @@ message ScannerActivityResponse {
bool dirty_usage_pending = 9;
}
message BackgroundHealStatusRequest {
uint32 protocol_version = 1;
}
message BackgroundHealStatusRequest {}
message BackgroundHealStatusResponse {
bool success = 1;
+26 -46
View File
@@ -20,7 +20,7 @@ use crate::admin::storage_api::bucket::utils::is_valid_object_prefix;
use crate::server::ADMIN_PREFIX;
use crate::server::RemoteAddr;
use crate::storage::rpc::node_service::heal::{
HealControlCoordinator, NodeHealStatusSnapshot, capture_node_heal_status, decode_node_heal_status,
HealControlCoordinator, NodeHealProgress, NodeHealStatusSnapshot, capture_node_heal_status, decode_node_heal_status,
decode_node_replacement_recovery_status, heal_control_coordinator, heal_topology_fingerprint,
};
use bytes::Bytes;
@@ -298,7 +298,14 @@ fn background_heal_runtime_state(
}
}
type BackgroundHealProgress = rustfs_heal::HealProgress;
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct BackgroundHealProgress {
objects_scanned: u64,
objects_healed: u64,
objects_failed: u64,
bytes_processed: u64,
}
#[derive(Debug)]
struct ClusterHealStatusSnapshot {
@@ -337,10 +344,17 @@ fn add_operations(total: &mut rustfs_heal::HealOperationsSnapshot, next: rustfs_
add_source_counts(&mut total.retrying_by_source, next.retrying_by_source);
}
fn add_progress(total: &mut BackgroundHealProgress, next: NodeHealProgress) {
total.objects_scanned = total.objects_scanned.saturating_add(next.objects_scanned);
total.objects_healed = total.objects_healed.saturating_add(next.objects_healed);
total.objects_failed = total.objects_failed.saturating_add(next.objects_failed);
total.bytes_processed = total.bytes_processed.saturating_add(next.bytes_processed);
}
fn aggregate_cluster_heal_status(snapshots: Vec<NodeHealStatusSnapshot>) -> ClusterHealStatusSnapshot {
let mut info = BackgroundHealInfo::default();
let mut operations = rustfs_heal::HealOperationsSnapshot::default();
let mut progress = Vec::new();
let mut progress = None;
let mut any_services_enabled = false;
let mut any_initialized = false;
@@ -357,12 +371,18 @@ fn aggregate_cluster_heal_status(snapshots: Vec<NodeHealStatusSnapshot>) -> Clus
}
add_operations(&mut operations, snapshot.operations);
if let Some(next) = snapshot.progress {
progress.push(next);
add_progress(
progress.get_or_insert(BackgroundHealProgress {
objects_scanned: 0,
objects_healed: 0,
objects_failed: 0,
bytes_processed: 0,
}),
next,
);
}
}
let progress = rustfs_heal::aggregate_heal_progress(progress);
let state = if operations.queue_length > 0 || operations.active_tasks > 0 || operations.retrying_tasks > 0 {
HealRuntimeState::Active
} else if any_initialized {
@@ -2181,19 +2201,10 @@ mod tests {
};
let progress = BackgroundHealProgress {
kind: rustfs_heal::heal::progress::HealProgressKind::ObjectSweep,
objects_scanned: 7,
objects_healed: 3,
objects_failed: 1,
skipped_objects: 3,
objects_total_count: 10,
objects_total_size: 8192,
bytes_processed: 4096,
progress_percentage: 50.0,
progress_state: rustfs_heal::heal::progress::HealProgressState::Running,
baseline_generation: Some(42),
baseline_known: true,
..Default::default()
};
let encoded = encode_background_heal_status(
@@ -2209,14 +2220,7 @@ mod tests {
assert_eq!(json["progress"]["objectsScanned"], 7);
assert_eq!(json["progress"]["objectsHealed"], 3);
assert_eq!(json["progress"]["objectsFailed"], 1);
assert_eq!(json["progress"]["skippedObjects"], 3);
assert_eq!(json["progress"]["objectsTotalCount"], 10);
assert_eq!(json["progress"]["objectsTotalSize"], 8192);
assert_eq!(json["progress"]["bytesProcessed"], 4096);
assert_eq!(json["progress"]["progressState"], "running");
assert_eq!(json["progress"]["baselineGeneration"], 42);
assert_eq!(json["progress"]["baselineKnown"], true);
assert_eq!(json["progress"]["counterUnknown"], false);
}
#[test]
@@ -2238,18 +2242,10 @@ mod tests {
..Default::default()
},
Some(NodeHealProgress {
kind: rustfs_heal::heal::progress::HealProgressKind::ObjectSweep,
objects_scanned: 3,
objects_healed: 1,
objects_failed: 0,
skipped_objects: 2,
objects_total_count: 6,
objects_total_size: 400,
bytes_processed: 100,
progress_state: rustfs_heal::heal::progress::HealProgressState::Running,
baseline_generation: Some(9),
baseline_known: true,
..Default::default()
}),
);
let peer = NodeHealStatusSnapshot::for_test(
@@ -2269,17 +2265,10 @@ mod tests {
..Default::default()
},
Some(NodeHealProgress {
kind: rustfs_heal::heal::progress::HealProgressKind::ObjectSweep,
objects_scanned: 5,
objects_healed: 4,
objects_failed: 1,
objects_total_count: 4,
objects_total_size: 1600,
bytes_processed: 900,
progress_state: rustfs_heal::heal::progress::HealProgressState::Running,
baseline_generation: Some(9),
baseline_known: true,
..Default::default()
}),
);
@@ -2298,15 +2287,7 @@ mod tests {
assert_eq!(progress.objects_scanned, 8);
assert_eq!(progress.objects_healed, 5);
assert_eq!(progress.objects_failed, 1);
assert_eq!(progress.skipped_objects, 2);
assert_eq!(progress.objects_total_count, 10);
assert_eq!(progress.objects_total_size, 2000);
assert_eq!(progress.bytes_processed, 1000);
assert_eq!(progress.progress_percentage, 50.0);
assert_eq!(progress.progress_state, rustfs_heal::heal::progress::HealProgressState::Running);
assert_eq!(progress.baseline_generation, Some(9));
assert!(progress.baseline_known);
assert!(!progress.counter_unknown);
assert_eq!(peer_first.state, HealRuntimeState::Active);
assert_eq!(peer_first.operations, local_first.operations);
@@ -2346,7 +2327,6 @@ mod tests {
objects_healed: value,
objects_failed: value,
bytes_processed: value,
..Default::default()
};
let saturated = NodeHealStatusSnapshot::for_test(
true,
+5 -4
View File
@@ -554,10 +554,11 @@ fn apply_replication_timestamps_from_headers(headers: &HeaderMap<HeaderValue>, o
// Persist into the internal metadata keys so a later outbound replication
// pass (replication_target_boundary) reads the source's modification
// times instead of falling back to mod_time.
// TODO(P1-6): receiver-side LWW is still missing — when the stored
// per-category timestamp is newer than the inbound one, the existing
// tags/retention/legal-hold should win instead of being overwritten.
// times instead of falling back to mod_time. Receiver-side LWW happens at
// the set layer under the object write lock
// (ecstore set_disk::ops::object::merge_replication_metadata_lww,
// rustfs/backlog#1953): a category whose stored timestamp is newer than
// the inbound one keeps the local values.
for (timestamp, suffix) in [
(opts.replication_tagging_timestamp, SUFFIX_TAGGING_TIMESTAMP),
(opts.replication_retention_timestamp, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP),
+2 -2
View File
@@ -1912,7 +1912,7 @@ impl Node for NodeService {
async fn background_heal_status(
&self,
request: Request<BackgroundHealStatusRequest>,
_request: Request<BackgroundHealStatusRequest>,
) -> Result<Response<BackgroundHealStatusResponse>, Status> {
if self.resolve_object_store().is_none() {
return Ok(Response::new(BackgroundHealStatusResponse {
@@ -1922,7 +1922,7 @@ impl Node for NodeService {
}));
}
let snapshot = heal::capture_node_heal_status(rustfs_scanner::scanner::BackgroundHealInfo::default()).await;
match heal::encode_node_heal_status(&snapshot, request.into_inner().protocol_version) {
match heal::encode_node_heal_status(&snapshot) {
Ok(bg_heal_state) => Ok(Response::new(BackgroundHealStatusResponse {
success: true,
bg_heal_state: bg_heal_state.into(),
+29 -174
View File
@@ -25,8 +25,7 @@ use std::io::Cursor;
use super::super::encode_msgpack_map;
const NODE_HEAL_STATUS_PREVIOUS_VERSION: u8 = 1;
const NODE_HEAL_STATUS_VERSION: u8 = 2;
const NODE_HEAL_STATUS_VERSION: u8 = 1;
const NODE_HEAL_STATUS_MAX_SIZE: usize = 64 * 1024;
const NODE_REPLACEMENT_RECOVERY_STATUS_VERSION: u8 = 1;
const NODE_REPLACEMENT_RECOVERY_STATUS_MAX_SIZE: usize = 64 * 1024;
@@ -173,38 +172,13 @@ pub(crate) fn heal_topology_fingerprint(endpoint_pools: &EndpointServerPools) ->
Ok(hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower))
}
pub(crate) type NodeHealProgress = rustfs_heal::HealProgress;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct NodeHealProgressV1 {
objects_scanned: u64,
objects_healed: u64,
objects_failed: u64,
bytes_processed: u64,
}
impl From<&NodeHealProgress> for NodeHealProgressV1 {
fn from(progress: &NodeHealProgress) -> Self {
Self {
objects_scanned: progress.objects_scanned,
objects_healed: progress.objects_healed,
objects_failed: progress.objects_failed,
bytes_processed: progress.bytes_processed,
}
}
}
impl From<NodeHealProgressV1> for NodeHealProgress {
fn from(progress: NodeHealProgressV1) -> Self {
Self {
objects_scanned: progress.objects_scanned,
objects_healed: progress.objects_healed,
objects_failed: progress.objects_failed,
bytes_processed: progress.bytes_processed,
..Default::default()
}
}
pub(crate) struct NodeHealProgress {
pub objects_scanned: u64,
pub objects_healed: u64,
pub objects_failed: u64,
pub bytes_processed: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -246,48 +220,6 @@ pub(crate) struct NodeHealStatusSnapshot {
pub progress: Option<NodeHealProgress>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct NodeHealStatusSnapshotV1 {
version: u8,
services_enabled: bool,
initialized: bool,
info: NodeHealInfo,
operations: HealOperationsSnapshot,
progress: Option<NodeHealProgressV1>,
}
impl From<&NodeHealStatusSnapshot> for NodeHealStatusSnapshotV1 {
fn from(snapshot: &NodeHealStatusSnapshot) -> Self {
Self {
version: NODE_HEAL_STATUS_PREVIOUS_VERSION,
services_enabled: snapshot.services_enabled,
initialized: snapshot.initialized,
info: snapshot.info.clone(),
operations: snapshot.operations,
progress: snapshot.progress.as_ref().map(NodeHealProgressV1::from),
}
}
}
impl From<NodeHealStatusSnapshotV1> for NodeHealStatusSnapshot {
fn from(snapshot: NodeHealStatusSnapshotV1) -> Self {
Self {
version: snapshot.version,
services_enabled: snapshot.services_enabled,
initialized: snapshot.initialized,
info: snapshot.info,
operations: snapshot.operations,
progress: snapshot.progress.map(NodeHealProgress::from),
}
}
}
#[derive(Deserialize)]
struct NodeHealStatusVersion {
version: u8,
}
impl NodeHealStatusSnapshot {
#[cfg(test)]
pub(crate) fn for_test(
@@ -317,7 +249,14 @@ impl NodeHealStatusSnapshot {
}
pub(crate) async fn capture_node_heal_status(info: BackgroundHealInfo) -> NodeHealStatusSnapshot {
let progress = rustfs_heal::current_heal_progress_snapshot().await;
let progress = rustfs_heal::current_heal_progress_snapshot()
.await
.map(|progress| NodeHealProgress {
objects_scanned: progress.objects_scanned,
objects_healed: progress.objects_healed,
objects_failed: progress.objects_failed,
bytes_processed: progress.bytes_processed,
});
NodeHealStatusSnapshot {
version: NODE_HEAL_STATUS_VERSION,
@@ -329,44 +268,23 @@ pub(crate) async fn capture_node_heal_status(info: BackgroundHealInfo) -> NodeHe
}
}
pub(crate) fn encode_node_heal_status(snapshot: &NodeHealStatusSnapshot, protocol_version: u32) -> Result<Vec<u8>, String> {
let encoded = if protocol_version < rustfs_protos::BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION {
encode_msgpack_map(&NodeHealStatusSnapshotV1::from(snapshot))
} else {
let mut snapshot = snapshot.clone();
snapshot.version = NODE_HEAL_STATUS_VERSION;
encode_msgpack_map(&snapshot)
};
encoded.map_err(|err| format!("failed to encode node heal status: {err}"))
pub(crate) fn encode_node_heal_status(snapshot: &NodeHealStatusSnapshot) -> Result<Vec<u8>, String> {
encode_msgpack_map(snapshot).map_err(|err| format!("failed to encode node heal status: {err}"))
}
pub(crate) fn decode_node_heal_status(data: &[u8]) -> Result<NodeHealStatusSnapshot, String> {
if data.len() > NODE_HEAL_STATUS_MAX_SIZE {
return Err("node heal status exceeds size limit".to_string());
}
let decode_version = || {
let mut deserializer = Deserializer::new(Cursor::new(data));
NodeHealStatusVersion::deserialize(&mut deserializer)
.map(|version| (version, deserializer))
.map_err(|err| format!("failed to decode node heal status: {err}"))
};
let (version, deserializer) = decode_version()?;
if usize::try_from(deserializer.get_ref().position()).ok() != Some(data.len()) {
return Err("node heal status contains trailing data".to_string());
}
let mut deserializer = Deserializer::new(Cursor::new(data));
let snapshot = match version.version {
NODE_HEAL_STATUS_PREVIOUS_VERSION => {
NodeHealStatusSnapshotV1::deserialize(&mut deserializer).map(NodeHealStatusSnapshot::from)
}
NODE_HEAL_STATUS_VERSION => NodeHealStatusSnapshot::deserialize(&mut deserializer),
version => return Err(format!("unsupported node heal status version: {version}")),
}
.map_err(|err| format!("failed to decode node heal status: {err}"))?;
let snapshot = NodeHealStatusSnapshot::deserialize(&mut deserializer)
.map_err(|err| format!("failed to decode node heal status: {err}"))?;
if usize::try_from(deserializer.get_ref().position()).ok() != Some(data.len()) {
return Err("node heal status contains trailing data".to_string());
}
if snapshot.version != NODE_HEAL_STATUS_VERSION {
return Err(format!("unsupported node heal status version: {}", snapshot.version));
}
Ok(snapshot)
}
@@ -469,10 +387,9 @@ pub(crate) fn decode_node_replacement_recovery_status(data: &[u8]) -> Result<Nod
#[cfg(test)]
mod tests {
use super::{
NODE_HEAL_STATUS_MAX_SIZE, NODE_HEAL_STATUS_PREVIOUS_VERSION, NODE_HEAL_STATUS_VERSION, NodeHealProgress,
NodeHealStatusSnapshot, NodeReplacementRecoveryStatusSnapshot, decode_node_heal_status,
decode_node_replacement_recovery_status, encode_node_heal_status, encode_node_replacement_recovery_status,
heal_control_coordinator, heal_topology_fingerprint,
NODE_HEAL_STATUS_MAX_SIZE, NODE_HEAL_STATUS_VERSION, NodeHealProgress, NodeHealStatusSnapshot,
NodeReplacementRecoveryStatusSnapshot, decode_node_heal_status, decode_node_replacement_recovery_status,
encode_node_heal_status, encode_node_replacement_recovery_status, heal_control_coordinator, heal_topology_fingerprint,
};
use crate::storage::storage_api::{
Endpoint,
@@ -616,72 +533,20 @@ mod tests {
..Default::default()
},
Some(NodeHealProgress {
kind: rustfs_heal::heal::progress::HealProgressKind::ObjectSweep,
objects_scanned: 7,
objects_healed: 5,
objects_failed: 1,
skipped_objects: 1,
objects_total_count: 10,
objects_total_size: 2048,
bytes_processed: 1024,
progress_percentage: 50.0,
progress_state: rustfs_heal::heal::progress::HealProgressState::Running,
baseline_generation: Some(42),
baseline_known: true,
..Default::default()
}),
);
let encoded = encode_node_heal_status(&snapshot, rustfs_protos::BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION)
.expect("snapshot should encode");
let encoded = encode_node_heal_status(&snapshot).expect("snapshot should encode");
let decoded = decode_node_heal_status(&encoded).expect("snapshot should decode");
assert_eq!(decoded.version, NODE_HEAL_STATUS_VERSION);
assert_eq!(decoded.operations.queue_length, 2);
assert_eq!(decoded.progress, snapshot.progress);
}
#[test]
fn node_heal_status_v1_encoding_preserves_rolling_compatibility() {
let snapshot = NodeHealStatusSnapshot::for_test(
true,
true,
BackgroundHealInfo::default(),
HealOperationsSnapshot::default(),
Some(NodeHealProgress {
objects_scanned: 7,
objects_healed: 5,
objects_failed: 1,
skipped_objects: 3,
bytes_processed: 1024,
progress_state: rustfs_heal::heal::progress::HealProgressState::Running,
baseline_known: true,
..Default::default()
}),
);
let encoded =
encode_node_heal_status(&snapshot, u32::from(NODE_HEAL_STATUS_PREVIOUS_VERSION)).expect("v1 snapshot should encode");
let wire: serde_json::Value = rmp_serde::from_slice(&encoded).expect("v1 snapshot should decode as JSON");
let progress = wire["progress"].as_object().expect("v1 progress should be a map");
assert_eq!(wire["version"], NODE_HEAL_STATUS_PREVIOUS_VERSION);
assert_eq!(progress.len(), 4);
assert_eq!(progress["objectsScanned"], 7);
assert!(!progress.contains_key("skippedObjects"));
let decoded = decode_node_heal_status(&encoded).expect("v1 snapshot should decode");
let progress = decoded.progress.as_ref().expect("v1 progress should be present");
assert_eq!(decoded.version, NODE_HEAL_STATUS_PREVIOUS_VERSION);
assert_eq!(progress.objects_scanned, 7);
assert_eq!(progress.progress_state, rustfs_heal::heal::progress::HealProgressState::Unknown);
assert!(!progress.baseline_known);
let upgraded = encode_node_heal_status(&decoded, rustfs_protos::BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION)
.expect("decoded v1 snapshot should upgrade to v2");
let upgraded = decode_node_heal_status(&upgraded).expect("upgraded snapshot should decode");
assert_eq!(upgraded.version, NODE_HEAL_STATUS_VERSION);
assert_eq!(upgraded.progress.as_ref(), Some(progress));
}
#[test]
fn node_heal_status_rejects_unknown_version() {
let mut snapshot = NodeHealStatusSnapshot::for_test(
@@ -693,7 +558,7 @@ mod tests {
);
snapshot.version += 1;
let encoded = rmp_serde::to_vec_named(&snapshot).expect("snapshot should encode");
let encoded = encode_node_heal_status(&snapshot).expect("snapshot should encode");
let err = decode_node_heal_status(&encoded).expect_err("unknown version should fail closed");
assert!(err.contains("unsupported node heal status version"));
}
@@ -714,22 +579,13 @@ mod tests {
"activeBySource": {"scanner": 0, "admin": 1, "autoHeal": 0, "internal": 0, "readRepair": 0},
"retryingBySource": {"scanner": 0, "admin": 0, "autoHeal": 0, "internal": 0, "readRepair": 0}
},
"progress": {
"objectsScanned": 7,
"objectsHealed": 5,
"objectsFailed": 1,
"bytesProcessed": 1024
}
"progress": null
});
let encoded = rmp_serde::to_vec_named(&fixture).expect("fixture should encode");
let decoded = decode_node_heal_status(&encoded).expect("fixed v1 fixture should decode");
assert_eq!(decoded.info().bitrot_start_cycle, 9);
assert_eq!(decoded.operations.queue_length, 2);
assert_eq!(decoded.operations.queued_by_source.mrf, 0);
let progress = decoded.progress.expect("legacy progress should decode");
assert_eq!(progress.objects_scanned, 7);
assert!(!progress.baseline_known);
assert_eq!(progress.progress_state, rustfs_heal::heal::progress::HealProgressState::Unknown);
}
#[test]
@@ -775,8 +631,7 @@ mod tests {
HealOperationsSnapshot::default(),
None,
);
let encoded = encode_node_heal_status(&snapshot, rustfs_protos::BACKGROUND_HEAL_STATUS_PROTOCOL_VERSION)
.expect("snapshot should encode");
let encoded = encode_node_heal_status(&snapshot).expect("snapshot should encode");
let encoded_json: serde_json::Value = rmp_serde::from_slice(&encoded).expect("encoded snapshot should decode as JSON");
assert_eq!(encoded_json["info"]["bitrotStartTime"], serde_json::json!("2023-11-14T22:13:20.123456Z"));
}