mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 04:16:38 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| da3fd83aa7 | |||
| a2e7036cb1 |
@@ -585,9 +585,12 @@ impl VersionsHistogram {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replication statistics for a single target
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||
pub struct ReplicationStats {
|
||||
/// Replication statistics for a single target.
|
||||
///
|
||||
/// Renamed from `ReplicationStats`; serde field names are preserved
|
||||
/// byte-identically to maintain wire compatibility with existing snapshots.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ReplicationTargetUsage {
|
||||
pub pending_size: u64,
|
||||
pub replicated_size: u64,
|
||||
pub failed_size: u64,
|
||||
@@ -600,7 +603,7 @@ pub struct ReplicationStats {
|
||||
pub replicated_count: u64,
|
||||
}
|
||||
|
||||
impl ReplicationStats {
|
||||
impl ReplicationTargetUsage {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
let Self {
|
||||
pending_size,
|
||||
@@ -636,7 +639,7 @@ impl ReplicationStats {
|
||||
/// Replication statistics for all targets
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||
pub struct ReplicationAllStats {
|
||||
pub targets: HashMap<String, ReplicationStats>,
|
||||
pub targets: HashMap<String, ReplicationTargetUsage>,
|
||||
pub replica_size: u64,
|
||||
pub replica_count: u64,
|
||||
}
|
||||
@@ -649,7 +652,7 @@ impl ReplicationAllStats {
|
||||
targets,
|
||||
} = self;
|
||||
|
||||
*replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationStats::is_empty)
|
||||
*replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationTargetUsage::is_empty)
|
||||
}
|
||||
|
||||
#[deprecated(note = "use is_empty instead")]
|
||||
@@ -2466,7 +2469,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn replication_stats_empty_checks_every_field() {
|
||||
type SetField = fn(&mut ReplicationStats);
|
||||
type SetField = fn(&mut ReplicationTargetUsage);
|
||||
|
||||
let cases: [(&str, SetField); 10] = [
|
||||
("pending_size", |stats| stats.pending_size = 1),
|
||||
@@ -2481,9 +2484,9 @@ mod tests {
|
||||
("replicated_count", |stats| stats.replicated_count = 1),
|
||||
];
|
||||
|
||||
assert!(ReplicationStats::default().is_empty());
|
||||
assert!(ReplicationTargetUsage::default().is_empty());
|
||||
for (field, set_nonzero) in cases {
|
||||
let mut stats = ReplicationStats::default();
|
||||
let mut stats = ReplicationTargetUsage::default();
|
||||
set_nonzero(&mut stats);
|
||||
assert!(!stats.is_empty(), "{field} must make replication stats non-empty");
|
||||
}
|
||||
@@ -2514,17 +2517,17 @@ mod tests {
|
||||
}
|
||||
|
||||
let empty_targets = ReplicationAllStats {
|
||||
targets: HashMap::from([("arn:test:empty".to_string(), ReplicationStats::default())]),
|
||||
targets: HashMap::from([("arn:test:empty".to_string(), ReplicationTargetUsage::default())]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(empty_targets.is_empty(), "all-empty targets must keep aggregate stats empty");
|
||||
|
||||
let stats = ReplicationAllStats {
|
||||
targets: HashMap::from([
|
||||
("arn:test:empty".to_string(), ReplicationStats::default()),
|
||||
("arn:test:empty".to_string(), ReplicationTargetUsage::default()),
|
||||
(
|
||||
"arn:test:non-empty".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
@@ -2565,7 +2568,7 @@ mod tests {
|
||||
replication_stats: Some(ReplicationAllStats {
|
||||
targets: HashMap::from([(
|
||||
"arn:test:pending".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
@@ -2714,7 +2717,7 @@ mod tests {
|
||||
targets: HashMap::from([
|
||||
(
|
||||
"arn:self-only".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
pending_size: 7,
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
@@ -2722,7 +2725,7 @@ mod tests {
|
||||
),
|
||||
(
|
||||
"arn:shared".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
failed_size: 3,
|
||||
failed_count: 1,
|
||||
missed_threshold_size: 2,
|
||||
@@ -2741,7 +2744,7 @@ mod tests {
|
||||
targets: HashMap::from([
|
||||
(
|
||||
"arn:shared".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
failed_size: 5,
|
||||
failed_count: 2,
|
||||
after_threshold_size: 4,
|
||||
@@ -2751,7 +2754,7 @@ mod tests {
|
||||
),
|
||||
(
|
||||
"arn:other-only".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
replicated_size: 11,
|
||||
replicated_count: 3,
|
||||
..Default::default()
|
||||
@@ -2993,7 +2996,9 @@ mod tests {
|
||||
fn replication_target_deserialization_preserves_large_historical_maps() {
|
||||
let mut stats = ReplicationAllStats::default();
|
||||
for index in 0..=1024 {
|
||||
stats.targets.insert(format!("target-{index}"), ReplicationStats::default());
|
||||
stats
|
||||
.targets
|
||||
.insert(format!("target-{index}"), ReplicationTargetUsage::default());
|
||||
}
|
||||
let encoded = rmp_serde::to_vec_named(&stats).expect("large replication target fixture should encode");
|
||||
let decoded = rmp_serde::from_slice::<ReplicationAllStats>(&encoded)
|
||||
@@ -3002,6 +3007,47 @@ mod tests {
|
||||
assert_eq!(decoded.targets.len(), stats.targets.len());
|
||||
}
|
||||
|
||||
/// Round-trip test: encoding a [`ReplicationTargetUsage`] and decoding it back
|
||||
/// must produce the exact same value. This guards against accidental serde
|
||||
/// field-name drift during the `ReplicationStats` -> `ReplicationTargetUsage`
|
||||
/// rename. Wire-level field names are the serialized Rust field identifiers,
|
||||
/// which must remain byte-identical.
|
||||
#[test]
|
||||
fn replication_target_usage_rmp_round_trip() {
|
||||
let original = ReplicationTargetUsage {
|
||||
pending_size: 100,
|
||||
replicated_size: 2_000,
|
||||
failed_size: 50,
|
||||
failed_count: 3,
|
||||
pending_count: 7,
|
||||
missed_threshold_size: 11,
|
||||
after_threshold_size: 22,
|
||||
missed_threshold_count: 1,
|
||||
after_threshold_count: 2,
|
||||
replicated_count: 99,
|
||||
};
|
||||
|
||||
let buf = rmp_serde::to_vec_named(&original).expect("encode ReplicationTargetUsage to msgpack");
|
||||
let decoded: ReplicationTargetUsage = rmp_serde::from_slice(&buf).expect("decode ReplicationTargetUsage from msgpack");
|
||||
assert_eq!(original, decoded, "round-trip through rmp must preserve every field");
|
||||
|
||||
// Also verify that encoding as an unnamed sequence and then decoding
|
||||
// with named fields produces the correct mapping (this catches reordering).
|
||||
let named_buf = rmp_serde::to_vec_named(&original).expect("re-encode for field-name pinning");
|
||||
// Spot-check that known field names appear in the named encoding.
|
||||
let named_str = String::from_utf8_lossy(&named_buf);
|
||||
assert!(named_str.contains("pending_size"), "field 'pending_size' must survive the rename");
|
||||
assert!(named_str.contains("replicated_size"), "field 'replicated_size' must survive the rename");
|
||||
assert!(
|
||||
named_str.contains("missed_threshold_size"),
|
||||
"field 'missed_threshold_size' must survive the rename"
|
||||
);
|
||||
assert!(
|
||||
named_str.contains("after_threshold_count"),
|
||||
"field 'after_threshold_count' must survive the rename"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_merge_rejects_noncanonical_histograms_without_mutation() {
|
||||
let mut entry = DataUsageEntry {
|
||||
|
||||
@@ -89,7 +89,6 @@ const DECOMMISSION_STAGE_SOURCE_CLEANUP: &str = "source_cleanup";
|
||||
const DECOMMISSION_STAGE_ENTRY_FINISHED: &str = "entry_finished";
|
||||
const DECOMMISSION_PROGRESS_SAVE_INTERVAL: Duration = Duration::seconds(30);
|
||||
const DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD: usize = 1000;
|
||||
const DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF: Duration = Duration::seconds(1);
|
||||
const DECOMMISSION_BUCKET_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_BUCKET_CONCURRENCY";
|
||||
const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4;
|
||||
const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30;
|
||||
@@ -639,6 +638,22 @@ fn track_decommission_current_object(meta: &mut PoolMeta, idx: usize, bucket: &s
|
||||
track_decommission_current_object_stage(meta, idx, bucket, object, "")
|
||||
}
|
||||
|
||||
fn touch_decommission_progress(meta: &mut PoolMeta, idx: usize) -> Result<()> {
|
||||
let pool_count = meta.pools.len();
|
||||
ensure_valid_decommission_pool_index(pool_count, idx)?;
|
||||
|
||||
let Some(pool) = meta.pools.get_mut(idx) else {
|
||||
return Err(invalid_decommission_pool_index_error(pool_count, idx));
|
||||
};
|
||||
let Some(info) = pool.decommission.as_mut() else {
|
||||
return Err(decommission_metadata_not_initialized_error("touch decommission progress"));
|
||||
};
|
||||
|
||||
pool.last_update = OffsetDateTime::now_utc();
|
||||
info.mark_progress_saved();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn resolve_decommission_update_after_result(result: Result<bool>) -> Result<bool> {
|
||||
result.map_err(|err| Error::other(format!("decommission metadata update failed: {err}")))
|
||||
}
|
||||
@@ -1468,7 +1483,6 @@ impl TryFrom<PersistedPoolDecommissionInfo> for PoolDecommissionInfo {
|
||||
terminal_reload_attempt_at: value.terminal_reload_attempt_at,
|
||||
terminal_reload_failures: value.terminal_reload_failures,
|
||||
progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
|
||||
progress_save_retry_after: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1500,7 +1514,6 @@ impl TryFrom<LegacyPoolDecommissionInfo> for PoolDecommissionInfo {
|
||||
terminal_reload_attempt_at: None,
|
||||
terminal_reload_failures: Vec::new(),
|
||||
progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
|
||||
progress_save_retry_after: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1614,82 +1627,6 @@ impl PoolMeta {
|
||||
}
|
||||
}
|
||||
|
||||
fn decommission_progress_checkpoint(
|
||||
&self,
|
||||
idx: usize,
|
||||
duration: Duration,
|
||||
now: OffsetDateTime,
|
||||
) -> Result<Option<DecommissionProgressCheckpoint>> {
|
||||
let pool_count = self.pools.len();
|
||||
ensure_valid_decommission_pool_index(pool_count, idx)?;
|
||||
|
||||
let Some(pool) = self.pools.get(idx) else {
|
||||
return Err(invalid_decommission_pool_index_error(pool_count, idx));
|
||||
};
|
||||
let Some(info) = pool.decommission.as_ref() else {
|
||||
return Err(decommission_metadata_not_initialized_error("update decommission metadata timestamp"));
|
||||
};
|
||||
|
||||
if info.progress_save_retry_after.is_some_and(|retry_after| now < retry_after) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let time_threshold_reached = now.unix_timestamp() - pool.last_update.unix_timestamp() >= duration.whole_seconds();
|
||||
let item_threshold_reached = info.items_since_last_progress_save() >= DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD;
|
||||
if !time_threshold_reached && !item_threshold_reached {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
Ok(Some(DecommissionProgressCheckpoint {
|
||||
start_time: info.start_time,
|
||||
queued: info.queued,
|
||||
counted_items: info.counted_items(),
|
||||
checkpoint_at: now,
|
||||
}))
|
||||
}
|
||||
|
||||
fn commit_decommission_progress_checkpoint(&mut self, idx: usize, checkpoint: DecommissionProgressCheckpoint) -> bool {
|
||||
let Some(pool) = self.pools.get_mut(idx) else {
|
||||
return false;
|
||||
};
|
||||
let Some(info) = pool.decommission.as_mut() else {
|
||||
return false;
|
||||
};
|
||||
|
||||
if info.start_time != checkpoint.start_time
|
||||
|| info.queued != checkpoint.queued
|
||||
|| !is_decommission_active(info.complete, info.failed, info.canceled)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
info.progress_save_item_baseline = info.progress_save_item_baseline.max(checkpoint.counted_items);
|
||||
info.progress_save_retry_after = None;
|
||||
pool.last_update = pool.last_update.max(checkpoint.checkpoint_at);
|
||||
true
|
||||
}
|
||||
|
||||
fn defer_decommission_progress_checkpoint(
|
||||
&mut self,
|
||||
idx: usize,
|
||||
checkpoint: DecommissionProgressCheckpoint,
|
||||
retry_after: OffsetDateTime,
|
||||
) {
|
||||
let Some(pool) = self.pools.get_mut(idx) else {
|
||||
return;
|
||||
};
|
||||
let Some(info) = pool.decommission.as_mut() else {
|
||||
return;
|
||||
};
|
||||
|
||||
if info.start_time == checkpoint.start_time
|
||||
&& info.queued == checkpoint.queued
|
||||
&& is_decommission_active(info.complete, info.failed, info.canceled)
|
||||
{
|
||||
info.progress_save_retry_after = Some(retry_after);
|
||||
}
|
||||
}
|
||||
|
||||
fn load_from_config_data(&mut self, data: Vec<u8>) -> Result<()> {
|
||||
if data.is_empty() {
|
||||
return Ok(());
|
||||
@@ -2050,9 +1987,30 @@ impl PoolMeta {
|
||||
}
|
||||
|
||||
pub fn update_after(&mut self, idx: usize, duration: Duration) -> Result<bool> {
|
||||
Ok(self
|
||||
.decommission_progress_checkpoint(idx, duration, OffsetDateTime::now_utc())?
|
||||
.is_some())
|
||||
let pool_count = self.pools.len();
|
||||
ensure_valid_decommission_pool_index(pool_count, idx)?;
|
||||
|
||||
let (last_update, item_threshold_reached) = match self.pools.get(idx) {
|
||||
Some(pool) if let Some(info) = pool.decommission.as_ref() => (
|
||||
pool.last_update,
|
||||
info.items_since_last_progress_save() >= DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
|
||||
),
|
||||
Some(_) => {
|
||||
return Err(decommission_metadata_not_initialized_error("update decommission metadata timestamp"));
|
||||
}
|
||||
None => return Err(invalid_decommission_pool_index_error(pool_count, idx)),
|
||||
};
|
||||
let now = OffsetDateTime::now_utc();
|
||||
|
||||
if now.unix_timestamp() - last_update.unix_timestamp() >= duration.whole_seconds() || item_threshold_reached {
|
||||
let Some(pool) = self.pools.get_mut(idx) else {
|
||||
return Err(invalid_decommission_pool_index_error(pool_count, idx));
|
||||
};
|
||||
pool.last_update = now;
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
pub fn validate(&self, pools: Vec<Arc<Sets>>) -> Result<bool> {
|
||||
@@ -2193,16 +2151,6 @@ pub struct PoolDecommissionInfo {
|
||||
pub terminal_reload_failures: Vec<String>,
|
||||
#[serde(skip)]
|
||||
pub progress_save_item_baseline: usize,
|
||||
#[serde(skip)]
|
||||
pub progress_save_retry_after: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct DecommissionProgressCheckpoint {
|
||||
start_time: Option<OffsetDateTime>,
|
||||
queued: bool,
|
||||
counted_items: usize,
|
||||
checkpoint_at: OffsetDateTime,
|
||||
}
|
||||
|
||||
impl PoolDecommissionInfo {
|
||||
@@ -2237,7 +2185,6 @@ impl PoolDecommissionInfo {
|
||||
|
||||
fn mark_progress_saved(&mut self) {
|
||||
self.progress_save_item_baseline = self.counted_items();
|
||||
self.progress_save_retry_after = None;
|
||||
}
|
||||
|
||||
pub fn bucket_push(&mut self, bucket: &DecomBucketInfo) {
|
||||
@@ -2542,40 +2489,6 @@ impl ECStore {
|
||||
snapshot.save(self.pools.clone()).await
|
||||
}
|
||||
|
||||
async fn save_decommission_progress_checkpoint(&self, idx: usize) -> Result<bool> {
|
||||
// Lock order: save gate, then the short pool metadata read/write sections. Peer
|
||||
// reloads are intentionally performed by the caller after both locks are released.
|
||||
let _save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (snapshot, checkpoint) = {
|
||||
let pool_meta = self.pool_meta.read().await;
|
||||
let Some(checkpoint) = pool_meta.decommission_progress_checkpoint(
|
||||
idx,
|
||||
DECOMMISSION_PROGRESS_SAVE_INTERVAL,
|
||||
OffsetDateTime::now_utc(),
|
||||
)?
|
||||
else {
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
let mut snapshot = pool_meta.clone();
|
||||
let Some(pool) = snapshot.pools.get_mut(idx) else {
|
||||
return Err(invalid_decommission_pool_index_error(snapshot.pools.len(), idx));
|
||||
};
|
||||
pool.last_update = checkpoint.checkpoint_at;
|
||||
(snapshot, checkpoint)
|
||||
};
|
||||
|
||||
if let Err(err) = snapshot.save(self.pools.clone()).await {
|
||||
let retry_after = OffsetDateTime::now_utc() + DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
pool_meta.defer_decommission_progress_checkpoint(idx, checkpoint, retry_after);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
Ok(pool_meta.commit_decommission_progress_checkpoint(idx, checkpoint))
|
||||
}
|
||||
|
||||
async fn save_current_pool_meta_for_decommission_start(
|
||||
&self,
|
||||
indices: &[usize],
|
||||
@@ -2958,7 +2871,7 @@ impl ECStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn track_decommission_entry_progress_stage(
|
||||
async fn save_decommission_entry_progress_stage(
|
||||
&self,
|
||||
idx: usize,
|
||||
bucket: &str,
|
||||
@@ -2969,6 +2882,22 @@ impl ECStore {
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
track_decommission_current_object_stage(&mut pool_meta, idx, bucket, object, stage)
|
||||
.map_err(|err| with_decommission_entry_context(stage, bucket, object, err))?;
|
||||
touch_decommission_progress(&mut pool_meta, idx)
|
||||
.map_err(|err| with_decommission_entry_context(stage, bucket, object, err))?;
|
||||
}
|
||||
|
||||
if let Some(err) = resolve_decommission_progress_save_result(self.save_current_pool_meta().await) {
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
stage,
|
||||
error = ?err,
|
||||
"Decommission progress stage save failed"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -3236,7 +3165,7 @@ impl ECStore {
|
||||
let bucket_name = bucket.clone();
|
||||
let object_name = rd.object_info.name.clone();
|
||||
|
||||
self.track_decommission_entry_progress_stage(
|
||||
self.save_decommission_entry_progress_stage(
|
||||
idx,
|
||||
bucket_name.as_str(),
|
||||
object_name.as_str(),
|
||||
@@ -3330,7 +3259,7 @@ impl ECStore {
|
||||
}
|
||||
decommission_cancel_signal_result(rx.is_cancelled())?;
|
||||
|
||||
self.track_decommission_entry_progress_stage(
|
||||
self.save_decommission_entry_progress_stage(
|
||||
idx,
|
||||
bucket.as_str(),
|
||||
entry.name.as_str(),
|
||||
@@ -3338,7 +3267,7 @@ impl ECStore {
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.track_decommission_entry_progress_stage(
|
||||
self.save_decommission_entry_progress_stage(
|
||||
idx,
|
||||
bucket.as_str(),
|
||||
entry.name.as_str(),
|
||||
@@ -3405,42 +3334,34 @@ impl ECStore {
|
||||
}
|
||||
};
|
||||
|
||||
self.track_decommission_entry_progress_stage(
|
||||
idx,
|
||||
bucket.as_str(),
|
||||
entry.name.as_str(),
|
||||
DECOMMISSION_STAGE_ENTRY_FINISHED,
|
||||
)
|
||||
.await?;
|
||||
self.save_decommission_entry_progress_stage(idx, bucket.as_str(), entry.name.as_str(), DECOMMISSION_STAGE_ENTRY_FINISHED)
|
||||
.await?;
|
||||
|
||||
if should_save_progress {
|
||||
match self.save_decommission_progress_checkpoint(idx).await {
|
||||
Ok(true) => {
|
||||
if let Some(notification_sys) = runtime_sources::notification_sys()
|
||||
&& let Err(err) = resolve_decommission_entry_reload_result(
|
||||
notification_sys.reload_pool_meta().await,
|
||||
bucket.as_str(),
|
||||
entry.name.as_str(),
|
||||
)
|
||||
{
|
||||
warn!("{err}");
|
||||
}
|
||||
}
|
||||
Ok(false) => {}
|
||||
Err(err) => {
|
||||
if let Some(err) = resolve_decommission_progress_save_result(Err(err)) {
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
state = "progress_save_failed",
|
||||
error = %err,
|
||||
"Decommission progress save failed; continuing and will retry at the next checkpoint"
|
||||
);
|
||||
}
|
||||
let save_result = self.save_current_pool_meta().await;
|
||||
if let Some(err) = resolve_decommission_progress_save_result(save_result) {
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_ENTRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_POOLS,
|
||||
pool_index = idx,
|
||||
bucket = %bucket,
|
||||
object = %entry.name,
|
||||
state = "progress_save_failed",
|
||||
error = %err,
|
||||
"Decommission progress save failed; continuing and will retry at the next checkpoint"
|
||||
);
|
||||
} else {
|
||||
let mut pool_meta = self.pool_meta.write().await;
|
||||
pool_meta.mark_decommission_progress_saved();
|
||||
if let Some(notification_sys) = runtime_sources::notification_sys()
|
||||
&& let Err(err) = resolve_decommission_entry_reload_result(
|
||||
notification_sys.reload_pool_meta().await,
|
||||
bucket.as_str(),
|
||||
entry.name.as_str(),
|
||||
)
|
||||
{
|
||||
warn!("{err}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5343,11 +5264,11 @@ pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usi
|
||||
#[cfg(test)]
|
||||
mod pools_tests {
|
||||
use super::{
|
||||
DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF,
|
||||
DecomBucketInfo, DecommissionStartPoolState, DecommissionTerminalState, ListCallback, PoolDecommissionInfo, PoolMeta,
|
||||
PoolSpaceInfo, PoolStatus, apply_decommission_status_space_info, bind_decommission_cancelers,
|
||||
bind_missing_decommission_cancelers, cancel_decommission_canceler, classify_decommission_terminal_state,
|
||||
count_decommission_item, decommission_cancel_signal_result, decommission_item_size, decommission_meta_bucket_options,
|
||||
DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DecomBucketInfo,
|
||||
DecommissionStartPoolState, DecommissionTerminalState, ListCallback, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo,
|
||||
PoolStatus, apply_decommission_status_space_info, bind_decommission_cancelers, bind_missing_decommission_cancelers,
|
||||
cancel_decommission_canceler, classify_decommission_terminal_state, count_decommission_item,
|
||||
decommission_cancel_signal_result, decommission_item_size, decommission_meta_bucket_options,
|
||||
decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency,
|
||||
ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_listing_disks_available,
|
||||
ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool,
|
||||
@@ -5372,8 +5293,9 @@ mod pools_tests {
|
||||
should_preserve_decommission_canceled_state, should_reject_decommission_cancel_as_terminal,
|
||||
should_retry_decommission_cancel_reload, should_retry_decommission_listing, should_skip_canceled_decommission_routine,
|
||||
split_decommission_buckets, take_and_cancel_decommission_canceler, take_decommission_canceler,
|
||||
track_decommission_current_object, track_decommission_current_object_stage, validate_start_decommission_request,
|
||||
wait_decommission_listing_retry, wait_decommission_worker_drain, with_decommission_entry_context,
|
||||
touch_decommission_progress, track_decommission_current_object, track_decommission_current_object_stage,
|
||||
validate_start_decommission_request, wait_decommission_listing_retry, wait_decommission_worker_drain,
|
||||
with_decommission_entry_context,
|
||||
};
|
||||
use crate::data_movement;
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
@@ -6616,7 +6538,7 @@ mod pools_tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_track_decommission_stage_does_not_advance_checkpoint_state() {
|
||||
fn test_touch_decommission_progress_updates_last_update_and_save_baseline() {
|
||||
let mut meta = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
@@ -6631,13 +6553,11 @@ mod pools_tests {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
track_decommission_current_object_stage(&mut meta, 0, "bucket", "object", "migrate_object")
|
||||
.expect("valid decommission progress should be tracked");
|
||||
touch_decommission_progress(&mut meta, 0).expect("valid decommission progress should be touched");
|
||||
|
||||
assert_eq!(meta.pools[0].last_update, OffsetDateTime::UNIX_EPOCH);
|
||||
assert!(meta.pools[0].last_update > OffsetDateTime::UNIX_EPOCH);
|
||||
let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
|
||||
assert_eq!(info.items_since_last_progress_save(), 5);
|
||||
assert_eq!(info.stage, "migrate_object");
|
||||
assert_eq!(info.items_since_last_progress_save(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -6712,134 +6632,6 @@ mod pools_tests {
|
||||
assert_eq!(info.items_since_last_progress_save(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pool_meta_update_after_does_not_advance_last_update_before_save() {
|
||||
let last_update = OffsetDateTime::UNIX_EPOCH;
|
||||
let mut meta = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: "pool-0".to_string(),
|
||||
last_update,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
start_time: Some(last_update),
|
||||
items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
|
||||
..Default::default()
|
||||
}),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
meta.update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
|
||||
.expect("item threshold should request a checkpoint")
|
||||
);
|
||||
assert_eq!(meta.pools[0].last_update, last_update);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decommission_progress_checkpoint_commits_exact_snapshot_watermark() {
|
||||
let start_time = OffsetDateTime::UNIX_EPOCH;
|
||||
let checkpoint_at = start_time + Duration::seconds(30);
|
||||
let mut meta = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: "pool-0".to_string(),
|
||||
last_update: start_time,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
start_time: Some(start_time),
|
||||
items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
|
||||
..Default::default()
|
||||
}),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let checkpoint = meta
|
||||
.decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at)
|
||||
.expect("valid decommission state should produce a checkpoint")
|
||||
.expect("item threshold should produce a checkpoint");
|
||||
meta.count_item(0, 1, false);
|
||||
|
||||
assert!(meta.commit_decommission_progress_checkpoint(0, checkpoint));
|
||||
let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
|
||||
assert_eq!(info.progress_save_item_baseline, checkpoint.counted_items);
|
||||
assert_eq!(info.items_since_last_progress_save(), 1);
|
||||
assert_eq!(meta.pools[0].last_update, checkpoint_at);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decommission_progress_checkpoint_backoff_does_not_advance_baseline() {
|
||||
let start_time = OffsetDateTime::UNIX_EPOCH;
|
||||
let checkpoint_at = start_time + Duration::seconds(30);
|
||||
let retry_after = checkpoint_at + DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
|
||||
let mut meta = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: "pool-0".to_string(),
|
||||
last_update: start_time,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
start_time: Some(start_time),
|
||||
items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
|
||||
..Default::default()
|
||||
}),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let checkpoint = meta
|
||||
.decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at)
|
||||
.expect("valid decommission state should produce a checkpoint")
|
||||
.expect("item threshold should produce a checkpoint");
|
||||
meta.defer_decommission_progress_checkpoint(0, checkpoint, retry_after);
|
||||
|
||||
assert!(
|
||||
meta.decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at)
|
||||
.expect("retry backoff check should succeed")
|
||||
.is_none()
|
||||
);
|
||||
assert_eq!(meta.pools[0].last_update, start_time);
|
||||
assert_eq!(
|
||||
meta.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("decommission info should exist")
|
||||
.progress_save_item_baseline,
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decommission_progress_checkpoint_count_scales_with_threshold() {
|
||||
let start_time = OffsetDateTime::UNIX_EPOCH;
|
||||
let checkpoint_at = start_time;
|
||||
let mut meta = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: "pool-0".to_string(),
|
||||
last_update: start_time,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
start_time: Some(start_time),
|
||||
..Default::default()
|
||||
}),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
let mut checkpoint_count = 0;
|
||||
|
||||
for _ in 0..(DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD * 10) {
|
||||
meta.count_item(0, 1, false);
|
||||
if let Some(checkpoint) = meta
|
||||
.decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at)
|
||||
.expect("valid decommission state should produce a checkpoint")
|
||||
{
|
||||
checkpoint_count += 1;
|
||||
assert!(meta.commit_decommission_progress_checkpoint(0, checkpoint));
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(checkpoint_count, 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ensure_decommission_not_rebalancing_rejects_running_rebalance() {
|
||||
let err = ensure_decommission_not_rebalancing(true).expect_err("rebalance running should be rejected");
|
||||
|
||||
@@ -16,7 +16,7 @@ use super::persistence::DataUsageCacheLoadAttempt;
|
||||
use super::*;
|
||||
use crate::storage_api::scanner_io::{HTTPRangeSpec, ObjectIO};
|
||||
use crate::{ScannerGetObjectReader, ScannerPutObjReader};
|
||||
use rustfs_data_usage::{ReplicationAllStats, ReplicationStats};
|
||||
use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage};
|
||||
use serde_json::Value;
|
||||
use std::io::Cursor;
|
||||
use std::pin::Pin;
|
||||
@@ -1636,7 +1636,7 @@ fn size_recursive_prunes_empty_and_preserves_threshold_replication_stats() {
|
||||
replication_stats: Some(ReplicationAllStats {
|
||||
targets: HashMap::from([(
|
||||
"arn:test:threshold".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
after_threshold_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use rustfs_data_usage::{ReplicationAllStats, ReplicationStats};
|
||||
use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage};
|
||||
|
||||
const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([7; 32]);
|
||||
|
||||
@@ -271,7 +271,7 @@ fn completed_data_usage_info_flattens_nested_bucket_entries() {
|
||||
replication_stats: Some(ReplicationAllStats {
|
||||
targets: HashMap::from([(
|
||||
"arn:target".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
replicated_size: 2048,
|
||||
replicated_count: 2,
|
||||
..Default::default()
|
||||
|
||||
@@ -206,13 +206,6 @@ def check_runner_selection(root: Path) -> list[str]:
|
||||
return errors
|
||||
|
||||
|
||||
def check_s3_tests_runner(root: Path) -> list[str]:
|
||||
runner = (root / "scripts/s3-tests/run.sh").read_text()
|
||||
if "--showlocals" in runner:
|
||||
return ["scripts/s3-tests/run.sh: pytest failure diagnostics must not dump local values"]
|
||||
return []
|
||||
|
||||
|
||||
def profile_selection(root: Path, profile: str) -> str:
|
||||
if not re.fullmatch(r"e2e-[a-z0-9-]+", profile):
|
||||
raise ValueError(f"invalid e2e profile name: {profile}")
|
||||
@@ -279,7 +272,6 @@ def validate(root: Path) -> list[str]:
|
||||
errors.extend(check_e2e_modules(root))
|
||||
errors.extend(check_fuzz_targets(root))
|
||||
errors.extend(check_runner_selection(root))
|
||||
errors.extend(check_s3_tests_runner(root))
|
||||
errors.extend(check_profile_definitions(root))
|
||||
return errors
|
||||
|
||||
@@ -349,23 +341,6 @@ class SelfTests(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(len(check_fuzz_targets(root)), 1)
|
||||
|
||||
def test_s3_runner_rejects_unbounded_failure_locals(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
runner = root / "scripts/s3-tests/run.sh"
|
||||
runner.parent.mkdir(parents=True)
|
||||
runner.write_text("tox -- -vv -ra --tb=long\n")
|
||||
self.assertEqual(check_s3_tests_runner(root), [])
|
||||
runner.write_text("tox -- -vv -ra --showlocals --tb=long\n")
|
||||
self.assertEqual(len(check_s3_tests_runner(root)), 1)
|
||||
with (
|
||||
mock.patch(__name__ + ".check_e2e_modules", return_value=[]),
|
||||
mock.patch(__name__ + ".check_fuzz_targets", return_value=[]),
|
||||
mock.patch(__name__ + ".check_runner_selection", return_value=[]),
|
||||
mock.patch(__name__ + ".check_profile_definitions", return_value=[]),
|
||||
):
|
||||
self.assertEqual(len(validate(root)), 1)
|
||||
|
||||
def test_profile_listing_enforces_selection(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
@@ -436,7 +411,7 @@ def main() -> int:
|
||||
for error in errors:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 1
|
||||
print("OK: e2e modules, runner selection, fuzz matrices, profiles, and bounded diagnostics are wired")
|
||||
print("OK: e2e modules, runner selection, fuzz matrices, and profile guards are wired")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -1028,11 +1028,10 @@ else
|
||||
fi
|
||||
|
||||
# Run tests from s3tests/functional
|
||||
# Failure locals can contain multi-MiB request bodies; keep tracebacks without expanding local values.
|
||||
set +e
|
||||
S3TEST_CONF="${CONF_OUTPUT_PATH}" \
|
||||
tox -- \
|
||||
-vv -ra --tb=long \
|
||||
-vv -ra --showlocals --tb=long \
|
||||
--maxfail="${MAXFAIL}" \
|
||||
--timeout="${TEST_TIMEOUT}" \
|
||||
--junitxml="${ARTIFACTS_DIR}/junit.xml" \
|
||||
|
||||
Reference in New Issue
Block a user