gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 00:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(lifecycle): harden manual transition rollout checks (#5344)

Browse Source 
Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-07-28 01:29:06 +08:00
committed by GitHub
parent 0e42a3d1d9
commit 6d3ce90c0f
7 changed files with 271 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/e2e_test/src/reliant/tiering.rs
+50 -37
View File
@@ -400,6 +400,49 @@ struct ManualTransitionRunReport {
continuation_token: Option<String>,
}
fn assert_completed_or_in_flight_partial(state: &str, report: &ManualTransitionRunReport, context: &str) {
match state {
"completed" => {}
"partial" if report.skipped_already_in_flight > 0 => {}
_ => panic!("{context}: state={state}, report={report:#?}"),
}
}
fn assert_conflict_winner_report(state: &str, report: &ManualTransitionRunReport, expected_objects: u64, context: &str) {
assert_completed_or_in_flight_partial(state, report, context);
if report.skipped_already_in_flight > 0 {
assert!(
report.scanned <= expected_objects,
"{context}: scanned more objects than the conflict scope contains: {report:#?}"
);
assert!(
report.eligible <= expected_objects,
"{context}: marked more objects eligible than the conflict scope contains: {report:#?}"
);
assert_eq!(
report.enqueued + report.skipped_already_in_flight,
report.eligible,
"{context}: partial in-flight accounting must cover every eligible object: {report:#?}"
);
} else {
assert_eq!(report.scanned, expected_objects, "{context}: {report:#?}");
assert_eq!(
report.eligible + report.skipped_already_transitioned,
expected_objects,
"{context}: {report:#?}"
);
assert_eq!(
report.enqueued + report.skipped_already_in_flight,
expected_objects,
"{context}: {report:#?}"
);
}
assert_eq!(
report.transition_completed, report.enqueued,
"{context}: winner must wait for all queued transitions: {report:#?}"
);
}
#[derive(Debug, Deserialize)]
struct ManualTransitionQueueSnapshot {
queue_capacity: u64,
@@ -880,7 +923,7 @@ async fn test_manual_transition_run_black_box_semantics() -> TestResult {
assert_eq!(due.mode, "enqueue_only");
assert!(due.job_id.is_none());
assert!(due.status_endpoint.is_none());
assert_eq!(due.state, "completed");
assert_completed_or_in_flight_partial(&due.state, &due.report, "due manual transition run");
assert_eq!(due.report.bucket, MANUAL_DUE_BUCKET);
assert_eq!(due.report.prefix, "manual-due/");
assert_eq!(due.report.tier.as_deref(), Some(TIER_NAME));
@@ -1314,29 +1357,16 @@ async fn test_manual_transition_async_overlapping_scope_conflict_reports_active_
let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, StdDuration::from_secs(30)).await?;
assert_eq!(terminal.job_id, job_id);
assert_eq!(terminal.status, "completed", "terminal conflict winner response: {terminal:#?}");
assert!(!terminal.report.dry_run);
assert_eq!(terminal.report.bucket, MANUAL_ASYNC_CONFLICT_BUCKET);
assert_eq!(terminal.report.prefix, accepted.report.prefix);
assert_eq!(
terminal.report.scanned, MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
"terminal conflict winner response: {terminal:#?}"
);
assert_eq!(
terminal.report.eligible + terminal.report.skipped_already_transitioned,
assert_conflict_winner_report(
&terminal.status,
&terminal.report,
MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
"terminal conflict winner response: {terminal:#?}"
"terminal conflict winner response",
);
assert_eq!(terminal.report.dry_run_eligible, 0, "terminal conflict winner response: {terminal:#?}");
assert_eq!(
terminal.report.enqueued + terminal.report.skipped_already_in_flight,
terminal.report.eligible,
"terminal conflict winner response: {terminal:#?}"
);
assert_eq!(
terminal.report.transition_completed, terminal.report.enqueued,
"terminal conflict winner must wait for all queued transitions: {terminal:#?}"
);
assert_eq!(terminal.report.transition_failed, 0, "terminal conflict winner response: {terminal:#?}");
assert_eq!(terminal.report.tier_failure, 0, "terminal conflict winner response: {terminal:#?}");
let after_remote_count = cold_tier_object_count(&cold_client).await?;
@@ -1440,26 +1470,9 @@ async fn test_manual_transition_async_same_scope_conflict_reports_active_job() -
let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, StdDuration::from_secs(30)).await?;
assert_eq!(terminal.job_id, job_id);
assert_eq!(
terminal.status, "completed",
"terminal same-scope conflict winner response: {terminal:#?}"
);
assert_eq!(terminal.report.bucket, MANUAL_ASYNC_SAME_SCOPE_CONFLICT_BUCKET);
assert_eq!(terminal.report.prefix, MANUAL_ASYNC_SAME_SCOPE_CONFLICT_PREFIX);
assert_eq!(terminal.report.scanned, 50, "terminal same-scope conflict winner response: {terminal:#?}");
assert_eq!(
terminal.report.eligible, 50,
"terminal same-scope conflict winner response: {terminal:#?}"
);
assert_eq!(
terminal.report.enqueued + terminal.report.skipped_already_in_flight,
50,
"terminal same-scope conflict winner response: {terminal:#?}"
);
assert_eq!(
terminal.report.transition_completed, terminal.report.enqueued,
"terminal same-scope conflict winner must wait for all queued transitions: {terminal:#?}"
);
assert_conflict_winner_report(&terminal.status, &terminal.report, 50, "terminal same-scope conflict winner response");
assert_eq!(
terminal.report.transition_failed, 0,
"terminal same-scope conflict winner response: {terminal:#?}"
@@ -2189,7 +2202,7 @@ async fn test_manual_transition_run_queue_pressure_partial() -> TestResult {
assert_eq!(response.report.bucket, MANUAL_QUEUE_PRESSURE_BUCKET);
assert_eq!(response.report.prefix, MANUAL_QUEUE_PRESSURE_PREFIX);
assert!(
response.report.skipped_queue_full > 0,
response.report.skipped_queue_full > 0 || response.report.skipped_already_in_flight > 0,
"expected queue-pressure path to skip at least one object: {:#?}",
response.report
);
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+4 -1
View File
@@ -3452,7 +3452,10 @@ impl ManualTransitionRunReport {
}
pub fn has_partial_enqueue(&self) -> bool {
self.skipped_queue_full > 0 || self.skipped_queue_closed > 0 || self.skipped_queue_timeout > 0
self.skipped_already_in_flight > 0
|| self.skipped_queue_full > 0
|| self.skipped_queue_closed > 0
|| self.skipped_queue_timeout > 0
}
pub fn was_truncated(&self) -> bool {
crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs
+19
View File
@@ -2460,6 +2460,25 @@ mod tests {
assert!(record.error.is_none());
}
#[test]
fn manual_transition_job_record_in_flight_skip_report_is_partial() {
let options = ManualTransitionRunOptions::default();
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
record.complete(
ManualTransitionRunReport {
eligible: 1,
skipped_already_in_flight: 1,
..Default::default()
},
ManualTransitionQueueSnapshot::default(),
);
assert_eq!(record.state, ManualTransitionJobState::Partial);
assert_eq!(record.report.skipped_already_in_flight, 1);
assert!(record.error.is_none());
}
#[test]
fn manual_transition_job_record_queue_pressure_reports_persist_readback() {
let options = ManualTransitionRunOptions {