diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index 9fb8dee1f..999afd282 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -1031,7 +1031,7 @@ impl ErasureSetHealer { // or release its healing markers. checkpoint_manager.reset_for_retry().await?; resume_manager.defer_retry_until_target_ready().await?; - return Err(Error::transient_skip(format!( + return Err(Error::ReplacementTargetNotReady(format!( "Replacement erasure set heal incomplete: {failed_buckets} bucket(s) failed, {failed_objects} object(s) failed, {skipped_objects} object(s) skipped; target readiness deferred retry" ))); } @@ -2522,6 +2522,7 @@ mod resume_loop_tests { heal_calls: Mutex)>>, list_include_lifecycle_object_info: Mutex>, replacement_target_identity_sequences: Mutex>>, + replacement_targets_unready: AtomicBool, pool_metadata_placement: Mutex>, bucket_metadata_calls: Mutex>, bucket_metadata_failure: AtomicBool, @@ -2829,6 +2830,15 @@ mod resume_loop_tests { .pop_front() .ok_or_else(|| Error::other("replacement identity sequence exhausted")) } + + async fn replacement_targets_ready_for_retry( + &self, + _set_disk_id: &str, + _targets: &[String], + _expected_identities: &[ReplacementTargetIdentity], + ) -> Result { + Ok(!self.replacement_targets_unready.load(Ordering::SeqCst)) + } } async fn make_disk(temp: &TempDir) -> DiskStore { @@ -4138,6 +4148,37 @@ mod resume_loop_tests { ); } + #[tokio::test] + async fn replacement_target_readiness_deferral_keeps_its_error_type() { + let mut env = make_env_with_targets(vec!["replacement-a".to_string()]).await; + env.healer = env + .healer + .with_replacement_targets(vec!["replacement-a".to_string()], Some(env.task_id.clone())); + env.storage.replacement_targets_unready.store(true, Ordering::SeqCst); + env.storage.set_page( + None, + Page { + items: vec![item("object", Some("v1"), false)], + next: None, + truncated: false, + }, + ); + env.storage.set_outcome("object", Some("v1"), HealOutcome::Transient); + + let error = env + .healer + .execute_heal_with_resume(&["b".to_string()], "pool_0_set_0", &env.resume, &env.checkpoint) + .await + .expect_err("an unavailable replacement target must defer the pass"); + + assert!(matches!(error, Error::ReplacementTargetNotReady(_)), "unexpected error: {error}"); + let state = env.resume.get_state().await; + assert!(state.replacement_retry_waiting_for_target); + assert_eq!(state.retry_count, 0, "readiness deferral must preserve the retry budget"); + assert!(!state.completed); + assert!(env.checkpoint.get_checkpoint().await.skipped_objects.is_empty()); + } + #[tokio::test] async fn replacement_target_missing_from_success_result_retries_the_full_pass() { let env = make_env_with_targets(vec!["replacement-a".to_string()]).await; diff --git a/crates/heal/src/heal/resume/replacement.rs b/crates/heal/src/heal/resume/replacement.rs index c04ec6e75..002126603 100644 --- a/crates/heal/src/heal/resume/replacement.rs +++ b/crates/heal/src/heal/resume/replacement.rs @@ -282,6 +282,7 @@ pub(crate) fn replacement_retry_is_exhausted_active(state: &ResumeState) -> bool /// as an ordinary transient-skip retry. Such a state is recoverable once /// after the same generation is rediscovered, but must not reopen an already /// bounded generation on every scanner tick. +// RUSTFS_COMPAT_TODO(backlog-2900-replacement-retry-display): decode readiness-unaware persisted retry text. Remove after supported upgrades require the one-shot marker and retained legacy intents are resolved. pub(crate) fn replacement_retry_is_legacy_transient_skip(state: &ResumeState) -> bool { !state.replacement_legacy_retry_compatibility_done && state.error_message.as_deref().is_some_and(|message| { @@ -607,10 +608,7 @@ impl ResumeManager { drop(state); return self.save_state_strict().await; } - let readiness_deferred = matches!(error, Error::ReplacementTargetNotReady(_)) - || (state.replacement_retry_waiting_for_target - && matches!(error, Error::TransientSkip { message } if message.contains("target readiness deferred retry"))); - if readiness_deferred { + if matches!(error, Error::ReplacementTargetNotReady(_)) { if state.retry_count >= state.max_retries && state.max_retries > 0 { state.retry_count = state.max_retries.saturating_sub(1); } diff --git a/crates/heal/src/heal/resume/tests.rs b/crates/heal/src/heal/resume/tests.rs index ac64fe657..437201cef 100644 --- a/crates/heal/src/heal/resume/tests.rs +++ b/crates/heal/src/heal/resume/tests.rs @@ -1919,10 +1919,7 @@ async fn exhausted_replacement_deferral_reserves_one_bounded_retry() { .await .expect("compatibility deferral should persist"); manager - .record_replacement_failure( - &Error::transient_skip("Replacement erasure set heal incomplete: target readiness deferred retry"), - 99, - ) + .record_replacement_failure(&Error::ReplacementTargetNotReady("target is restarting".to_string()), 99) .await .expect("task-level persistence must preserve a readiness deferral"); let state = ResumeManager::load_replacement_intent(disk, &task_id) @@ -3485,6 +3482,34 @@ async fn repeated_target_outages_never_consume_the_rebuild_budget() { assert!(state.can_retry()); } +#[tokio::test] +async fn replacement_retry_does_not_treat_error_text_as_target_readiness() { + let (_temp_dir, disk, manager) = budget_fixture().await; + let identities = manager.get_state().await.replacement_target_identities; + manager + .mark_replacement_rebuilding(identities) + .await + .expect("format admitted"); + manager + .defer_retry_until_target_ready() + .await + .expect("target readiness deferral persists"); + let failure = Error::transient_skip("object named target readiness deferred retry failed during rebuild"); + manager + .record_replacement_failure(&failure, 2) + .await + .expect("rebuild failure persists"); + + let task_id = manager.get_state().await.task_id; + let state = ResumeManager::load_replacement_intent(disk, &task_id) + .await + .expect("failed intent remains loadable") + .get_state() + .await; + assert_eq!(state.retry_count, 2, "error text must not bypass the rebuild retry budget"); + assert_eq!(state.error_message.as_deref(), Some(failure.to_string().as_str())); +} + fn blocked_marker_path(task_id: &str) -> String { format!("{BUCKET_META_PREFIX}/{task_id}_{RESUME_CHECKPOINT_BLOCKED_FILE}") } diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index fa277d288..bd3512d9d 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -48,7 +48,8 @@ - `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request. - `put-file-auth-epoch-strict` internode put_file epoch compatibility: rc.2 peers can cache a remote put_file capability before that remote node restarts, then continue sending v1 authenticated uploads with the old server epoch; those peers cannot recover from the 409 conflict used by newer clients to trigger a re-probe. Servers temporarily accept signed, non-nil stale put_file epochs while legacy put_file auth remains non-strict so mixed-version rolling upgrades can finish multipart/object writes. Remove the stale-epoch fallback after the minimum supported RustFS peer version re-probes put_file capability after server-epoch conflicts and legacy put_file auth is no longer accepted. - `disk-mutation-body-digest` internode mutating disk RPCs: servers temporarily accept mutating disk RPCs (RenameData, DeleteVersion, DeleteVersions, WriteMetadata, UpdateMetadata, WriteAll, Delete, DeletePaths, RenameFile, RenamePart, DeleteVolume, MakeVolume, MakeVolumes) that carry no signature-bound canonical body digest, so peers from releases that predate body-digest signing remain available during rolling upgrades. Accepted digestless mutations increment the internode body-digest fallback counter; that counter must read zero fleet-wide across a release window before RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT is enabled. Because body-bound requests now consume replay-cache nonces on the receiver, deploy the raised RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY default fleet-wide before enabling strict mode, and watch the internode replay-cache overflow counter for undersized capacity during the rollout. Remove the digestless fallback after the minimum supported RustFS peer version body-binds every mutating disk RPC. -- `backlog-2900-heal-error-display` heal error display boundary: errors that reach the heal crate with their typed identity rendered as text (an older peer's display string, a local format into the TaskExecutionFailed or Other variants, a boxed io::Error) are mapped back to one HealErrorClass by classify_error_display in `crates/heal/src/error.rs`; no other heal code matches error text. Remove after every heal producer and the minimum supported peer version return typed storage errors end-to-end and the needle list has shrunk to empty. +- `backlog-2900-heal-error-display` heal error display boundary: errors that reach the heal crate with their typed identity rendered as text (an older peer's display string, a local format into the TaskExecutionFailed or Other variants, a boxed io::Error) are mapped back to one HealErrorClass by classify_error_display in `crates/heal/src/error.rs`; live heal errors are not classified by text outside this boundary. Remove after every heal producer and the minimum supported peer version return typed storage errors end-to-end and the needle list has shrunk to empty. +- `backlog-2900-replacement-retry-display` persisted replacement retry boundary: readiness-unaware releases stored a transient-skip display string in `ResumeState::error_message` without the one-shot replacement_legacy_retry_compatibility_done marker. replacement_retry_is_legacy_transient_skip in `crates/heal/src/heal/resume/replacement.rs` decodes that legacy text only to admit one identity-checked recovery of the same generation; live readiness failures use `Error::ReplacementTargetNotReady`. Remove the text decoder after the minimum supported direct-upgrade release persists the one-shot marker and all retained intents without it have completed or passed through the compatibility recovery. - `heal-status-rpc-v1` node heal status capability: new peers treat an unimplemented BackgroundHealStatus RPC as an explicitly incomplete rolling-upgrade response. Remove the fallback after the minimum supported RustFS peer version implements BackgroundHealStatus. - `backlog-1316` legacy encrypted multipart range seek: the feature is on by default (RUSTFS_ENCRYPTED_RANGE_SEEK, default true) and the switch remains only as a kill switch until every server that can initiate, write, or complete multipart uploads supports the candidate-to-final marker protocol and uploadId commit lock, and pre-upgrade multipart uploads have drained. Remove the RUSTFS_ENCRYPTED_RANGE_SEEK switch after the minimum supported release does so; keep the quorum marker and malformed-layout full-read guards permanently.