mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-05 03:35:38 +00:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8ddc3cd436 | |||
| ccf8e2362c | |||
| 2a6b2f31c7 | |||
| b33693fc19 | |||
| dacb617ff1 | |||
| 9ed1d46090 | |||
| 6eb60f8e72 | |||
| 8dd3cabd41 |
@@ -479,9 +479,11 @@ pub mod notification {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
|
||||
pub use crate::services::notification_sys::{
|
||||
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
|
||||
acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
|
||||
new_global_notification_sys, scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
|
||||
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, LegacyTransitionStateReconcileFleetProofToken, NotificationPeerErr,
|
||||
NotificationSys, ScannerPublicationLeaseGrant, acquire_cross_pool_fence_fleet_proof,
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
|
||||
legacy_transition_state_reconcile_fleet_proof_matches, new_global_notification_sys,
|
||||
scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ use super::{
|
||||
manual_transition_job, tier_delete_journal, transition_transaction,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::services::tier::tier_probe_intent;
|
||||
|
||||
pub(crate) const ILM_META_PREFIX: &str = "ilm";
|
||||
const ILM_META_OBJECT_PREFIX: &str = "ilm/";
|
||||
@@ -35,6 +36,7 @@ pub(crate) enum DurableIlmRecordKind {
|
||||
TierDeleteJournal,
|
||||
TierDeleteDispatchManifest,
|
||||
TransitionTransaction,
|
||||
TierProbeIntent,
|
||||
ManualTransitionJob,
|
||||
ManualTransitionScope,
|
||||
ManualTransitionTask,
|
||||
@@ -73,6 +75,12 @@ pub(crate) const TRANSITION_TRANSACTION_NAMESPACE: DurableIlmNamespace = Durable
|
||||
max_record_size: transition_transaction::MAX_TRANSITION_TRANSACTION_SIZE,
|
||||
kind: DurableIlmRecordKind::TransitionTransaction,
|
||||
};
|
||||
pub(crate) const TIER_PROBE_INTENT_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "tier-probe-intent",
|
||||
prefix: tier_probe_intent::TIER_PROBE_INTENT_RECORD_PREFIX,
|
||||
max_record_size: tier_probe_intent::MAX_TIER_PROBE_INTENT_SIZE,
|
||||
kind: DurableIlmRecordKind::TierProbeIntent,
|
||||
};
|
||||
pub(crate) const MANUAL_TRANSITION_JOB_NAMESPACE: DurableIlmNamespace = DurableIlmNamespace {
|
||||
name: "manual-transition-job",
|
||||
prefix: "ilm/manual-transition/jobs",
|
||||
@@ -98,11 +106,12 @@ pub(crate) const MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE: DurableIlmNamespace
|
||||
kind: DurableIlmRecordKind::ManualTransitionWorkerResult,
|
||||
};
|
||||
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 8] = [
|
||||
pub(crate) const DURABLE_ILM_NAMESPACES: [DurableIlmNamespace; 9] = [
|
||||
TIER_DELETE_JOURNAL_NAMESPACE,
|
||||
TIER_DELETE_JOURNAL_V6_NAMESPACE,
|
||||
TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
|
||||
TRANSITION_TRANSACTION_NAMESPACE,
|
||||
TIER_PROBE_INTENT_NAMESPACE,
|
||||
MANUAL_TRANSITION_JOB_NAMESPACE,
|
||||
MANUAL_TRANSITION_SCOPE_NAMESPACE,
|
||||
MANUAL_TRANSITION_TASK_NAMESPACE,
|
||||
@@ -200,6 +209,15 @@ pub(crate) enum DurableIlmRecordCheckpoint {
|
||||
revision: u64,
|
||||
state: transition_transaction::TransitionTransactionState,
|
||||
},
|
||||
TierProbeIntent {
|
||||
content_sha256: String,
|
||||
identity_sha256: String,
|
||||
remote_version_sha256: String,
|
||||
remote_version_known: bool,
|
||||
owner_fence_sha256: String,
|
||||
revision: u64,
|
||||
state: tier_probe_intent::TierProbeIntentState,
|
||||
},
|
||||
ManualTransitionJob {
|
||||
content_sha256: String,
|
||||
identity_sha256: String,
|
||||
@@ -232,6 +250,7 @@ impl DurableIlmRecordCheckpoint {
|
||||
| Self::TierDeleteDispatchManifest { content_sha256, .. }
|
||||
| Self::TierDeleteDispatchParent { content_sha256, .. }
|
||||
| Self::TransitionTransaction { content_sha256, .. }
|
||||
| Self::TierProbeIntent { content_sha256, .. }
|
||||
| Self::ManualTransitionJob { content_sha256, .. }
|
||||
| Self::ManualTransitionScope { content_sha256, .. }
|
||||
| Self::ManualTransitionTask { content_sha256 }
|
||||
@@ -421,6 +440,32 @@ impl DurableIlmRecordCheckpoint {
|
||||
.is_some_and(|expected_revision| *next_revision == expected_revision)
|
||||
&& (!previous_remote_version_known || previous_remote_version == next_remote_version)
|
||||
}
|
||||
(
|
||||
Self::TierProbeIntent {
|
||||
identity_sha256: previous_identity,
|
||||
remote_version_sha256: previous_remote_version,
|
||||
remote_version_known: previous_remote_version_known,
|
||||
owner_fence_sha256: previous_owner_fence,
|
||||
revision: previous_revision,
|
||||
state: previous_state,
|
||||
..
|
||||
},
|
||||
Self::TierProbeIntent {
|
||||
identity_sha256: next_identity,
|
||||
remote_version_sha256: next_remote_version,
|
||||
owner_fence_sha256: next_owner_fence,
|
||||
revision: next_revision,
|
||||
state: next_state,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
previous_identity == next_identity
|
||||
&& previous_owner_fence == next_owner_fence
|
||||
&& next_revision
|
||||
.checked_sub(*previous_revision)
|
||||
.is_some_and(|distance| distance == 1 && tier_probe_state_reaches(*previous_state, *next_state, distance))
|
||||
&& (!previous_remote_version_known || previous_remote_version == next_remote_version)
|
||||
}
|
||||
(
|
||||
Self::ManualTransitionJob {
|
||||
content_sha256: previous_content,
|
||||
@@ -500,6 +545,14 @@ impl DurableIlmRecordCheckpoint {
|
||||
/// after the exact terminal ETag and terminal receipt were committed, to
|
||||
/// purge older object versions exposed by that deletion.
|
||||
pub(crate) fn is_predecessor_of_terminal(&self, terminal: &Self) -> bool {
|
||||
if let Self::TierProbeIntent { state, .. } = terminal
|
||||
&& !matches!(
|
||||
state,
|
||||
tier_probe_intent::TierProbeIntentState::AbortedNoRemote | tier_probe_intent::TierProbeIntentState::Completed
|
||||
)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if self == terminal || self.validate_successor(terminal).is_ok() {
|
||||
return true;
|
||||
}
|
||||
@@ -568,6 +621,37 @@ impl DurableIlmRecordCheckpoint {
|
||||
}
|
||||
})
|
||||
}
|
||||
(
|
||||
Self::TierProbeIntent {
|
||||
identity_sha256: previous_identity,
|
||||
remote_version_sha256: previous_remote_version,
|
||||
remote_version_known: previous_remote_version_known,
|
||||
owner_fence_sha256: previous_owner_fence,
|
||||
revision: previous_revision,
|
||||
state: previous_state,
|
||||
..
|
||||
},
|
||||
Self::TierProbeIntent {
|
||||
identity_sha256: terminal_identity,
|
||||
remote_version_sha256: terminal_remote_version,
|
||||
owner_fence_sha256: terminal_owner_fence,
|
||||
revision: terminal_revision,
|
||||
state: terminal_state,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
previous_identity == terminal_identity
|
||||
&& previous_owner_fence == terminal_owner_fence
|
||||
&& matches!(
|
||||
terminal_state,
|
||||
tier_probe_intent::TierProbeIntentState::AbortedNoRemote
|
||||
| tier_probe_intent::TierProbeIntentState::Completed
|
||||
)
|
||||
&& terminal_revision
|
||||
.checked_sub(*previous_revision)
|
||||
.is_some_and(|distance| tier_probe_state_reaches(*previous_state, *terminal_state, distance))
|
||||
&& (!previous_remote_version_known || previous_remote_version == terminal_remote_version)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -606,6 +690,23 @@ fn transition_state_distance(
|
||||
}
|
||||
}
|
||||
|
||||
fn tier_probe_state_reaches(
|
||||
from: tier_probe_intent::TierProbeIntentState,
|
||||
to: tier_probe_intent::TierProbeIntentState,
|
||||
revision_distance: u64,
|
||||
) -> bool {
|
||||
use tier_probe_intent::TierProbeIntentState::{AbortedNoRemote, CleanupPending, Completed, UploadOutcomeUnknown, Uploaded};
|
||||
|
||||
match (from, to) {
|
||||
(UploadOutcomeUnknown, Uploaded | CleanupPending | AbortedNoRemote) => revision_distance == 1,
|
||||
(UploadOutcomeUnknown, Completed) => matches!(revision_distance, 2 | 3),
|
||||
(Uploaded, CleanupPending) => revision_distance == 1,
|
||||
(Uploaded, Completed) => revision_distance == 2,
|
||||
(CleanupPending, Completed) => revision_distance == 1,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn manual_job_state_reaches(
|
||||
from: manual_transition_job::ManualTransitionJobState,
|
||||
to: manual_transition_job::ManualTransitionJobState,
|
||||
@@ -1082,6 +1183,42 @@ pub(crate) fn validate_durable_ilm_record(path: &str, data: &[u8]) -> Result<Val
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::TierProbeIntent => {
|
||||
let probe_id = tier_probe_intent::tier_probe_intent_id_from_record_object_name(path)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
let intent =
|
||||
tier_probe_intent::TierProbeIntent::decode(probe_id, data).map_err(|err| Error::other(err.to_string()))?;
|
||||
let canonical =
|
||||
tier_probe_intent::tier_probe_intent_record_object_name(probe_id).map_err(|err| Error::other(err.to_string()))?;
|
||||
if canonical != path {
|
||||
return Err(Error::other("tier probe intent path is not canonical"));
|
||||
}
|
||||
let identity_sha256 = checkpoint_hash(&(
|
||||
intent.probe_id,
|
||||
&intent.operation,
|
||||
&intent.tier_name,
|
||||
intent.destination_id,
|
||||
&intent.probe_object,
|
||||
&intent.creator_id,
|
||||
intent.creator_epoch,
|
||||
intent.created_at_unix_nanos,
|
||||
))?;
|
||||
let remote_version_sha256 = checkpoint_hash(&intent.remote_version)?;
|
||||
let owner_fence_sha256 = checkpoint_hash(&intent.owner)?;
|
||||
(
|
||||
"probe_id",
|
||||
probe_id.to_string(),
|
||||
DurableIlmRecordCheckpoint::TierProbeIntent {
|
||||
content_sha256,
|
||||
identity_sha256,
|
||||
remote_version_sha256,
|
||||
remote_version_known: !intent.remote_version.is_unknown(),
|
||||
owner_fence_sha256,
|
||||
revision: intent.revision,
|
||||
state: intent.state,
|
||||
},
|
||||
)
|
||||
}
|
||||
DurableIlmRecordKind::ManualTransitionJob => {
|
||||
let job_id = manual_transition_job::manual_transition_job_id_from_record_object_name(path)
|
||||
.map_err(|err| Error::other(err.to_string()))?;
|
||||
@@ -1237,6 +1374,102 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn tier_probe_intent_fixture() -> tier_probe_intent::TierProbeIntent {
|
||||
let probe_id = Uuid::parse_str("36e2220e-9ad2-495b-b3bc-c4d2caf70a31").expect("fixture uuid should parse");
|
||||
tier_probe_intent::TierProbeIntent {
|
||||
probe_id,
|
||||
revision: 1,
|
||||
state: tier_probe_intent::TierProbeIntentState::UploadOutcomeUnknown,
|
||||
operation: tier_probe_intent::TierProbeOperationIdentity::Verify {
|
||||
config_etag: "config-etag".to_string(),
|
||||
backend_identity: [1; 32],
|
||||
},
|
||||
tier_name: "COLD-A".to_string(),
|
||||
destination_id: [1; 32],
|
||||
probe_object: tier_probe_intent::tier_probe_object_name(probe_id),
|
||||
creator_id: "node-a".to_string(),
|
||||
creator_epoch: Uuid::parse_str("76746062-c05a-40b7-9e38-d2722d7e0332").expect("fixture creator epoch should parse"),
|
||||
created_at_unix_nanos: 1_780_000_000_000_000_000,
|
||||
owner: tier_probe_intent::TierProbeOwnerFence {
|
||||
owner_id: "node-a".to_string(),
|
||||
owner_epoch: Uuid::parse_str("76746062-c05a-40b7-9e38-d2722d7e0332").expect("fixture owner epoch should parse"),
|
||||
not_after_unix_nanos: 1_780_000_900_000_000_000,
|
||||
},
|
||||
remote_version: tier_probe_intent::TierProbeRemoteVersion::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn tier_probe_checkpoint(intent: &tier_probe_intent::TierProbeIntent) -> DurableIlmRecordCheckpoint {
|
||||
let path =
|
||||
tier_probe_intent::tier_probe_intent_record_object_name(intent.probe_id).expect("tier probe path should build");
|
||||
let encoded = intent.encode().expect("tier probe intent should encode");
|
||||
let namespace = classify_durable_ilm_record(&path)
|
||||
.expect("tier probe namespace should classify")
|
||||
.expect("tier probe intent should be durable");
|
||||
assert_eq!(namespace, &TIER_PROBE_INTENT_NAMESPACE);
|
||||
validate_durable_ilm_record(&path, &encoded)
|
||||
.expect("tier probe intent should validate")
|
||||
.checkpoint
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_probe_intent_checkpoint_tracks_exact_monotonic_generations() {
|
||||
let initial_intent = tier_probe_intent_fixture();
|
||||
let initial = tier_probe_checkpoint(&initial_intent);
|
||||
|
||||
let mut uploaded_intent = initial_intent;
|
||||
uploaded_intent
|
||||
.advance(
|
||||
tier_probe_intent::TierProbeIntentState::Uploaded,
|
||||
tier_probe_intent::TierProbeRemoteVersion::versioned("opaque-v1"),
|
||||
)
|
||||
.expect("uploaded state should advance");
|
||||
let uploaded = tier_probe_checkpoint(&uploaded_intent);
|
||||
initial
|
||||
.validate_successor(&uploaded)
|
||||
.expect("durable receipt may adopt the exact uploaded generation");
|
||||
|
||||
let mut cleanup_intent = uploaded_intent.clone();
|
||||
cleanup_intent
|
||||
.advance(
|
||||
tier_probe_intent::TierProbeIntentState::CleanupPending,
|
||||
uploaded_intent.remote_version.clone(),
|
||||
)
|
||||
.expect("cleanup state should advance");
|
||||
let cleanup = tier_probe_checkpoint(&cleanup_intent);
|
||||
uploaded
|
||||
.validate_successor(&cleanup)
|
||||
.expect("durable receipt may adopt the exact cleanup generation");
|
||||
|
||||
let mut completed_intent = cleanup_intent.clone();
|
||||
completed_intent
|
||||
.advance(tier_probe_intent::TierProbeIntentState::Completed, cleanup_intent.remote_version.clone())
|
||||
.expect("completed state should advance");
|
||||
let completed = tier_probe_checkpoint(&completed_intent);
|
||||
cleanup
|
||||
.validate_successor(&completed)
|
||||
.expect("durable receipt may adopt the exact terminal generation");
|
||||
assert!(
|
||||
initial.is_predecessor_of_terminal(&completed),
|
||||
"terminal cleanup must recognize the full acknowledged-PUT path"
|
||||
);
|
||||
assert!(
|
||||
initial.validate_successor(&completed).is_err(),
|
||||
"ordinary receipt advancement must not skip intermediate generations"
|
||||
);
|
||||
assert!(
|
||||
!initial.is_predecessor_of_terminal(&uploaded),
|
||||
"a nonterminal generation must not be accepted as terminal proof"
|
||||
);
|
||||
|
||||
let mut rebound = uploaded_intent;
|
||||
rebound.owner.owner_epoch = Uuid::new_v4();
|
||||
assert!(
|
||||
rebound.encode().is_err(),
|
||||
"dormant v1 must reject owner takeover before producing a checkpoint"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_dispatch_manifest_namespace_validates_monotonic_branches() {
|
||||
use tier_delete_journal::TierDeleteDispatchManifestState::{Aborted, Aborting, Completed, DispatchAuthorized, Preparing};
|
||||
|
||||
@@ -203,7 +203,7 @@ mod tests {
|
||||
use parking_lot::Mutex;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
fn encode_context(context: &HashMap<String, String>) -> String {
|
||||
fn encode_context(context: &BTreeMap<String, String>) -> String {
|
||||
let ordered = context.iter().collect::<BTreeMap<_, _>>();
|
||||
serde_json::to_string(&ordered).expect("context serializes")
|
||||
}
|
||||
|
||||
+11
-1142
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -62,12 +62,27 @@ const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
|
||||
const CROSS_POOL_FENCE_SUPPORTED_VERSION: u32 = 2;
|
||||
const TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION: u32 = 3;
|
||||
const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
|
||||
// Keep this synchronized with the version served by node_service. Including
|
||||
// the local member in the minimum prevents an older coordinator from
|
||||
// self-authorizing a policy implemented only by newer remote peers.
|
||||
const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
|
||||
/// Version 5 is reserved for a fleet whose every metadata writer preserves
|
||||
/// explicit transition version state and destination identity, and implements
|
||||
/// conditional per-generation `xl.meta` writes with strong readback. The node
|
||||
/// service must not advertise this version until the conditional writer from
|
||||
/// rustfs/backlog#684 is available.
|
||||
const LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION: u32 = 5;
|
||||
type CrossPoolFencePolicyResult = Result<BTreeMap<String, Uuid>>;
|
||||
|
||||
fn cross_pool_fence_policy_results(
|
||||
peer_epochs: BTreeMap<String, Uuid>,
|
||||
minimum_version: u32,
|
||||
) -> (CrossPoolFencePolicyResult, CrossPoolFencePolicyResult, CrossPoolFencePolicyResult) {
|
||||
) -> (
|
||||
CrossPoolFencePolicyResult,
|
||||
CrossPoolFencePolicyResult,
|
||||
CrossPoolFencePolicyResult,
|
||||
CrossPoolFencePolicyResult,
|
||||
) {
|
||||
let journal_result = if minimum_version >= TIER_DELETE_JOURNAL_POLICY_SUPPORTED_VERSION {
|
||||
Ok(peer_epochs.clone())
|
||||
} else {
|
||||
@@ -78,7 +93,18 @@ fn cross_pool_fence_policy_results(
|
||||
} else {
|
||||
Err(Error::other("decommission target fence policy capability version is unsupported"))
|
||||
};
|
||||
(Ok(peer_epochs), journal_result, decommission_target_fence_result)
|
||||
let legacy_transition_state_reconcile_result =
|
||||
if minimum_version >= LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION {
|
||||
Ok(peer_epochs.clone())
|
||||
} else {
|
||||
Err(Error::other("legacy transition state reconcile policy capability version is unsupported"))
|
||||
};
|
||||
(
|
||||
Ok(peer_epochs),
|
||||
journal_result,
|
||||
decommission_target_fence_result,
|
||||
legacy_transition_state_reconcile_result,
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -252,10 +278,21 @@ pub(crate) struct TierDeleteJournalFleetProofToken {
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
/// Effect-window authority for one legacy transition-state reconciliation.
|
||||
///
|
||||
/// The token intentionally cannot be cloned. Its permit keeps the admitted
|
||||
/// fleet generation alive until the caller finishes the final strong
|
||||
/// readback, while revocation makes every later validation fail immediately.
|
||||
pub struct LegacyTransitionStateReconcileFleetProofToken {
|
||||
token: FleetCapabilityProofToken,
|
||||
_permit: FleetCapabilityProofPermit,
|
||||
}
|
||||
|
||||
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static CROSS_POOL_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static TIER_DELETE_JOURNAL_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static DECOMMISSION_TARGET_FENCE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF: OnceLock<std::sync::RwLock<FleetCapabilityProofState>> = OnceLock::new();
|
||||
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
|
||||
|
||||
fn cross_pool_fence_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
@@ -274,6 +311,10 @@ fn decommission_target_fence_fleet_proof_slot() -> &'static std::sync::RwLock<Fl
|
||||
DECOMMISSION_TARGET_FENCE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn legacy_transition_state_reconcile_fleet_proof_slot() -> &'static std::sync::RwLock<FleetCapabilityProofState> {
|
||||
LEGACY_TRANSITION_STATE_RECONCILE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(FleetCapabilityProofState::default()))
|
||||
}
|
||||
|
||||
fn revoke_fleet_capability_proof_state(state: &mut FleetCapabilityProofState) {
|
||||
if let Some(proof) = state.proof.take() {
|
||||
proof.generation.revoke();
|
||||
@@ -444,6 +485,125 @@ pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalF
|
||||
stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint)
|
||||
}
|
||||
|
||||
/// Acquire one non-cloneable authority that must span the complete reconcile
|
||||
/// effect window, including its final strong readback.
|
||||
pub async fn acquire_legacy_transition_state_reconcile_fleet_proof() -> Option<LegacyTransitionStateReconcileFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let proof = {
|
||||
let state = legacy_transition_state_reconcile_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, expected_topology, Instant::now())?
|
||||
};
|
||||
let observed_peer_epochs = observe_legacy_transition_state_reconcile_fleet(expected_topology).await?;
|
||||
let state = legacy_transition_state_reconcile_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
|
||||
&state,
|
||||
&proof,
|
||||
expected_topology,
|
||||
&observed_peer_epochs,
|
||||
Instant::now(),
|
||||
)
|
||||
.then_some(proof)
|
||||
}
|
||||
|
||||
fn acquire_legacy_transition_state_reconcile_fleet_proof_from(
|
||||
state: &FleetCapabilityProofState,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> Option<LegacyTransitionStateReconcileFleetProofToken> {
|
||||
let token = acquire_fleet_capability_proof_from(state, expected_topology, now)?;
|
||||
let permit = state.proof.as_ref()?.generation.try_acquire()?;
|
||||
Some(LegacyTransitionStateReconcileFleetProofToken { token, _permit: permit })
|
||||
}
|
||||
|
||||
async fn observe_legacy_transition_state_reconcile_fleet(expected_topology: &str) -> Option<BTreeMap<String, Uuid>> {
|
||||
let notification_sys = get_global_notification_sys()?;
|
||||
let (peer_epochs, minimum_version) = timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
notification_sys.probe_cross_pool_fence_fleet(expected_topology),
|
||||
)
|
||||
.await
|
||||
.ok()?
|
||||
.ok()?;
|
||||
let (_, _, _, reconcile_result) = cross_pool_fence_policy_results(peer_epochs, minimum_version);
|
||||
reconcile_result.ok()
|
||||
}
|
||||
|
||||
/// Revalidate the exact fleet generation captured by a reconcile token with a
|
||||
/// fresh synchronous observation. Callers must await this before each
|
||||
/// conditional metadata write and after the final strong readback.
|
||||
pub async fn legacy_transition_state_reconcile_fleet_proof_matches(
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
) -> bool {
|
||||
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
|
||||
return false;
|
||||
};
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_with_observer(
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
proof,
|
||||
expected_topology,
|
||||
|| observe_legacy_transition_state_reconcile_fleet(expected_topology),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn legacy_transition_state_reconcile_fleet_proof_matches_with_observer<F, Fut>(
|
||||
slot: &std::sync::RwLock<FleetCapabilityProofState>,
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
expected_topology: &str,
|
||||
observe: F,
|
||||
) -> bool
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = Option<BTreeMap<String, Uuid>>>,
|
||||
{
|
||||
{
|
||||
let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if !legacy_transition_state_reconcile_fleet_proof_matches_at(&state, proof, expected_topology, Instant::now()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
let Some(observed_peer_epochs) = observe().await else {
|
||||
return false;
|
||||
};
|
||||
let state = slot.read().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
|
||||
&state,
|
||||
proof,
|
||||
expected_topology,
|
||||
&observed_peer_epochs,
|
||||
Instant::now(),
|
||||
)
|
||||
}
|
||||
|
||||
fn legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
state: &FleetCapabilityProofState,
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
expected_topology: &str,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
proof._permit.generation.is_accepting()
|
||||
&& fleet_capability_proof_matches_at(state, &proof.token, expected_topology, now)
|
||||
&& state
|
||||
.proof
|
||||
.as_ref()
|
||||
.is_some_and(|current| Arc::ptr_eq(¤t.generation, &proof._permit.generation))
|
||||
}
|
||||
|
||||
fn legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
|
||||
state: &FleetCapabilityProofState,
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
expected_topology: &str,
|
||||
observed_peer_epochs: &BTreeMap<String, Uuid>,
|
||||
now: Instant,
|
||||
) -> bool {
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_at(state, proof, expected_topology, now)
|
||||
&& proof.token.peer_epochs.as_ref() == observed_peer_epochs
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) fn tier_delete_journal_fleet_proof_has_inflight_for_test() -> bool {
|
||||
let state = tier_delete_journal_fleet_proof_slot()
|
||||
@@ -766,6 +926,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
cross_pool_fence_fleet_proof_slot(),
|
||||
tier_delete_journal_fleet_proof_slot(),
|
||||
decommission_target_fence_fleet_proof_slot(),
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
] {
|
||||
mark_fleet_capability_topology_conflict(slot);
|
||||
}
|
||||
@@ -798,11 +959,12 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
.unwrap_or_else(|_| Err(Error::other("cross-pool fence fleet capability probe timed out"))),
|
||||
None => Err(Error::other("cross-pool fence fleet capability notification system is unavailable")),
|
||||
};
|
||||
let (fence_result, journal_result, decommission_target_fence_result) = match fence_probe {
|
||||
let (fence_result, journal_result, decommission_target_fence_result, reconcile_result) = match fence_probe {
|
||||
Ok((peer_epochs, minimum_version)) => cross_pool_fence_policy_results(peer_epochs, minimum_version),
|
||||
Err(err) => {
|
||||
let message = err.to_string();
|
||||
(
|
||||
Err(Error::other(message.clone())),
|
||||
Err(Error::other(message.clone())),
|
||||
Err(Error::other(message.clone())),
|
||||
Err(Error::other(message)),
|
||||
@@ -818,6 +980,7 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
revoke_fleet_capability_proof(cross_pool_fence_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(tier_delete_journal_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(decommission_target_fence_fleet_proof_slot());
|
||||
revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot());
|
||||
} else if let Some(err) = publish_fleet_capability_probe_result(
|
||||
remote_version_state_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
@@ -880,6 +1043,24 @@ pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
if !topology_conflict
|
||||
&& let Some(err) = publish_fleet_capability_probe_result(
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
&topology_fingerprint,
|
||||
reconcile_result,
|
||||
Instant::now(),
|
||||
)
|
||||
{
|
||||
debug!(
|
||||
event = EVENT_NOTIFICATION_CAPABILITY_PROBE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
capability = "legacy_transition_state_reconcile_v1",
|
||||
state = "failed_closed",
|
||||
error = %err,
|
||||
"notification capability probe"
|
||||
);
|
||||
}
|
||||
sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
|
||||
}
|
||||
});
|
||||
@@ -959,7 +1140,7 @@ impl NotificationSys {
|
||||
client.probe_cross_pool_fence(topology_fingerprint.to_string()).await
|
||||
});
|
||||
let mut peer_epochs = BTreeMap::new();
|
||||
let mut minimum_version = u32::MAX;
|
||||
let mut minimum_version = LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION;
|
||||
for result in join_all(probes).await {
|
||||
let (peer, version, epoch) = result?;
|
||||
if version < CROSS_POOL_FENCE_SUPPORTED_VERSION {
|
||||
@@ -968,11 +1149,6 @@ impl NotificationSys {
|
||||
minimum_version = minimum_version.min(version);
|
||||
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
|
||||
}
|
||||
// A single-node deployment has no remote member to lower the local
|
||||
// policy version advertised by this binary.
|
||||
if minimum_version == u32::MAX {
|
||||
minimum_version = DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION;
|
||||
}
|
||||
Ok((peer_epochs, minimum_version))
|
||||
}
|
||||
}
|
||||
@@ -3190,20 +3366,36 @@ mod tests {
|
||||
#[test]
|
||||
fn cross_pool_policy_versions_authorize_only_their_supported_protocols() {
|
||||
let peers = BTreeMap::from([("node-b:9000".to_string(), Uuid::new_v4())]);
|
||||
let (generic_v2, journal_v2, decommission_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
|
||||
let (generic_v2, journal_v2, decommission_v2, reconcile_v2) = cross_pool_fence_policy_results(peers.clone(), 2);
|
||||
assert!(generic_v2.is_ok(), "v2 remains valid for existing cross-pool fencing");
|
||||
assert!(journal_v2.is_err(), "a mixed v2/v3 fleet must fail closed for journal-v6 deletion");
|
||||
assert!(decommission_v2.is_err(), "v2 cannot authorize the sticky per-target decommission fence");
|
||||
assert!(reconcile_v2.is_err(), "v2 cannot authorize legacy transition-state reconciliation");
|
||||
|
||||
let (generic_v3, journal_v3, decommission_v3) = cross_pool_fence_policy_results(peers.clone(), 3);
|
||||
let (generic_v3, journal_v3, decommission_v3, reconcile_v3) = cross_pool_fence_policy_results(peers.clone(), 3);
|
||||
assert!(generic_v3.is_ok());
|
||||
assert!(journal_v3.is_ok(), "an all-v3 fleet may authorize journal-v6 deletion");
|
||||
assert!(decommission_v3.is_err(), "v3 members do not understand the per-target decommission fence");
|
||||
assert!(reconcile_v3.is_err());
|
||||
|
||||
let (generic_v4, journal_v4, decommission_v4) = cross_pool_fence_policy_results(peers, 4);
|
||||
let (generic_v4, journal_v4, decommission_v4, reconcile_v4) =
|
||||
cross_pool_fence_policy_results(peers.clone(), LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(generic_v4.is_ok());
|
||||
assert!(journal_v4.is_ok());
|
||||
assert!(decommission_v4.is_ok(), "an all-v4 fleet may create sticky per-target reservations");
|
||||
assert!(
|
||||
reconcile_v4.is_err(),
|
||||
"the current local policy lacks the conditional xl.meta writer required by reconcile"
|
||||
);
|
||||
|
||||
let (generic_v5, journal_v5, decommission_v5, reconcile_v5) = cross_pool_fence_policy_results(peers, 5);
|
||||
assert!(generic_v5.is_ok());
|
||||
assert!(journal_v5.is_ok());
|
||||
assert!(decommission_v5.is_ok());
|
||||
assert!(
|
||||
reconcile_v5.is_ok(),
|
||||
"only an all-v5 fleet preserves destination identity and conditional reconcile writes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3458,6 +3650,234 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_admits_only_compatible_single_and_multi_node_fleets() {
|
||||
let now = Instant::now();
|
||||
for peers in [
|
||||
BTreeMap::new(),
|
||||
BTreeMap::from([("peer-a".to_string(), Uuid::new_v4()), ("peer-b".to_string(), Uuid::new_v4())]),
|
||||
] {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let (_, _, _, result) =
|
||||
cross_pool_fence_policy_results(peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", result, now).is_none());
|
||||
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("an all-compatible fleet should admit reconciliation")
|
||||
};
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
now,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_restart_drains_concurrent_effect_windows() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let (_, _, _, original_result) =
|
||||
cross_pool_fence_policy_results(original_peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", original_result, now).is_none());
|
||||
|
||||
let (first, second) = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
(
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the first reconcile writer should be admitted"),
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the second reconcile writer should be admitted"),
|
||||
)
|
||||
};
|
||||
|
||||
let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let (_, _, _, restarted_result) =
|
||||
cross_pool_fence_policy_results(restarted_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
let blocked =
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", restarted_result, now + Duration::from_millis(1))
|
||||
.expect("a restarted member must revoke the old generation and wait for both writers");
|
||||
assert!(blocked.to_string().contains("previous generation to drain"));
|
||||
{
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(state.proof.is_none());
|
||||
assert!(state.draining_generation.is_some());
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&first,
|
||||
"topology-a",
|
||||
now + Duration::from_millis(1),
|
||||
));
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&second,
|
||||
"topology-a",
|
||||
now + Duration::from_millis(1),
|
||||
));
|
||||
}
|
||||
|
||||
drop(first);
|
||||
let (_, _, _, still_blocked_result) =
|
||||
cross_pool_fence_policy_results(restarted_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", still_blocked_result, now + Duration::from_millis(2),)
|
||||
.is_some(),
|
||||
"one remaining writer must keep the successor generation closed"
|
||||
);
|
||||
|
||||
drop(second);
|
||||
let (_, _, _, admitted_result) =
|
||||
cross_pool_fence_policy_results(restarted_peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", admitted_result, now + Duration::from_millis(3),)
|
||||
.is_none(),
|
||||
"the restarted generation may publish only after every old writer drains"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_fresh_observation_closes_the_polling_window() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let original_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let (_, _, _, original_result) =
|
||||
cross_pool_fence_policy_results(original_peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", original_result, now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the original fleet should admit reconciliation")
|
||||
};
|
||||
|
||||
let restarted_peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_at(&state, &admitted, "topology-a", now),
|
||||
"the periodic cache has not observed the restart yet"
|
||||
);
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_observation_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
&restarted_peers,
|
||||
now,
|
||||
));
|
||||
|
||||
let (_, _, _, downgraded) = cross_pool_fence_policy_results(original_peers, 4);
|
||||
assert!(
|
||||
downgraded.is_err(),
|
||||
"a synchronous observation of a downgraded peer must fail before any cached proof can authorize a write"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_transition_state_reconcile_invalid_token_skips_fleet_observation() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(peers), now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the original fleet should admit reconciliation")
|
||||
};
|
||||
revoke_fleet_capability_proof(&slot);
|
||||
|
||||
assert!(
|
||||
!legacy_transition_state_reconcile_fleet_proof_matches_with_observer(&slot, &admitted, "topology-a", || async {
|
||||
panic!("an invalid local generation must not trigger a fleet observation");
|
||||
},)
|
||||
.await
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_membership_and_topology_changes_revoke_authority() {
|
||||
let now = Instant::now();
|
||||
for replacement in [
|
||||
BTreeMap::from([("peer-a".to_string(), Uuid::new_v4()), ("peer-b".to_string(), Uuid::new_v4())]),
|
||||
BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]),
|
||||
] {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let original = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(original), now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the original fleet should admit reconciliation")
|
||||
};
|
||||
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(&slot, "topology-a", Ok(replacement), now + Duration::from_millis(1),)
|
||||
.is_some(),
|
||||
"membership or process-epoch replacement must wait for the admitted writer"
|
||||
);
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
now + Duration::from_millis(1),
|
||||
));
|
||||
}
|
||||
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", Ok(BTreeMap::new()), now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("the original topology should admit reconciliation")
|
||||
};
|
||||
mark_fleet_capability_topology_conflict(&slot);
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(state.topology_conflict);
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
now,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transition_state_reconcile_capability_downgrade_fails_closed() {
|
||||
let slot = std::sync::RwLock::new(FleetCapabilityProofState::default());
|
||||
let now = Instant::now();
|
||||
let peers = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
|
||||
let (_, _, _, compatible_result) =
|
||||
cross_pool_fence_policy_results(peers.clone(), LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(publish_fleet_capability_probe_result(&slot, "topology-a", compatible_result, now).is_none());
|
||||
let admitted = {
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now)
|
||||
.expect("v5 should admit reconciliation")
|
||||
};
|
||||
|
||||
let (_, _, _, downgraded_result) =
|
||||
cross_pool_fence_policy_results(peers, LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION - 1);
|
||||
let err = publish_fleet_capability_probe_result(&slot, "topology-a", downgraded_result, now + Duration::from_millis(1))
|
||||
.expect("a v4 member must revoke reconcile authority");
|
||||
assert!(err.to_string().contains("reconcile policy capability version is unsupported"));
|
||||
let state = slot.read().expect("reconcile proof slot should not poison");
|
||||
assert!(state.proof.is_none());
|
||||
assert!(!legacy_transition_state_reconcile_fleet_proof_matches_at(
|
||||
&state,
|
||||
&admitted,
|
||||
"topology-a",
|
||||
now + Duration::from_millis(1),
|
||||
));
|
||||
assert!(
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof_from(&state, "topology-a", now + Duration::from_millis(1),)
|
||||
.is_none(),
|
||||
"a downgraded fleet must remain inspect-only"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
|
||||
let now = Instant::now();
|
||||
@@ -3539,6 +3959,57 @@ mod tests {
|
||||
assert!(err.to_string().contains("incomplete"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_transition_state_reconcile_probe_rejects_missing_or_unreachable_members() {
|
||||
let missing = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: vec![None],
|
||||
peer_topology_hosts: vec!["peer-a".to_string()],
|
||||
peer_admin_caches: Vec::new(),
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
let missing_err = missing
|
||||
.probe_cross_pool_fence_fleet("topology-a")
|
||||
.await
|
||||
.expect_err("a missing member slot must prevent reconcile capability proof");
|
||||
assert!(missing_err.to_string().contains("incomplete"));
|
||||
|
||||
let unreachable = NotificationSys {
|
||||
peer_clients: vec![None],
|
||||
all_peer_clients: vec![None, None],
|
||||
peer_topology_hosts: vec!["peer-a".to_string()],
|
||||
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
let unreachable_err = unreachable
|
||||
.probe_cross_pool_fence_fleet("topology-a")
|
||||
.await
|
||||
.expect_err("an unreachable member must prevent reconcile capability proof");
|
||||
assert!(unreachable_err.to_string().contains("unreachable"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_transition_state_reconcile_single_node_stays_closed_before_local_cas_support() {
|
||||
let notification_sys = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: vec![None],
|
||||
peer_topology_hosts: Vec::new(),
|
||||
peer_admin_caches: Vec::new(),
|
||||
tier_config_reload_workers: Default::default(),
|
||||
};
|
||||
let (peers, minimum_version) = notification_sys
|
||||
.probe_cross_pool_fence_fleet("topology-a")
|
||||
.await
|
||||
.expect("a single-node capability probe should complete");
|
||||
assert!(peers.is_empty());
|
||||
assert_eq!(minimum_version, LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
|
||||
let (_, _, _, reconcile_result) = cross_pool_fence_policy_results(peers, minimum_version);
|
||||
assert!(
|
||||
reconcile_result.is_err(),
|
||||
"the current node must not self-authorize reconcile before the conditional writer lands"
|
||||
);
|
||||
}
|
||||
|
||||
fn build_props(endpoint: &str) -> ServerProperties {
|
||||
ServerProperties {
|
||||
endpoint: endpoint.to_string(),
|
||||
|
||||
@@ -21,6 +21,7 @@ pub mod tier_gen;
|
||||
pub mod tier_handlers;
|
||||
pub(crate) mod tier_mutation_intent;
|
||||
pub mod tier_mutation_peer;
|
||||
pub(crate) mod tier_probe_intent;
|
||||
pub mod warm_backend;
|
||||
pub mod warm_backend_aliyun;
|
||||
pub mod warm_backend_azure;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -25,19 +25,24 @@ use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
async fn prime_metadata_generation(set_disks: &SetDisks, bucket: &str, object: &str) -> GetObjectMetadataCacheKey {
|
||||
set_disks
|
||||
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
|
||||
.await
|
||||
.expect("object metadata should resolve");
|
||||
let generation = set_disks
|
||||
.get_object_metadata_cache_generation(bucket, object)
|
||||
.expect("metadata generation should be active");
|
||||
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
|
||||
assert!(
|
||||
set_disks.get_object_metadata_cache.get(&key).await.is_some(),
|
||||
"metadata read should publish the generation under test"
|
||||
);
|
||||
key
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
set_disks
|
||||
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
|
||||
.await
|
||||
.expect("object metadata should resolve");
|
||||
let generation = set_disks
|
||||
.get_object_metadata_cache_generation(bucket, object)
|
||||
.expect("metadata generation should be active");
|
||||
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
|
||||
if set_disks.get_object_metadata_cache.get(&key).await.is_some() {
|
||||
return key;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("metadata read should publish the generation under test")
|
||||
}
|
||||
|
||||
async fn assert_generation_reclaimed(set_disks: &SetDisks, key: &GetObjectMetadataCacheKey) {
|
||||
@@ -60,8 +65,17 @@ async fn transition_and_restore_reclaim_prior_metadata_generations() {
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
// Cache priming must not race a quorum-acknowledged PUT's remaining rename tail.
|
||||
let original = set_disks
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
let source_generation = prime_metadata_generation(&set_disks, bucket, object).await;
|
||||
@@ -164,8 +178,17 @@ async fn prepared_snapshot_transition_duplicate_and_late_get_use_committed_remot
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
// Cache priming must not race a quorum-acknowledged PUT's remaining rename tail.
|
||||
let original = set_disks
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
|
||||
|
||||
@@ -864,6 +864,11 @@ mod tests {
|
||||
save_tier_mutation_intent_record, save_tier_mutation_intent_record_if_current,
|
||||
},
|
||||
tier_mutation_peer::{TierMutationPeerError, TierMutationPeerState, handle_tier_mutation_peer_request},
|
||||
tier_probe_intent::{
|
||||
TierProbeIntent, TierProbeIntentState, TierProbeOperationIdentity, TierProbeOwnerFence, TierProbeRemoteVersion,
|
||||
delete_tier_probe_intent_record_if_current, load_tier_probe_intent_record,
|
||||
save_tier_probe_intent_record_if_absent, save_tier_probe_intent_record_if_current,
|
||||
},
|
||||
warm_backend::{TransitionCandidateProbe, WarmBackend},
|
||||
},
|
||||
set_disk::SetDiskTransitionUploadedCommitBarrier as TransitionUploadedCommitBarrier,
|
||||
@@ -17196,6 +17201,147 @@ mod tests {
|
||||
assert!(matches!(err, Error::ConfigNotFound));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn tier_probe_intent_store_enforces_create_cas_and_terminal_delete_preconditions() {
|
||||
let temp_dir = tempfile::tempdir().expect("create temp store dir");
|
||||
let (_ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "tier-probe-intent-cas", &[4])).await;
|
||||
let probe_id = uuid::Uuid::new_v4();
|
||||
let creator_epoch = uuid::Uuid::new_v4();
|
||||
let initial = TierProbeIntent {
|
||||
probe_id,
|
||||
revision: 1,
|
||||
state: TierProbeIntentState::UploadOutcomeUnknown,
|
||||
operation: TierProbeOperationIdentity::Verify {
|
||||
config_etag: "config-etag".to_string(),
|
||||
backend_identity: [1; 32],
|
||||
},
|
||||
tier_name: "COLD-A".to_string(),
|
||||
destination_id: [1; 32],
|
||||
probe_object: format!("rustfs-tier-probe-{probe_id}"),
|
||||
creator_id: "node-a".to_string(),
|
||||
creator_epoch,
|
||||
created_at_unix_nanos: 1_780_000_000_000_000_000,
|
||||
owner: TierProbeOwnerFence {
|
||||
owner_id: "node-a".to_string(),
|
||||
owner_epoch: creator_epoch,
|
||||
not_after_unix_nanos: 1_780_000_900_000_000_000,
|
||||
},
|
||||
remote_version: TierProbeRemoteVersion::default(),
|
||||
};
|
||||
|
||||
save_tier_probe_intent_record_if_absent(store.clone(), &initial)
|
||||
.await
|
||||
.expect("initial probe intent should persist with create-only semantics");
|
||||
let duplicate = save_tier_probe_intent_record_if_absent(store.clone(), &initial)
|
||||
.await
|
||||
.expect_err("duplicate create must fail closed");
|
||||
assert!(matches!(duplicate, Error::PreconditionFailed));
|
||||
|
||||
let observed_initial = load_tier_probe_intent_record(store.clone(), probe_id)
|
||||
.await
|
||||
.expect("initial probe intent should load with an ETag");
|
||||
assert_eq!(observed_initial.intent(), &initial);
|
||||
|
||||
let nonterminal_delete = delete_tier_probe_intent_record_if_current(store.clone(), &observed_initial)
|
||||
.await
|
||||
.expect_err("nonterminal evidence must not be deleted");
|
||||
assert!(nonterminal_delete.to_string().contains("must be terminal"));
|
||||
|
||||
let mut fabricated_current_intent = initial.clone();
|
||||
fabricated_current_intent.tier_name = "COLD-B".to_string();
|
||||
let mut fabricated_successor = fabricated_current_intent.clone();
|
||||
fabricated_successor
|
||||
.advance(
|
||||
TierProbeIntentState::Uploaded,
|
||||
TierProbeRemoteVersion::versioned(uuid::Uuid::new_v4().to_string()),
|
||||
)
|
||||
.expect("fabricated successor should be internally valid");
|
||||
let fabricated_current = observed_initial.with_intent_for_test(fabricated_current_intent.clone());
|
||||
let crossed_cas = save_tier_probe_intent_record_if_current(store.clone(), &fabricated_current, &fabricated_successor)
|
||||
.await
|
||||
.expect_err("a live ETag must not authorize a different caller record");
|
||||
assert!(matches!(crossed_cas, Error::PreconditionFailed));
|
||||
assert_eq!(
|
||||
load_tier_probe_intent_record(store.clone(), probe_id)
|
||||
.await
|
||||
.expect("crossed CAS must retain the authoritative record")
|
||||
.intent(),
|
||||
&initial
|
||||
);
|
||||
|
||||
let mut fabricated_terminal_intent = fabricated_current_intent;
|
||||
fabricated_terminal_intent
|
||||
.advance(TierProbeIntentState::AbortedNoRemote, TierProbeRemoteVersion::default())
|
||||
.expect("fabricated terminal should be internally valid");
|
||||
let fabricated_terminal = observed_initial.with_intent_for_test(fabricated_terminal_intent);
|
||||
let crossed_delete = delete_tier_probe_intent_record_if_current(store.clone(), &fabricated_terminal)
|
||||
.await
|
||||
.expect_err("a live ETag must not delete for a different caller record");
|
||||
assert!(matches!(crossed_delete, Error::PreconditionFailed));
|
||||
assert_eq!(
|
||||
load_tier_probe_intent_record(store.clone(), probe_id)
|
||||
.await
|
||||
.expect("crossed delete must retain the authoritative record")
|
||||
.intent(),
|
||||
&initial
|
||||
);
|
||||
|
||||
let remote_version = TierProbeRemoteVersion::versioned(uuid::Uuid::new_v4().to_string());
|
||||
let mut uploaded = observed_initial.intent().clone();
|
||||
uploaded
|
||||
.advance(TierProbeIntentState::Uploaded, remote_version.clone())
|
||||
.expect("known PUT result should advance");
|
||||
save_tier_probe_intent_record_if_current(store.clone(), &observed_initial, &uploaded)
|
||||
.await
|
||||
.expect("the matching initial ETag should admit one successor");
|
||||
|
||||
let stale_cas = save_tier_probe_intent_record_if_current(store.clone(), &observed_initial, &uploaded)
|
||||
.await
|
||||
.expect_err("a consumed ETag must not overwrite the current generation");
|
||||
assert!(matches!(stale_cas, Error::PreconditionFailed));
|
||||
|
||||
let observed_uploaded = load_tier_probe_intent_record(store.clone(), probe_id)
|
||||
.await
|
||||
.expect("uploaded generation should load");
|
||||
assert_eq!(observed_uploaded.intent(), &uploaded);
|
||||
let mut cleanup = observed_uploaded.intent().clone();
|
||||
cleanup
|
||||
.advance(TierProbeIntentState::CleanupPending, remote_version.clone())
|
||||
.expect("known candidate should become cleanup-pending");
|
||||
save_tier_probe_intent_record_if_current(store.clone(), &observed_uploaded, &cleanup)
|
||||
.await
|
||||
.expect("cleanup generation should persist by exact ETag");
|
||||
|
||||
let observed_cleanup = load_tier_probe_intent_record(store.clone(), probe_id)
|
||||
.await
|
||||
.expect("cleanup generation should load");
|
||||
let mut completed = observed_cleanup.intent().clone();
|
||||
completed
|
||||
.advance(TierProbeIntentState::Completed, remote_version)
|
||||
.expect("exact cleanup should become terminal");
|
||||
save_tier_probe_intent_record_if_current(store.clone(), &observed_cleanup, &completed)
|
||||
.await
|
||||
.expect("terminal generation should persist by exact ETag");
|
||||
|
||||
let stale_terminal = observed_cleanup.with_intent_for_test(completed.clone());
|
||||
let stale_delete = delete_tier_probe_intent_record_if_current(store.clone(), &stale_terminal)
|
||||
.await
|
||||
.expect_err("a stale ETag must not delete terminal evidence");
|
||||
assert!(matches!(stale_delete, Error::PreconditionFailed));
|
||||
|
||||
let observed_completed = load_tier_probe_intent_record(store.clone(), probe_id)
|
||||
.await
|
||||
.expect("terminal generation should remain after stale delete");
|
||||
assert_eq!(observed_completed.intent(), &completed);
|
||||
delete_tier_probe_intent_record_if_current(store.clone(), &observed_completed)
|
||||
.await
|
||||
.expect("the exact terminal ETag should delete the record");
|
||||
assert!(matches!(load_tier_probe_intent_record(store, probe_id).await, Err(Error::ConfigNotFound)));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
|
||||
@@ -7,7 +7,7 @@ For crate ownership, read [crate-boundaries.md](crate-boundaries.md): ECStore ow
|
||||
|
||||
## Model
|
||||
|
||||
Heal and every foreground or background write path serialize on the same object-level namespace write lock (a quorum lock RPC in distributed mode, the in-process lock manager on a single node; granularity is the object, the version component is always `None`), and heal holds its guard across the whole rename commit. MinIO's `x-minio-healing` marker is an out-of-lock defence against version-cleanup logic inside `RenameData` interleaving with a heal commit; RustFS's commit model has no such interleaving, so no persistent marker exists (`x-minio-healing` does not occur in `crates/` or `rustfs/`) and none is needed. Three layers replace it:
|
||||
Heal and every foreground or background write path serialize on the same object-level namespace write lock (a quorum lock RPC in distributed mode, the in-process lock manager on a single node; granularity is the object, the version component is always `None`), and heal holds its guard across the whole rename commit. This describes the intended lock scope while the guard remains valid; it does not prove rejection of an already-dispatched disk syscall after distributed lease loss. The authority, delayed-mutation, and recovery boundary is specified in [unified-object-generation.md](unified-object-generation.md). MinIO's `x-minio-healing` marker is an out-of-lock defence against version-cleanup logic inside `RenameData` interleaving with a heal commit; RustFS's commit model has no such interleaving, so no persistent marker exists (`x-minio-healing` does not occur in `crates/` or `rustfs/`) and none is needed. Three layers replace it:
|
||||
|
||||
| Layer | Mechanism | Owner |
|
||||
| --- | --- | --- |
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# ILM And Tiering Persistence Contracts
|
||||
|
||||
**Use this when:** changing an ILM transition, tier configuration mutation, manual transition job, tier-delete recovery path, pool decommission, or any code that can create, transfer, or destroy ownership of a remote-tier object.
|
||||
**Source of truth:** `TransitionTransaction` and `process_transition_transaction_record` in `crates/ecstore/src/bucket/lifecycle/transition_transaction.rs`; `TierMutationIntent` and its conditional store helpers in `crates/ecstore/src/services/tier/tier_mutation_intent.rs`; `TierConfigMgr::update_candidate_with_config_lock` and mutation recovery in `crates/ecstore/src/services/tier/tier.rs`; `handle_tier_mutation_peer_request` in `crates/ecstore/src/services/tier/tier_mutation_peer.rs`; the record encoders and CAS helpers in `crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs`; manual-job execution/recovery and `cleanup_free_version_exact` in `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs`; `process_tier_delete_journal_entry` and manifest recovery in `crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs`; the free-version scan/re-enqueue path `recover_tier_free_versions` in `crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs`; `DURABLE_ILM_NAMESPACES` and `validate_durable_ilm_record` in `crates/ecstore/src/bucket/lifecycle/durable_namespace.rs`; and `record_durable_ilm_decommission_progress`, receipt verification, and receipt cleanup in `crates/ecstore/src/core/pools.rs`.
|
||||
**Source of truth:** `TransitionTransaction` and `process_transition_transaction_record` in `crates/ecstore/src/bucket/lifecycle/transition_transaction.rs`; `TierMutationIntent` and its conditional store helpers in `crates/ecstore/src/services/tier/tier_mutation_intent.rs`; the dormant validation-probe record and conditional primitives in `crates/ecstore/src/services/tier/tier_probe_intent.rs`; `TierConfigMgr::update_candidate_with_config_lock` and mutation recovery in `crates/ecstore/src/services/tier/tier.rs`; `handle_tier_mutation_peer_request` in `crates/ecstore/src/services/tier/tier_mutation_peer.rs`; the record encoders and CAS helpers in `crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs`; manual-job execution/recovery and `cleanup_free_version_exact` in `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs`; `process_tier_delete_journal_entry` and manifest recovery in `crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs`; the free-version scan/re-enqueue path `recover_tier_free_versions` in `crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs`; `DURABLE_ILM_NAMESPACES` and `validate_durable_ilm_record` in `crates/ecstore/src/bucket/lifecycle/durable_namespace.rs`; and `record_durable_ilm_decommission_progress`, receipt verification, and receipt cleanup in `crates/ecstore/src/core/pools.rs`.
|
||||
|
||||
This document separates three kinds of statement:
|
||||
|
||||
@@ -44,6 +44,7 @@ All keys below are objects in the internal metadata bucket. The table gives the
|
||||
| Transition transaction | `rustfs-transition-transaction-v1`; successor v2 is approved below but not implemented | `ilm/transition-transactions/records/<aa>/<bb>/<transaction-id>.json` | The transition attempt creates it; transition commit/recovery cleans it | Immutable/fence identity: deployment, transaction, fixed v1 `owner_epoch`, write, source identity, tier/backend fingerprint, canonical remote object, deadline. Mutable: state, remote version, revision. `TransitionCleanupProof` is only a transient admission input to `mark_cleanup_pending`; it is not persisted in the record | Create-only maximum-parity write; exact record and ETag read before successor `If-Match`; terminal receipt followed by exact ETag conditional delete |
|
||||
| Tier mutation peer intent | `rustfs-tier-mutation-intent-v1` | `tier/mutation-intents/records/<aa>/<bb>/<mutation-id>.json` | The receiving peer creates and converges it; the mutation recovery path cleans it | Immutable: mutation ID/kind, old config ETag, candidate digest, sorted affected target identities, expiry. Mutable: revision, state, committed config ETag | Create with `If-None-Match: *`; transition/delete with ETag `If-Match`; maximum parity |
|
||||
| Tier mutation coordinator intent | `rustfs-tier-mutation-intent-v1` | `tier/mutation-intents/coordinators/<aa>/<bb>/<mutation-id>.json` | The initiating node creates it; coordinator recovery cleans it after peer convergence | Same mutation identity and mutable fields as the peer record | Same conditional-write contract as the peer intent |
|
||||
| Tier validation probe intent | Dormant `rustfs-tier-probe-intent-v1`; no writer or recovery is enabled | `ilm/tier-probe-intents/records/<aa>/<bb>/<probe-id>.json` | No current runtime owner because no path creates the record; v1 permits only the immutable creator as owner | Immutable probe, operation-generation, destination, random remote object, creator identity, and v1 owner fence. Mutable: revision, state, and monotonic remote-version proof | Conditional create/CAS/delete primitives exist but are not called by Add/Edit/Verify or recovery |
|
||||
| Manual job | `rustfs-manual-transition-job-v1` | `ilm/manual-transition/jobs/<aa>/<bb>/<job-id>.json` | The admin run creates it; the active owner or recovery lease advances it. There is no current record GC owner | Immutable: job ID, bucket-level scope, options, creation time. Mutable: owner/lease, state, cancel bit, cursor, progress/report, queue snapshot, timestamps/error | Initial UUID-key write uses maximum parity without create-only precondition; later updates use ETag CAS |
|
||||
| Manual scope admission | `rustfs-manual-transition-job-v1` | `ilm/manual-transition/scopes/<aa>/<bb>/<scope-digest>.json` | Job admission creates/renews it; the job owner removes it after terminalization | Immutable bucket/run-vs-dry-run scope; mutable job/lease ownership and expiry | Create-only, renew/delete by ETag CAS |
|
||||
| Manual task | `rustfs-manual-transition-task-v1` | `ilm/manual-transition/tasks/<job shards>/<job-id>/<task-key>.json` | The scanner persists it before queue admission; no current GC owner | Immutable job plus exact bucket/object/version/tier work identity | Append-only create with `If-None-Match: *` and maximum parity |
|
||||
@@ -56,7 +57,7 @@ All keys below are objects in the internal metadata bucket. The table gives the
|
||||
| Decommission expected-receipt manifest | `v1` | `decommission/ilm-manifests/<run-token>.json` | The source-pool decommission coordinator creates and cleans it | Run token plus exact sorted receipt-path count/digest | Create-only, exact readback, and verification before pool removal |
|
||||
| Recovery control, export, and disposition | `rustfs-ilm-recovery-control-v1`, `rustfs-ilm-recovery-export-v1`, and `rustfs-ilm-recovery-disposition-v1` are approved below but not implemented | `ilm/recovery-controls/...`, `ilm/recovery-exports/...`, and `ilm/recovery-dispositions/...` under protocol/shard/operation identities | Recovery owns one control for an exact source generation; the authenticated operator creates immutable export/disposition evidence; their collectors never own remote DELETE | Source protocol/path, all-pool copy-set manifest, ETags/content digests, owner lease, retry state, redacted error code, action, actor/reason, and terminal proof | Create-only, ETag CAS, all-pool strong readback, terminal receipt when covered by decommission, and exact conditional cleanup |
|
||||
|
||||
`durable_namespace.rs` registers exactly the two tier-journal namespaces, the dispatch-record namespace shared by single manifests, chunk children, and chunk parents, the transaction namespace, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object.
|
||||
`durable_namespace.rs` registers exactly the two tier-journal namespaces, the dispatch-record namespace shared by single manifests, chunk children, and chunk parents, the transition-transaction namespace, the dormant tier-validation-probe namespace, and four manual-job namespaces. A path beginning with `ilm/` that is not in that registry is an error during decommission rather than an ignorable object.
|
||||
|
||||
## Durable fences and write primitives
|
||||
|
||||
@@ -190,6 +191,35 @@ Intent transitions retry an ETag race at most three times before returning a ret
|
||||
- Lease drain, peer Prepare, and reference proof run outside both exclusive guards after the coordinator Prepared record and local fence are durable. The commit path reacquires namespace WRITE then `admin_updates` and repeats the full generation/identity proof. Expiry is checked as an additional rejection boundary and never replaces config-generation proof.
|
||||
- **Open:** a dedicated operator reconcile/status surface and bounded retention for irreconcilable coordinator/peer records.
|
||||
|
||||
## Tier validation probe intent
|
||||
|
||||
### Dormant current contract
|
||||
|
||||
`TierProbeIntent` defines a strict, checksum-protected `rustfs-tier-probe-intent-v1` envelope and the canonical key `ilm/tier-probe-intents/records/<aa>/<bb>/<probe-id>.json`. The same non-nil probe UUID also determines the remote object name `rustfs-tier-probe-<probe-id>`. The path parser rejects uppercase UUID aliases, wrong shards, extra components, and path/payload/object-name disagreement. The durable namespace registry validates this record so pool decommission cannot silently treat it as an ordinary object.
|
||||
|
||||
The format binds Add and Edit to the same durable mutation identity tuple `(mutation_id, old_config_etag, candidate_digest)`. The old config ETag is required even for Add because it identifies the complete persisted tier configuration generation, not whether the named destination tier already exists. Verify instead requires the current persisted config ETag and credential-independent backend identity. These are mutually exclusive tagged variants. Every record also requires the tier name, matching destination identity, immutable creator identity/epoch, positive creation time, and an owner fence with nonempty owner, non-nil epoch, and later `not_after` timestamp. In v1 that owner identity must remain exactly equal to the immutable creator identity. It never persists the process-local driver revision or credential-bearing driver fingerprint.
|
||||
|
||||
The dormant state graph is:
|
||||
|
||||
```text
|
||||
UploadOutcomeUnknown -> Uploaded -> CleanupPending -> Completed
|
||||
\-> CleanupPending -> Completed
|
||||
\-> AbortedNoRemote
|
||||
```
|
||||
|
||||
`UploadOutcomeUnknown` and `AbortedNoRemote` carry no remote version. `Uploaded`, `CleanupPending`, and `Completed` carry either explicit unversioned semantics or one exact nonempty opaque version. Once known, that remote version cannot change. Revision advances by one for each edge: `UploadOutcomeUnknown` is revision 1, `Uploaded` and `AbortedNoRemote` are revision 2, `CleanupPending` is revision 2 or 3, and `Completed` is revision 3 or 4. The strict decoder rejects any other state/revision pairing. The direct `UploadOutcomeUnknown -> CleanupPending` edge is reserved for a future authoritative provider probe that discovers the exact cleanup candidate after the original PUT response was lost.
|
||||
|
||||
Create-only, ETag CAS, exact-ETag delete, and read-with-ETag primitives exist for the record, plus a crate-level read-only inspection result that always reports both writer and destructive recovery as disabled. No Add, Edit, Verify, startup loop, periodic loop, admin HTTP route, or remote backend operation currently calls these mutation primitives. Consequently this version creates no records and authorizes no remote PUT or DELETE.
|
||||
|
||||
### Activation requirements
|
||||
|
||||
- Add/Edit must create the mutation identity before validation and reread the same `(mutation_id, old_config_etag, candidate_digest)` before every probe-intent successor. Verify must reread the same config ETag, tier, and backend identity. A credential rotation may supply usable current credentials only when the persisted destination identity remains exact; it may not weaken operation-generation checks.
|
||||
- Before enabling any writer, every required node must advertise a probe-intent-specific read/retain/recovery capability. This protocol does not reuse the legacy transition-state reconciliation capability or its token. Unknown or older nodes keep validation in the current process-local mode and no durable v1 record is written.
|
||||
- The v1 owner fence is immutable and must equal the creator identity. Any future takeover requires a new schema with explicit takeover proof, plus an approved lease duration, clock-skew allowance, durable owner/epoch CAS, and revalidation order. Expiry alone never permits remote DELETE.
|
||||
- Remote-version discovery and deletion must use the provider-bound, destination-bound implementation and bounded request API approved for that target. Unknown, multiple, changing, unsupported, or unavailable results retain the record.
|
||||
- A successful or response-lost state write must strongly reread the exact record. Remote DELETE requires a current operation-generation proof, exact destination, current credentials for that same destination, a valid fleet fence, durable takeover, and the same known remote version immediately before and after the call.
|
||||
- Terminal retention, bounded scanning, metrics, and any HTTP inspect/reconcile route remain unapproved. Raw age is never cleanup evidence, and this dormant core API must not be presented as an operator endpoint.
|
||||
|
||||
## Manual transition job, task, result, and checkpoint
|
||||
|
||||
### Current contract
|
||||
|
||||
@@ -1,112 +1,177 @@
|
||||
# Object Transaction UUID And Generation-Fencing Contract
|
||||
# Object Generation Authority And Recovery Contract
|
||||
|
||||
**Use this when:** adding or changing anything that fences a commit, scopes a read lease, gates old-directory cleanup, binds prepared pool reads, or settles quota against "the current version of an object", or when adding a field that rides internode RPC or `xl.meta`.
|
||||
**Source of truth:** `assign_object_transaction_epoch` in `crates/ecstore/src/set_disk/ops/object.rs` and `crates/ecstore/src/set_disk/ops/multipart.rs`; `FileInfo::set_object_transaction_epoch` in `crates/filemeta/src/fileinfo.rs`; `commit_rename_data_dir` and `RenameConvergence` in `crates/ecstore/src/set_disk/core/io_primitives.rs`; `PreparedPoolReadFallbackBarrier` in `crates/ecstore/src/store/rebalance.rs`; `crates/protos/src/node.proto`; env constants in `crates/config/src/constants/object.rs` and `crates/config/src/constants/internode.rs`.
|
||||
**Use this when:** changing object commit fencing, rollback, old-directory cleanup, prepared reads, quota settlement, or the metadata and RPC fields used by those operations.
|
||||
**Source of truth:** `crates/ecstore/src/set_disk/ops/object.rs` (`assign_object_transaction_epoch`, `verify_object_transaction_epoch_fence`); `crates/ecstore/src/set_disk/core/io_primitives.rs` (`rename_data_owned_with_fence`, `commit_rename_data_dir`); `crates/ecstore/src/disk/local.rs` (`rename_data`, `write_all_meta`); `crates/lock/src/distributed_lock.rs` (`DistributedLockGuard`, `LockLostSignal`). The implementation boundary below distinguishes existing behavior from the selected design.
|
||||
|
||||
Design tracking lives in `rustfs/backlog#1326`. This document holds only the invariants.
|
||||
## Decision And Implementation Boundary
|
||||
|
||||
## Authority
|
||||
The selected minimum authority is a **durable, ordered per-object decision protocol attached to the existing namespace-lock participant group**. The object transaction UUID remains an opaque operation/idempotency identifier. It is not an ordered lock epoch. Extending the existing lock group requires durable promises, accepted values, quorum decisions, and recovery; adding a counter to today's lock response is insufficient.
|
||||
|
||||
The target contract requires **one per-object commit identity** consumed by commit fencing, read leases, cleanup, prepared reads, and quota settlement. No consumer may mint a second value and call it the same generation.
|
||||
An independent service holding every object's full manifest is not selected. It would add a new routing, membership, availability, and metadata ownership system and require a wider read/write migration. The selected protocol stores the current decision and recoverable outstanding successor with the existing lock participants; object payload and prepared metadata remain on the existing storage disks. This is still new consensus and persistence work, not a small `RenameData` patch.
|
||||
|
||||
What exists today is an **object transaction UUID**, not the target authority:
|
||||
**Implementation status:** this document does not implement or claim distributed generation authority. Existing fencing remains an opt-in coordinator equality recheck. `rustfs/backlog#2251` cannot be completed by forwarding the UUID to disks and adding local CAS. Its implementation must be split at the protocol boundaries in [Required Implementation Boundaries](#required-implementation-boundaries), with the availability and rollout changes reviewed before strict activation. The two original requirements “commit with a quorum while a disk is unreachable” and “every disk immediately rejects every older request” cannot both hold; the precise target below preserves quorum availability.
|
||||
|
||||
| Property | Current implementation |
|
||||
Related contracts remain authoritative for their domains: [erasure-coding.md](erasure-coding.md) defines data durability and voting, [heal-concurrency-model.md](heal-concurrency-model.md) defines namespace-lock scope, [placement-repair-invariants.md](placement-repair-invariants.md) defines placement and repair admission, and [minio-file-format-compat.md](minio-file-format-compat.md) defines format interoperability.
|
||||
|
||||
## Current Guarantee And Counterexamples
|
||||
|
||||
`assign_object_transaction_epoch` generates a random UUID for gated PUT and CompleteMultipartUpload. `FileInfo::set_object_transaction_epoch` in `crates/filemeta/src/fileinfo.rs` stores it under both internal metadata prefixes. `verify_object_transaction_epoch_fence` re-reads quorum metadata before the rename fanout, outside the eventual per-disk mutation critical section. `ObjectTransactionEpochFence::Absent` currently covers both an absent object and existing metadata without a UUID. Cleanup receipts compare UUID equality. None of these operations is a durable distributed CAS.
|
||||
|
||||
The lock implementation already bounds lease validity. `LockLostSignal::is_lost` includes the conservative deadline; `DistributedLockGuard::run_heartbeat` retains prior deadlines after transient RPC failure and reports loss when refresh quorum is no longer valid. `LocalClient` in `crates/lock/src/client/local.rs` keeps `LocalGuardEntry` in an in-memory map; `LockResponse` in `crates/lock/src/types.rs` contains no persisted ballot or accepted object decision. Restarting a lock participant therefore cannot supply the durable order required here.
|
||||
|
||||
These schedules disprove a local UUID-CAS replacement, even assuming a perfect local mutex and atomic metadata replacement. They are protocol counterexamples, not claims that a multi-node fault test has already been run.
|
||||
|
||||
| Schedule | Result and implication |
|
||||
|---|---|
|
||||
| Minting | `assign_object_transaction_epoch` mints a random non-nil UUID for PUT and CompleteMultipartUpload when the object-transaction gate is active. |
|
||||
| Persistence | Written through `FileInfo::set_object_transaction_epoch` into the version's internal metadata map under the dual-key contract (`x-rustfs-internal-*` / `x-minio-internal-*`). |
|
||||
| Fence check | The coordinator reads the current UUID (or `Absent`) and revalidates exact equality immediately before `rename_data`. |
|
||||
| Cleanup | Old-data cleanup receipts carry the committed UUID; reconciliation deletes only when the receipt UUID still equals the current object UUID. |
|
||||
| Four disks start at X; A captures expected X and stalls. B commits B on d1–d3, satisfying W=3. d4 has not heard from B. A reaches d4 with expected X. | d4's exact-CAS accepts A. A durable local `highest_ballot` also accepts if d4 never received B's ballot. Neither mechanism proves rejection on every disk after a quorum commit. |
|
||||
| A changes d1,d2 from X to A; after lock loss B changes d3,d4 from X to B. Each refuses the other's disks because expected X no longer matches. | Neither reaches W=3, even with all disks now reachable. Equality-CAS alone has no rule for choosing recovery, retaining uncertain work, or safely retiring it. This is a recovery/liveness counterexample, not proof of two successful intersecting write quorums. |
|
||||
| A reaches W=3; its reply is lost. A's coordinator restarts and sees a partial or changed disk view. A's rollback runs after B has replaced A. | Timeout is not evidence of abort. Restoring A's backup can erase B unless rollback names its own committed effect and consults a durable decision. |
|
||||
| A checks a lease/UUID; B later commits; A's blocking syscall resumes. | Another coordinator-side check, cancellation token, or process-local mutex cannot establish an atomic cross-node order. The mutation itself must consume the protocol state. |
|
||||
|
||||
This is an equality-CAS fence and cleanup identity. It is not a monotonic epoch, is not minted by the distributed lock grant, and is not compared atomically at each disk's `xl.meta` commit point. Documents and issues must call it the *object transaction UUID*, not proof that the generation authority exists.
|
||||
A lower-ballot write on an isolated stale disk cannot become the authoritative object. A disk that has applied B must never replace B with A. An uncontacted disk may retain older committed materialization until recovery; it must not vote that state as a newer decision or authorize cleanup. Requiring all disks to learn B before acknowledging it would change W to N or require successful isolation of every unreachable disk. That availability change is rejected for the selected design and must not be hidden inside E03's tests.
|
||||
|
||||
### Authority modes (one must be selected)
|
||||
## Authority, Identity, And State
|
||||
|
||||
| Mode | Contract | Persistence requirement |
|
||||
|---|---|---|
|
||||
| Total-ordered fencing epoch | A lock grant returns a durable per-object `(term, counter)`; every disk rejects a lower epoch at the atomic metadata commit point; the value never regresses across lock-plane restart, failover, or minority recovery. | Quorum-persisted before grant, or derived from a durable term whose full comparison cannot regress. The in-memory distributed lock entry alone is insufficient. |
|
||||
| Opaque commit-generation identity | Consumers compare exact identity only; no `<` / `>` semantics. The authoritative commit performs an atomic expected-generation CAS; lease, cleanup, prepared-read, and quota contracts are phrased as "references this exact generation". | Atomic expected-identity comparison plus durable crash recovery. |
|
||||
The authority key is `(bucket incarnation, bucket, object key)`, covering the whole object version set. It is not the S3 version ID. Deleting a noncurrent version, updating tags, or recording replication status can change the authoritative revision while the current S3 version remains the same. The bucket incarnation prevents reuse after bucket deletion/recreation.
|
||||
|
||||
The current UUID proves neither a durable total order nor a per-disk atomic CAS, so it does not decide between the modes.
|
||||
The protocol has separate typed values:
|
||||
|
||||
## Consumer Binding
|
||||
- `Ballot = (configuration epoch, counter, durable proposer ID)`, compared lexicographically only within the specified authority configuration. The proposer persists a counter before use and raises it above every observed promise. Restart never resets it; exhaustion is an error. A different process boot gets a new transport epoch, not permission to reuse a ballot for different bytes.
|
||||
- `Generation = (object revision, operation UUID)`, allocated by a chosen successor decision. Revisions increase from the committed predecessor; UUIDs are compared for equality only. Never infer generation order from modification time, version ID, or UUID bytes.
|
||||
- `DecisionValue = (authority key, predecessor generation, successor generation, operation kind, semantic metadata digest, per-disk prepared metadata/data receipts, outcome identity)`. Disk-specific erasure indices, checksums, and metadata blobs are bound by individual receipts, not assumed byte-identical across disks. The semantic digest includes the full version set and relevant metadata, including fields omitted from ordinary read voting.
|
||||
|
||||
| Consumer | Binds generation how | Key invariant | Current state |
|
||||
|---|---|---|---|
|
||||
| Commit fence (PUT / CompleteMultipartUpload) | Checked at `rename`, rollback restore/delete, and cleanup mutation points using the selected rule | A stale writer is rejected on **all** disks; an already-ACK'd write is never rolled back | Opt-in UUID equality recheck before rename; no per-disk atomic comparison |
|
||||
| Read lease | Lease binds the exact generation observed at read time; GC runs only after every lease on that generation is released | Lease visible across nodes; crashed reader's lease reclaimed by TTL | Streaming/multipart GET holds the namespace read lock through EOF/drop (part-boundary coverage: `#6887`); no cross-node generation-bound registry |
|
||||
| Old-dir GC | Cleanup job carries the committed generation and confirms no lease owns `old_dir` before deleting | `old_dir != committed_dir`; a still-referenced directory is never deleted | UUID receipt equality (`#6077`); no lease consultation |
|
||||
| Prepared pool read | The prepared bundle carries the generation resolved during pool lookup; the chosen pool reuses it only after a match | Mismatch forces fallback to full metadata fanout | `PreparedPoolReadFallbackBarrier` (`#6889`) is a pool-local identity that fails closed / refetches on pool state change; it is not a cross-pool authority |
|
||||
| Quota reservation | Reserve / settle record binds the exact object generation (and the ordered epoch too, if selected) | A late commit cannot settle quota for a different committed generation | Durable per-bucket ledger with independent snapshot-lease fence tokens (`#6058`); not bound to the transaction UUID |
|
||||
A decision does not require a new `FileInfo` positional field. The operation UUID stays in the existing metadata map. Durable authority records and local recovery records carry the revision, predecessor, ballot, and complete decision identity. They use separately versioned records; they cannot be inferred from a version's UUID alone. `xl.meta` remains a recoverable materialization of the chosen decision in strict mode.
|
||||
|
||||
## Fence Coverage: Three Disk-Write Points
|
||||
|
||||
Checking generation only before the `rename` fanout is insufficient. The commit sequence is `tmp sync → data-dir rename → xl.meta commit → directory sync` in `crates/ecstore/src/disk/local.rs`, and `crates/ecstore/src/set_disk/core/io_primitives.rs` has two further detachable disk-write points:
|
||||
|
||||
1. **Rollback restore/delete.** On quorum failure each disk can restore backup metadata or delete the failed version. A stale writer's rollback must compare the expected generation, or it can overwrite or delete the winner's committed metadata. Panic, cancel, and timeout outcomes must be reaped into coordinator convergence rather than skip rollback through an early return.
|
||||
2. **`commit_rename_data_dir`.** A cancel-then-detach disk-write point; the coordinator's "reap all child tasks" must include it so a cancelled writer cannot bypass fence or lease and keep deleting directories.
|
||||
|
||||
If generation is validated only after the data-dir rename, a fenced writer may already have renamed its data-dir into the object path, leaving a staged orphan. Either move the fence ahead of the data-dir rename, or declare that orphan an accepted residue accounted for by GC metrics.
|
||||
|
||||
`RenameConvergence` (`AllSuccessIdentical` / `PartialCommit` / `SignatureDivergent` / `Unknown`) is a *post-commit* heal signal on the same `rename_data` path; the fence is a *commit* gate. They compose: the fence decides whether a convergence is produced, `RenameConvergence` classifies it. A fence-aware convergence variant would be an additive enum change.
|
||||
|
||||
## Transport And Security
|
||||
|
||||
Generation and derived tokens (lease, reservation) cross node boundaries in internode RPC bodies; every such flow must be signature-bound.
|
||||
|
||||
| Rule | Detail |
|
||||
| Input/state | Required treatment |
|
||||
|---|---|
|
||||
| HMAC scope | Target audience, exact service/method, timestamp, nonce, canonical body digest, receiver replay (boot) epoch. The receiver consumes the nonce in a bounded replay cache; a transmitted-but-unconsumed nonce is not replay protection. |
|
||||
| Current substrate | RPC v2/v3 in `crates/ecstore/src/cluster/rpc/http_auth.rs` binds all of the above. Body-bound policy covers mutating disk RPCs including `RenameData`, whose versioned canonical body includes every `RenameDataRequest` field, so the `FileInfo` metadata map carrying the UUID is authenticated. |
|
||||
| Strict switches | `RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT`, `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT`, `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT` (`crates/config/src/constants/internode.rs`) are default-off rollout gates governed by [compat-cleanup-register.md](compat-cleanup-register.md). A generation capability may claim strong transport binding only after the relevant strict modes have converged fleet-wide. |
|
||||
| Acceptance tests per consumer | Method substitution, canonical body tamper, nonce replay, receiver restart, stripped-strict-metadata negatives. |
|
||||
| Object never present | An explicit absent bootstrap state, established during strict cutover; no client-supplied `None` may authorize creation. |
|
||||
| Object deleted, including deletion of its last version | A durable tombstone head with its own revision; never revert to never-present. This prevents ABA and resurrection after a laggard rejoins. |
|
||||
| Null version | A real version-set member, distinct from absent. Replacing/removing it changes the object head. |
|
||||
| Delete marker | A real version-set member and, when latest, a deleted-current state. Preserve marker type and existing S3 behavior. |
|
||||
| Noncurrent version mutation | Compare the whole-object predecessor and the selected version's identity; commit a successor even if the latest S3 version is unchanged. |
|
||||
| Valid legacy metadata with no UUID | Import once during fenced cutover after ordinary quorum/format validation; assign a bootstrap generation in the authority. Never equate this with absent or silently import while old writers are still admitted. |
|
||||
| Missing, malformed, nil, or conflicting dual generation keys after strict enrollment | Typed corrupt/unsupported-state error; retain data for repair. No default to zero, absent, or a new UUID. |
|
||||
| Missing/lagging/replacement disk | A non-authoritative materialization target. Recover the chosen decision and validate its data before it can contribute; do not demand that it already equals the predecessor or blindly overwrite it. |
|
||||
|
||||
## Encoding Rules
|
||||
## Durable Decision Protocol
|
||||
|
||||
| Rule | Reason |
|
||||
Use the existing namespace-lock participant identities and routing, with an explicitly persisted authority configuration. Lock voters and erasure disks are different sets: `Qlock = floor(lock_participants / 2) + 1` decides authority; the existing per-operation `Wdata` decides recoverable object durability. A lock vote is not a shard receipt. One-node deployments still persist their single voter's state. Membership changes cannot be inferred from whichever RPC endpoints answered.
|
||||
|
||||
Each participant persists, per authority key and successor slot, its highest promise, highest accepted `(ballot, DecisionValue)`, and the last learned committed head. Promises and accepted records survive unlock, TTL expiry, restart, and log compaction. Durable records are written through a storage boundary below the object API; writing them through PUT would recursively acquire the same authority. `LocalClient` must not become a filesystem implementation: the lock crate consumes an injected durability interface, while the storage owner implements it.
|
||||
|
||||
The following is a protocol contract, not pseudocode to paste into the current rename implementation:
|
||||
|
||||
1. Acquire the existing object namespace write lock for admission. Read/recover the latest authority head through a fresh quorum promise/recovery barrier; reading only cached learned-head markers is insufficient because a quorum may have accepted a value before its commit notification arrived. Resolve any accepted successor before returning a head or allocating another slot. An unavailable decision quorum is an explicit failure. The lock still prevents ordinary competing work, but it is not the safety proof after lease loss.
|
||||
2. Stage the new shards and exact replacement metadata in transaction-owned paths. Obtain Wdata receipts only after the required file and directory syncs. Receipts bind disk identity/incarnation, authority configuration, key, operation UUID, blob digests, and data directories. Preparing must not replace live metadata, remove old directories, or reuse a winner's directory. Metadata-only/delete operations also stage a recoverable replacement version set.
|
||||
3. For successor slot `head.revision + 1`, obtain Qlock durable promises for a unique ballot. Each response returns its accepted value, if any. Adopt the value with the highest accepted ballot among the promise quorum. Only if none was accepted may the proposer offer its own candidate with that predecessor. A recovered candidate is not replaced merely because its coordinator timed out or its lease expired.
|
||||
4. Validate the candidate's Wdata preparation receipts and predecessor, then obtain Qlock durable accepts of the **same** value at that ballot. A participant accepts only at or above its promise and must reject a different value at the same ballot. A value becomes chosen at Qlock acceptance. Quorum intersection plus adoption of the highest accepted value prevents a different value from being chosen in that slot. Retrying the same operation cannot allocate a second successor.
|
||||
5. Learn/persist the chosen decision and publish it on storage disks through the guarded local recovery protocol below. Return S3 success only after both a durable chosen decision and Wdata durable materializations satisfy the existing operation's rules. Decision chosen but publication incomplete is `OutcomeUnknown/PendingRecovery`, never authorization to roll back the decision. A later proposer first resolves the prior slot before allocating the next one.
|
||||
6. On a lost ACK, resolve by operation UUID and exact request digest. The results are `NotChosen`, `ChosenPendingPublication`, `Committed`, or `SupersededAfterCommit`. Reusing a UUID for different input is invalid. A timeout without a recovered decision stays unknown. Record idempotency outcomes until the client retry horizon and all dependent cleanup/accounting records have passed a durable retirement watermark; old requests then fail as expired rather than being treated as new.
|
||||
|
||||
The decision and payload retention lifetimes are coupled. Accepted/staged data is not garbage merely because no coordinator is alive. A new promise quorum can adopt a previously accepted value and finish it. If the required payload has been physically lost, fail closed and repair; never choose a different value for an already chosen slot. Persisted voter state that is lost or corrupt requires catch-up/replacement, not an empty voter with the same identity.
|
||||
|
||||
This is the minimum extension that turns the lock grant into a recoverable authority. A promise-only grant lacks outcome recovery; per-disk promises without a common decision allow minority uncertainty to escape into reads. The protocol requires review and executable state-machine tests before production integration. It does not imply that the current lock RPC is already a consensus implementation.
|
||||
|
||||
## Single-Disk Publication And Recovery
|
||||
|
||||
Local and remote disks execute the same guarded primitive. A remote RPC handler must decode and authenticate the request, then call that primitive; checking only in the RPC handler leaves local callers and deferred syscalls uncovered. All operations touching the object's metadata, backup, or referenced directories participate.
|
||||
|
||||
Under one object mutation guard, re-read local durable recovery state, verify the decision/configuration and local nonregression condition, record a write-ahead intent, sync it, perform data-directory rename and atomic metadata replacement, sync affected directories, then persist applied outcome. The guard, including ownership of any namespace/deletion lease, stays with the blocking syscall until it completes, even if its caller is cancelled. An async task disappearing must not release a guard while its syscall still runs.
|
||||
|
||||
A single filesystem rename does not atomically commit a sidecar plus `xl.meta`. The write-ahead record binds predecessor/successor identities and exact metadata bytes; startup recovery runs before disk readiness. Recovery replays a chosen intent forward and completes syncs. An unchosen staged operation stays private until authority recovery makes its retirement safe. Old snapshots never overwrite a newer applied local revision. Conflicting bytes for the same decision are corruption. Treat write/fsync errors and torn records as unknown until decoded and reconciled, not as successful rollback.
|
||||
|
||||
A laggard need not contain the predecessor. Recovery fetches the chosen decision and its verified metadata, reconstructs or validates its shards under [erasure-coding.md](erasure-coding.md), and installs that state. An empty replacement uses a fresh disk incarnation and cannot reuse old preparation receipts. A disk whose durable state claims a later decision than the supplied one rejects the operation; a conflicting same-revision digest is quarantined. Never erase a divergent disk merely because it is in the minority.
|
||||
|
||||
`rollback_committed_rename_std`, `rollback_inline_metadata_commit_std`, and `restore_metadata_backup` in `crates/ecstore/src/disk/local.rs` must become decision-aware before strict mode includes them. The permitted rollback is limited to a transaction's unchosen private preparation, or restoration proven by recovery to be necessary before any newer local effect. A chosen operation is repaired forward. Neither a client timeout nor a rename-tail error permits reverting an acknowledged decision.
|
||||
|
||||
`RenameConvergence` remains a post-publication repair signal. `PartialCommit`, `SignatureDivergent`, and `Unknown` do not decide which transaction won. Keep their diagnostics and quorum accounting; resolve authority first. Early ACK may still precede minority-tail completion after the two quorum conditions hold. Tests that inspect all disks must synchronize the tail or assert the permitted minority residue separately.
|
||||
|
||||
## Writer Participation
|
||||
|
||||
Every semantic metadata change advances the whole-object generation, including metadata-only writes. A physical repair that reproduces exactly the already chosen bytes preserves the generation and consumes that chosen decision; it must not create a new semantic value. The table specifies participation, not a generated inventory of every call site.
|
||||
|
||||
| Writer and current code boundary | Required generation behavior |
|
||||
|---|---|
|
||||
| **Do not bump `XL_META_VERSION` or `XL_HEADER_VERSION`** (`crates/filemeta/src/filemeta.rs`). | `decode_xl_headers` in `crates/filemeta/src/filemeta/codec.rs` rejects newer values outright; a bump makes every new `xl.meta` unreadable by rolling-upgrade old nodes and by MinIO. See [minio-file-format-compat.md](minio-file-format-compat.md). |
|
||||
| **Do not add generation as a `FileInfo` struct field.** | Internode RPC serializes `FileInfo` with two msgpack encoders: positional-array encoding for the `read_version` family (a new positional field breaks mixed-version decode) and `encode_msgpack_named` (named-map) for `rename_data` in `rustfs/src/storage/rpc/node_service/disk.rs`. A field would have to be correct under both plus the JSON compatibility twin. Use the metadata map, which rides every encoder unchanged. |
|
||||
| **Metadata-map dual key.** | The UUID lives under `x-rustfs-internal-*` / `x-minio-internal-*`; missing, malformed, nil, or conflicting dual values fail closed when fencing is active. |
|
||||
| **No sidecar unless atomic.** | An epoch sidecar outside `xl.meta` is admissible only if it commits at the same atomic/CAS point as `xl.meta` with a specified crash-recovery protocol. None is implemented. |
|
||||
| **Regression guard.** | The real-MinIO `xl.meta` interop fixtures in `crates/filemeta/src/filemeta.rs` must keep passing: objects written by a new node stay readable by old RustFS nodes and by MinIO in both upgrade directions. |
|
||||
| PUT / data COPY: `put_object_with_old_current_size_inner`, `copy_object` in `crates/ecstore/src/set_disk/ops/object.rs` | Stage, choose, publish a successor; preserve source read protection. A metadata-only COPY is also a semantic successor, even if data directories are shared. |
|
||||
| MPU: `complete_multipart_upload`, `new_multipart_upload`, `abort_multipart_upload` in `crates/ecstore/src/set_disk/ops/multipart.rs` | Complete chooses the destination object's successor. Part staging/upload metadata and abort remain in the upload namespace; they cannot delete a directory transferred to a chosen object decision. |
|
||||
| DELETE, batch DELETE, null/marker removal: `delete_object`, `delete_objects_with_accounting`, `delete_object_version` in `crates/ecstore/src/set_disk/ops/object.rs`; lifecycle callers in `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs` | Each object has its own predecessor/decision; retain tombstone authority after the last version. Prefix deletion must enumerate decisions or prove a bucket-incarnation retirement barrier; a recursive bypass is forbidden in strict mode. |
|
||||
| Heal: `heal_object_with_explicit_version_regen` in `crates/ecstore/src/set_disk/ops/heal.rs` | Exact repair preserves the chosen generation and verifies full metadata identity. A version-list or semantic metadata change needs a successor. `no_lock` may skip admission only; it cannot bypass authority. In-place directory repair cannot remove a reader's live directory. |
|
||||
| Transition / restore: `transition_object`, `restore_transitioned_object`, `put_object_metadata` in `crates/ecstore/src/set_disk/ops/object.rs`; `finalize_restore_metadata`, `update_restore_metadata` in `crates/ecstore/src/set_disk/replication.rs` | Each metadata transition is a successor, preserving existing operation-ID, remote tuple, and tier lease checks. Bind the transition transaction to the exact predecessor/successor; a late finalizer cannot rebase onto another restore operation. |
|
||||
| Replication status and metadata/tag/retention writeback: `put_object_metadata`, `put_object_tags`, `delete_object_tags`, `merge_replication_metadata_lww` in `crates/ecstore/src/set_disk/ops/object.rs`; callers in `crates/ecstore/src/bucket/replication/replication_resyncer.rs` | Commit a field-scoped successor conditional on the exact version/content identity. On conflict, reload and revalidate the mutation; never replay a full stale `FileInfo`. Existing LWW category rules remain applicable within that validation. |
|
||||
| Rebalance/decommission: `migrate_entry_version` in `crates/ecstore/src/services/rebalance/migration.rs`; `decommission_tier_free_version`, `decommission_tiered_object` in `crates/ecstore/src/set_disk/mod.rs`; `crates/ecstore/src/data_movement/mod.rs` | The authority key and lock group remain stable across pools. Stage the destination, choose the location/ownership successor, then retire the exact source receipt. Do not mint independent source and destination authorities. Existing placement and tier ownership fences remain required. |
|
||||
| Generic metadata entry points: `write_unique_file_info`, `update_object_meta_with_opts` in `crates/ecstore/src/set_disk/core/io_primitives.rs`; `LocalDisk::write_metadata`, `update_metadata`, `delete_version`, `delete_versions_internal`, `write_all_meta` in `crates/ecstore/src/disk/local.rs` | Consume a validated decision/recovery context or reject strict writes to enrolled objects. None may invent a generation, reset it through `fresh`, replace corrupt metadata with an unproven empty version set, or bypass durability with `no_persistence`. |
|
||||
| Rollback and GC: `rename_data_owned_with_fence`, `commit_rename_data_dir`, `reclaim_orphan_data_dirs` in `crates/ecstore/src/set_disk/core/io_primitives.rs`; `reconcile_old_data_cleanup_receipts` in `crates/ecstore/src/set_disk/ops/object.rs` | Consume the owning decision and exact directory references. Cleanup does not change object contents or grant a new semantic generation. It must use a durable retirement decision and local reader/deletion guards. |
|
||||
|
||||
### Wire-encoding window (JSON and msgpack)
|
||||
Strict capability is withheld until every raw writer in `DiskAPI`, its local implementation, `DiskStore`, remote adapters, and server handlers has an enforced path. Internal authority persistence must use its own narrow storage primitive, not evade this rule by recursively calling generic object metadata writes.
|
||||
|
||||
- Dual-encoded RPC fields exist twice in `crates/protos/src/node.proto`: a JSON `string` field and a msgpack `bytes *_bin` field (e.g. `file_info` and `file_info_bin` on `RenameDataRequest`). Senders emit both; receivers (`decode_msgpack_or_json` in `crates/ecstore/src/cluster/rpc/remote_disk.rs`) prefer `_bin` and fall back to JSON only when `_bin` is empty.
|
||||
- `rustfs_protos::internode_rpc_msgpack_only()` drops the JSON copy only when both `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY` and `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` are set after the JSON-fallback metric reads zero fleet-wide.
|
||||
- Generation inside the `FileInfo` metadata map is carried in both copies automatically. Any new *top-level* generation datum must be added to both encodings and be safe under both msgpack encoders; a field in only one encoding is silently lost when a peer falls back.
|
||||
- `RenameDataRequest` has a versioned, injective canonical-body encoder covering both compatibility fields; a strict generation-capable request must reject missing or mismatched canonical-body metadata rather than downgrade to the unauthenticated JSON twin.
|
||||
## Reads, Garbage Collection, And Accounting
|
||||
|
||||
### Proto evolution
|
||||
Strict reads need the chosen head, not a majority of arbitrary prepared/live UUIDs. Resolve the decision under the namespace read lock before accepting object metadata; validate the selected current or explicit version against it, and wait for or repair missing materialization. HEAD, GET, ListObjects/ListObjectVersions, scanner reads used for deletion, and prepared pool reads all need this distinction. A query may return an error while a chosen write is recovering; it must not expose an unchosen candidate or resurrect a retired version. This read-decision adapter is part of the strict-mode scope and is a reason E03 is larger than disk CAS.
|
||||
|
||||
No top-level proto field is required by the metadata-map UUID. If an ordered epoch or explicit expected-generation is ever added to proto, it uses **proto3 `optional`** (explicit presence). A non-optional scalar is forbidden: an old coordinator talking to a new disk decodes absence as a plausible zero.
|
||||
Keep the current namespace read lock through EOF/drop, including multipart part boundaries. This design does not replace it with a new cross-node generation lease registry. A strict implementation must also bind every deferred local/remote part open to the resolved generation and acquire protection on the disk that owns the directory before handing the read capability out. A reader that loses authority or cannot renew its disk protection must fail before another open; it cannot continue on an unvalidated cached pathname.
|
||||
|
||||
## Mixed-Version Gate: One Direction
|
||||
`LocalDisk::acquire_snapshot_lease`, `renew_snapshot_lease`, `release_snapshot_lease`, and `delete_data_dir` in `crates/ecstore/src/disk/local.rs` provide disk-local path protection and deletion deferral. They are not proof of a fleet-wide object generation. Before reuse, their token must bind disk incarnation and exact generation/directory, and strict reads must reject a pre-restart token. Existing open file descriptors may finish reading an unlinked inode, but later part opens need a valid protected generation. This preserves streaming behavior without assuming a local mutex protects another node.
|
||||
|
||||
When generation enforcement is not explicitly requested, or fleet confirmation is absent, behavior falls back to current semantics. Fail-closed is reserved for an explicit administrator-confirmed strict rollout.
|
||||
GC consumes a durable retirement authorization for exact directories no longer referenced by **any** retained version or pending accepted decision. Include retirement in the chosen successor that removes the last reference; if it was not recorded there, choose a metadata-neutral successor that records it before deletion. That successor advances the authority revision while preserving the S3 version contents; the cleanup syscall itself never mints an identity. Retirement prevents future repairs/reads from creating new references; a new reference requires a new decision and cannot revive a retired directory. At the destructive syscall, hold the object/directory guard, recheck the local chosen metadata references, `old_dir != committed_dir`, retirement/configuration identity, and local snapshot protection. Any uncertainty defers deletion. A stale cleanup receipt matching an earlier UUID is not enough. Across executor restart, replay the same retirement ID idempotently; do not convert a lost reply into a broader recursive delete.
|
||||
|
||||
| Flag (`crates/config/src/constants/object.rs`) | Default | Effect |
|
||||
|---|---|---|
|
||||
| `RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE` | false | With either flag absent, PUT/MPU neither persists nor consumes the transaction UUID. |
|
||||
| `RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED` | false | With both enabled, failure to obtain or retain the live fleet proof rejects the commit before rename. |
|
||||
Prepared pool reads remain a separate optimization domain. In `crates/ecstore/src/store/rebalance.rs`, `prepare_latest_object_metadata_with_idx` collects candidates and revalidates a refetched winner with `validate_prepared_pool_refetch_identity` from `crates/ecstore/src/store/rebalance/support.rs`. `PreparedPoolReadFallbackBarrier` is a `#[cfg(test)]` scheduling fixture, not a production identity. Keep the all-pool resolution rules until a chosen location decision supplies equivalent evidence. The prepared bundle binds authority generation plus pool identity; mismatch requires a complete refetch or a typed failure, never selection of a different generation using the old bundle.
|
||||
|
||||
The fleet proof is currently borrowed from the remote-version-state writer rollout. It proves membership/process-epoch convergence for that feature only; it does not prove an epoch type, per-disk CAS support, or RPC strict-mode convergence, and must not be treated as the final generation handshake.
|
||||
Quota remains a separate per-bucket arbitration domain. `QuotaLedger` and `settle` in `crates/ecstore/src/bucket/quota/reservation.rs` key reservations by operation UUID, validate object/size, and update under the ledger fence. A late settlement cannot remove a different reservation key; an absent/mismatched key fails or follows the existing idempotent abort rule. That proves key isolation, not that current generation A committed or that a ledger storage write is immune to stale-disk mutation.
|
||||
|
||||
### Capability negotiation (target)
|
||||
For strict mode, add the decision identity to the reservation/settlement binding. Settle a committed **historical** decision even if it has since been superseded, but only against its original reservation and recorded old/new sizes; demanding that it still be current would leak valid reservations. Abort only a recovered unchosen/retired operation. Unknown outcomes stay reserved and reconcile. Retain the existing conservative usage floor and `commit_started` recovery behavior. Ledger writes themselves require the authority protocol, with a documented lock order and no recursion through their own reservation path. No quota token is compared numerically to an object ballot or used to revoke a newer object's leases.
|
||||
|
||||
Generation enforcement requires one **live fleet proof** containing at least: the selected authority version and comparison mode; the current membership/topology fingerprint and process epochs; support for every required disk mutation point; RPC signature/body/replay strict convergence; and the on-disk encoding version (the metadata-map UUID is version 1). Membership change or an old-node rejoin revokes the proof; revocation before commit fails an explicitly strict request and never rewrites or lowers a persisted generation. The proof may extend the authenticated fleet-proof machinery in `notification_sys` or the runtime capability contract; this document requires one shared token, not a mechanism.
|
||||
## Restart, Membership, And Rollout
|
||||
|
||||
## Open Decisions
|
||||
Authority configuration is durable and includes participant identities, routing, bucket incarnation, protocol version, and quorum rules. Changing storage pool placement must not remap the authority key. A replaced voter starts as a non-voter, catches up durable promise/accepted/chosen state, and only joins through a quorum-approved configuration transition. Configuration change requires intersecting old/new decision quorums; losing the old quorum is a recovery incident, not permission to bootstrap a new empty authority. Offline data disks rejoin through generation-aware catch-up, independent of voter admission.
|
||||
|
||||
Blockers for calling the contract implemented:
|
||||
A live capability proof must bind the authority configuration, topology, every participant process boot epoch, disk incarnations, writer/read/recovery protocol support, encoding version, and RPC signature/body/replay strictness. Restart, membership change, disk replacement, protocol downgrade, or a strict-transport setting change revokes it. Receivers revalidate before entering the publication critical section; accepted durable decisions survive proof revocation and are recovered under a fresh valid proof, never replayed as unvalidated requests. The remote-version-state fleet proof does not prove any of these generation capabilities.
|
||||
|
||||
1. **Authority mode.** Total order or opaque exact-CAS. Do not retrofit ordering semantics onto the existing random UUID.
|
||||
2. **Complete `xl.meta`-writer coverage.** Enumerate commit rename, rollback restore/delete, cleanup, heal, transition, restore, replication, and data movement; each path compares/carries the selected generation or is proved incapable of replacing the authoritative identity.
|
||||
3. **Rollback as expected-generation CAS.** The quorum-failure rollback in `rename_data` restores backup metadata, not just a private temp file; it must run only when the stored generation still matches the failed writer's expectation.
|
||||
4. **Generation capability proof.** Extend the fleet proof or the runtime capability contract; one revalidatable token.
|
||||
5. **Read-lease and GC crash recovery.** Cross-node registry, TTL reclamation, lease-holder crash behavior, GC-executor recovery.
|
||||
6. **Quota reserve → commit → settle binding.** Relate the ledger's independent mutation tokens to the selected generation, with a concrete late-settle rejection test, or prove the fence is a separate arbitration domain that cannot cross-settle.
|
||||
7. **Prepared reads stay pool-local.** `PreparedPoolReadFallbackBarrier` validates freshness only within the pool that produced it; cross-pool ordering requires a common authority, and the multi-pool wait cannot be short-circuited without one.
|
||||
8. **Hot-path cost is a blocking metric.** Measure any added consensus write, fsync, fleet-proof lookup, lease operation, or centralized serialization under 4 KiB and hot-key/hot-bucket A/B.
|
||||
9. **Test infrastructure.** Multi-node, multi-pool, directed network-fault, and large-object budget for restart, mixed-version, and cross-node lease acceptance.
|
||||
The existing `RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE` and `RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED` flags in `crates/config/src/constants/object.rs` retain their current default-off behavior. They do not become a claim that the new protocol exists. If generation strictness is explicitly selected, missing capability is an error, never silent downgrade. No new environment variable is introduced by this document; a production gate must be documented with its implementation.
|
||||
|
||||
Strict enrollment requires quiescing old writers and readers for the enrolled namespace, recovering ambiguous operations, validating/importing legacy heads, persisting a strict-format/protocol marker, and enabling the complete fleet. New disks reject unbound legacy mutation RPCs for that namespace. Old binaries must be prevented from opening a strict-enrolled drive by a startup compatibility gate they understand before enrollment; an environment flag known only to new binaries is insufficient. Until that prerequisite is deployed, do not activate strict mode in a mixed fleet. Disabling flags after enrollment cannot drop durable authority; downgrade requires a separately verified quiescent materialization/export operation. Ordinary un-enrolled compatibility deployments keep their current behavior.
|
||||
|
||||
## Encoding And Transport
|
||||
|
||||
- Do not bump `XL_META_VERSION` or `XL_HEADER_VERSION` in `crates/filemeta/src/filemeta.rs`. Do not add fields to positional-msgpack `FileInfo`; carry the UUID through the metadata map and protocol records through explicit versioned envelopes.
|
||||
- Write RustFS/MinIO internal metadata dual keys using `crates/utils/src/http/metadata_compat.rs`. Reject conflicting, nil, or malformed generation values; validate every persisted/RPC record again at consumption.
|
||||
- New proto values in `crates/protos/src/node.proto` require explicit presence (`optional` scalars or a present message), including absent/tombstone state. Bind expected/new generation, ballot, receipts, configuration, and outcome identity in the canonical body. The JSON and msgpack representations must carry identical semantics; absent data from an old peer cannot decode as a valid zero ballot.
|
||||
- `crates/ecstore/src/cluster/rpc/remote_disk.rs` and `rustfs/src/storage/rpc/node_service/disk.rs` must share local protocol behavior. Extend canonical encoders for every affected mutation, not only `RenameData`. Authenticate both compatibility representations and reject disagreement rather than falling back to a weaker JSON twin.
|
||||
- `crates/ecstore/src/cluster/rpc/http_auth.rs` supplies signature, canonical-body, and replay-scope checks. Strict generation capability requires fleet convergence of `RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT`, `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT`, and `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT` from `crates/config/src/constants/internode.rs`. Cover method substitution, body tamper, stripped metadata, consumed nonce replay, and receiver restart.
|
||||
- Preserve real-MinIO metadata fixture decoding and supported old-RustFS compatibility before strict enrollment. Do not promise that a live MinIO binary can start a RustFS-written drive set; [minio-file-format-compat.md](minio-file-format-compat.md) explicitly excludes that direction. Strict authority records also impose a new deployment boundary even though the `xl.meta` container version is unchanged.
|
||||
|
||||
## Required Failure Outcomes
|
||||
|
||||
Every scenario must first prove its intended barrier, quorum, or crash point was reached, then inspect authority records, decoded metadata, directory references, return/error categories, and full GET bytes. Single-process barriers are insufficient evidence for the network-partition cases.
|
||||
|
||||
| Interleaving | Unique permitted outcome |
|
||||
|---|---|
|
||||
| A pauses after coordinator verification; A loses lock quorum while data RPC remains reachable; B chooses and publishes; A resumes rename. | A cannot become the chosen successor for B's predecessor or overwrite B on a disk that applied B. Isolated older materialization cannot vote as current. Read B exactly; reconcile laggards under B's decision. |
|
||||
| A's publication result is unknown; B succeeds; A's rollback/cleanup resumes. | Recover A's decision. Never restore/delete B's metadata or referenced data; retire only A-owned unchosen paths or separately authorized dead directories. |
|
||||
| Reader acquires generation G, consumes part 1; replacement retires G; GC attempts to delete part 2's directory. | The reader's valid disk protection defers GC; otherwise the reader fails before its next open. Never silently serve another generation or delete a directory still covered by valid protection. Repeat across reader and disk restarts. |
|
||||
| A and B each stage or partially publish on two of four disks; one coordinator dies; all disks return. | The promise/accept recovery rule preserves any chosen value or adopts the highest accepted candidate and finishes that slot. No guessing from UUID order, no permanent exact-CAS split, no replacement of a chosen value. |
|
||||
| Process dies after staging sync, intent sync, data rename, metadata replace, directory sync, or accepted/decision reply. | Reopen durable records before readiness. Unchosen work stays private; chosen work is replayed forward; lost ACK resolves to the original operation identity. Torn/insufficient evidence fails closed. |
|
||||
| Null version becomes a delete marker; a noncurrent version is deleted; delayed heal/metadata write resumes. | Whole-object predecessor no longer matches. Exact repair uses the chosen version set; no resurrection, marker-to-object conversion, or reset to never-present. |
|
||||
| Old coordinator reaches a new strict disk, new coordinator reaches an old disk, voter restarts, or transport strictness changes. | Compatibility behavior only in an un-enrolled namespace. Strict admission fails until a fresh complete proof and supported startup gate exist; no zero/missing-field fallback. |
|
||||
| Replacement disk is empty, or a restored minority disk has an old promise and old metadata. | It cannot vote as an initialized authority. Catch up chosen state and reconstruct data under a fresh incarnation; stale requests cannot bypass enrollment by presenting absent metadata. |
|
||||
| Quota settlement for A arrives after B commits; prepared pool refetch sees B instead of A. | Only A's original chosen outcome may settle A's reservation; B is unaffected. Prepared A cannot supply metadata/data for B without a new validated preparation. |
|
||||
|
||||
## Required Implementation Boundaries
|
||||
|
||||
These are durable ownership and acceptance boundaries, not permission to close the disk-fencing work before the protocol exists. The decision-model and availability changes require architecture review before production implementation. No unrelated external consensus service or full-manifest rewrite is authorized by this contract.
|
||||
|
||||
| Boundary | Required implementation and exit evidence |
|
||||
|---|---|
|
||||
| Durable authority substrate | Injected lock-participant persistence; typed ballot/configuration/decision values; prepare/accept/recover state machine; restart-safe proposer identities; corruption and voter replacement handling. Model/exhaustively test two competing proposers, lost replies, minority recovery, and every durable transition. The same-slot different-value property must be impossible. |
|
||||
| Disk publication boundary | Separate private preparation from publication; implement write-ahead intent, decision receipts, atomic guarded mutation, idempotent recovery, and directory retirement. Include inline/non-inline, every crash point, canceled blocking syscalls, ACK loss, and empty/lagging disks. Existing rollback helpers cannot remain an unguarded alternate route. |
|
||||
| Writer and read integration | Route every writer in the table and every strict read/scan decision through the authority; preserve data quorum and S3 version semantics. Bind prepared reads, MPU ownership transfer, tier operations, and quota outcomes. Demonstrate no raw metadata entry point bypasses strict mode. |
|
||||
| Fleet activation | Deploy the startup downgrade barrier first; import legacy/absent heads during quiescence; implement configuration/proof revocation and both RPC encodings. Run real multi-node lock/data-plane partitions and mixed-binary/restart tests. Only then can strict E03 acceptance run and activation be considered. |
|
||||
|
||||
The conservative immediate action is to keep the existing compatibility behavior and improve its local convergence/recovery independently. Those fixes must describe their smaller guarantee and must not advertise E03's distributed safety. A strict-only local CAS helper can be built behind the inactive capability boundary, but it cannot enable the feature or close the authority work.
|
||||
|
||||
## Performance And Activation Criteria
|
||||
|
||||
Measure the existing implementation and the full proposed path on identical machines, disk/filesystem, durability settings, network, object population, concurrency, and warmup. Include single hot-key and many-key 4 KiB PUT, 1 MiB PUT, metadata-only writes, and CompleteMultipartUpload with fixed part counts. Report throughput, p50/p95/p99, peak retained preparation/recovery bytes, recovery time, per-operation RPCs/fsyncs, and the object mutation critical-section duration. Include a slow minority disk, one voter loss, and restart recovery; a throughput result alone is insufficient.
|
||||
|
||||
The unoptimized proposal adds a lock-quorum promise round and an accept round with durable writes, plus decision learning/publication and local intent/applied-state persistence. Wdata staging remains separate. Read resolution may add an authority quorum round. Record actual overlapping rounds and fsync group commits; do not claim these costs disappear because the existing lock RPC is reused. Never hold a global lock across shard I/O, wait for all disks on the successful path, or weaken fsync/bitrot/quorum to recover throughput.
|
||||
|
||||
Activation requires all failure scenarios to pass with no acknowledged-data loss or wrong-generation read; no unexplained RPC/fsync amplification beyond the implemented phase budget; and an explicit performance acceptance recorded with the review. Use a conservative review trigger of more than 10% throughput loss or 15% p99 growth in any fixed-workload comparison: exceeding it blocks default activation until the architecture/operations owners accept the measured tradeoff or the implementation removes it. These are proposed rollout budgets, not measurements or performance claims. Without a reproducible baseline, leave strict mode unavailable.
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
| Persisted free-version scan and re-enqueue after local-first expiry | `crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs` |
|
||||
| Fenced free-version remote delete, local-marker cleanup, and rescan | `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs` (`cleanup_free_version_exact`) |
|
||||
| Durable manual transition job/task/result records | `crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs` |
|
||||
| Dormant tier validation probe intent format and read-only core inspection | `crates/ecstore/src/services/tier/tier_probe_intent.rs` |
|
||||
| Manual run/status/cancel and transition-transaction reconcile admin routes | `rustfs/src/admin/handlers/ilm_transition.rs` |
|
||||
| `ObjectInfo` / `TransitionedObject` types | `crates/ecstore/src/object_api/types.rs` |
|
||||
| `FileMeta` / `FileInfo` / version metadata | `crates/filemeta/src/` |
|
||||
@@ -119,6 +120,12 @@ rc admin ilm transition run local/mybucket --prefix logs/ --tier cold --dry-run
|
||||
rc admin ilm transition run local/mybucket --prefix logs/ --tier cold --max-objects 1000 --max-duration-seconds 30
|
||||
```
|
||||
|
||||
## Validation probe crash recovery status
|
||||
|
||||
Tier Add, Edit, and Verify currently validate a destination with a unique `rustfs-tier-probe-<uuid>` object and perform bounded compensation while the process remains alive. The `rustfs-tier-probe-intent-v1` decoder, canonical durable namespace, conditional storage primitives, state machine, and crate-level inspection type are present only as a dormant foundation. No validation path writes this record, no startup or periodic recovery scans it, and no admin HTTP route exposes it. V1 requires the owner to remain exactly equal to the immutable creator; takeover would require a new schema with explicit proof. Both durable writing and destructive recovery remain disabled until the fleet capability, operation-generation revalidation, provider timeout, retention, and operator contracts are approved.
|
||||
|
||||
Do not search the internal metadata bucket for these records as evidence that validation is crash recoverable: a current server does not create them. If a process is killed after the remote probe PUT but before cleanup, inspect the destination provider manually and retain ambiguous candidates. Never delete an empty or guessed version, and do not hand-create a probe intent to authorize cleanup.
|
||||
|
||||
Inspect the aggregate counters before widening scope. Full object-key lists are intentionally not returned. If `RUSTFS_RPC_SECRET` or other credentials were pasted into an issue, chat, log, or ticket while debugging tiering, rotate them on every node, restart the cluster with the new value, and redact the exposed copy before sharing more diagnostics.
|
||||
|
||||
## Reconcile an unknown transition upload
|
||||
|
||||
Reference in New Issue
Block a user