mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-12 21:22:19 +00:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4a1cb4eb42 | |||
| b70a198df5 | |||
| 030f9a2175 | |||
| b1f9fab6ed | |||
| 6ec482d950 | |||
| e2fb22d114 |
@@ -438,6 +438,13 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn write_format_control(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::WriteFormatControlRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::WriteAllResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn compare_and_update_file(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::CompareAndUpdateFileRequest>,
|
||||
|
||||
@@ -492,7 +492,7 @@ pub mod disk {
|
||||
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
|
||||
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken,
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
|
||||
validate_batch_read_version_item_count,
|
||||
validate_batch_read_version_item_count, validate_format_control_path,
|
||||
};
|
||||
pub use crate::disk::{
|
||||
BackgroundMemoryBudget, PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler,
|
||||
|
||||
@@ -1309,7 +1309,7 @@ fn tonic_rpc_metric_operation(path: &str) -> &'static str {
|
||||
Some("ReadMultiple") => INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
Some("ReadVersion") => INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
Some("BatchReadVersion") => INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
Some("WriteAll") => INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
Some("WriteAll" | "WriteFormatControl") => INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
Some("CompareAndUpdateFile") => INTERNODE_OPERATION_GRPC_COMPARE_AND_UPDATE_FILE,
|
||||
Some("Lock") => INTERNODE_OPERATION_GRPC_LOCK,
|
||||
Some("UnLock") => INTERNODE_OPERATION_GRPC_UNLOCK,
|
||||
@@ -3195,6 +3195,10 @@ mod tests {
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/WriteAll"),
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/WriteFormatControl"),
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL
|
||||
);
|
||||
assert_eq!(
|
||||
tonic_rpc_metric_operation("/node_service.NodeService/CompareAndUpdateFile"),
|
||||
INTERNODE_OPERATION_GRPC_COMPARE_AND_UPDATE_FILE
|
||||
|
||||
@@ -77,8 +77,8 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteAllResponse,
|
||||
WriteFormatControlRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
@@ -273,6 +273,35 @@ fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result
|
||||
SnapshotLeaseToken::from_slice(&response.token)
|
||||
}
|
||||
|
||||
fn format_control_write_response(response: WriteAllResponse) -> Result<()> {
|
||||
match (response.success, response.error) {
|
||||
(true, None) => Ok(()),
|
||||
(false, Some(error)) if DiskError::from_u32(error.code).is_some() => Err(error.into()),
|
||||
_ => Err(DiskError::FileCorrupt),
|
||||
}
|
||||
}
|
||||
|
||||
// One dispatched format-control attempt ends at its validated durable reply.
|
||||
// Keeping response validation in the measured future classifies remote and
|
||||
// malformed replies as failures and accounts for cancellation through Drop.
|
||||
async fn complete_format_control_write(
|
||||
data_len: usize,
|
||||
future: impl std::future::Future<Output = std::result::Result<tonic::Response<WriteAllResponse>, tonic::Status>>,
|
||||
) -> Result<()> {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
|
||||
measure_client_rpc(INTERNODE_OPERATION_GRPC_WRITE_ALL, async {
|
||||
let response = match future.await {
|
||||
Ok(response) => response.into_inner(),
|
||||
Err(status) if status.code() == Code::Unimplemented => return Err(DiskError::MethodNotAllowed),
|
||||
Err(status) => return Err(status.into()),
|
||||
};
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_sent_bytes(data_len);
|
||||
format_control_write_response(response)
|
||||
})
|
||||
.await
|
||||
.inspect_err(|_| crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_error())
|
||||
}
|
||||
|
||||
fn conditional_file_update_from_wire(outcome: i32) -> Result<ConditionalFileUpdate> {
|
||||
match CompareAndUpdateFileOutcome::try_from(outcome) {
|
||||
Ok(CompareAndUpdateFileOutcome::CompareAndUpdateFileUpdated) => Ok(ConditionalFileUpdate::Updated),
|
||||
@@ -4935,6 +4964,30 @@ impl DiskAPI for RemoteDisk {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_format_control(&self, path: &str, data: Bytes) -> Result<()> {
|
||||
crate::disk::validate_format_control_path(path)?;
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let data_len = data.len();
|
||||
let disk = self.disk_ref().await;
|
||||
let mut client = self.get_client().await.inspect_err(|_| {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_error();
|
||||
})?;
|
||||
let mut request = Request::new(WriteFormatControlRequest {
|
||||
disk,
|
||||
path: path.to_owned(),
|
||||
data,
|
||||
});
|
||||
let canonical = rustfs_protos::canonical_write_format_control_request_body(request.get_ref())
|
||||
.map_err(|_| DiskError::InvalidPath)?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
complete_format_control_write(data_len, client.write_format_control(request)).await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn compare_and_update_file(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -5076,6 +5129,132 @@ impl DiskAPI for RemoteDisk {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn format_control_attempt_metrics_count_each_outcome_once() {
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use std::future::{Future, pending};
|
||||
use std::task::{Context, Poll, Waker};
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let enabled = rustfs_io_metrics::metrics_enabled();
|
||||
rustfs_io_metrics::set_metrics_enabled(true);
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("metrics runtime");
|
||||
runtime.block_on(async {
|
||||
super::complete_format_control_write(7, async {
|
||||
Ok(tonic::Response::new(super::WriteAllResponse {
|
||||
success: true,
|
||||
error: None,
|
||||
}))
|
||||
})
|
||||
.await
|
||||
.expect("durable reply succeeds");
|
||||
let error = super::complete_format_control_write(9, async {
|
||||
Ok(tonic::Response::new(super::WriteAllResponse {
|
||||
success: false,
|
||||
error: Some(super::DiskError::FileNotFound.into()),
|
||||
}))
|
||||
})
|
||||
.await
|
||||
.expect_err("typed remote failure is measured as an error");
|
||||
assert!(matches!(error, super::DiskError::FileNotFound));
|
||||
let error = super::complete_format_control_write(11, async { Err(tonic::Status::unimplemented("old peer")) })
|
||||
.await
|
||||
.expect_err("unsupported transport fails closed");
|
||||
assert!(matches!(error, super::DiskError::MethodNotAllowed));
|
||||
let mut cancelled = Box::pin(super::complete_format_control_write(13, pending()));
|
||||
assert!(matches!(cancelled.as_mut().poll(&mut Context::from_waker(Waker::noop())), Poll::Pending));
|
||||
drop(cancelled);
|
||||
});
|
||||
});
|
||||
rustfs_io_metrics::set_metrics_enabled(enabled);
|
||||
let entries = snapshotter.snapshot().into_vec();
|
||||
let value = |name: &str| {
|
||||
&entries
|
||||
.iter()
|
||||
.find(|(key, _, _, _)| {
|
||||
key.key().name() == name
|
||||
&& key
|
||||
.key()
|
||||
.labels()
|
||||
.any(|label| label.key() == "operation" && label.value() == "grpc_write_all")
|
||||
})
|
||||
.expect("format-control metric")
|
||||
.3
|
||||
};
|
||||
for (name, expected) in [
|
||||
("rustfs_system_network_internode_operation_requests_outgoing_total", 4),
|
||||
("rustfs_system_network_internode_operation_sent_bytes_total", 16),
|
||||
("rustfs_system_network_internode_operation_errors_total", 2),
|
||||
] {
|
||||
assert!(matches!(value(name), DebugValue::Counter(count) if *count == expected), "{name}");
|
||||
}
|
||||
assert!(
|
||||
matches!(value("rustfs_system_network_internode_operation_duration_ms"), DebugValue::Histogram(samples) if samples.len() == 4)
|
||||
);
|
||||
let mut outcomes = std::collections::HashMap::new();
|
||||
for (key, _, _, sample) in &entries {
|
||||
if key.key().name() == "rustfs_system_network_internode_rpc_completions_total" {
|
||||
let outcome = key
|
||||
.key()
|
||||
.labels()
|
||||
.find(|label| label.key() == "outcome")
|
||||
.expect("outcome label");
|
||||
let DebugValue::Counter(count) = sample else {
|
||||
panic!("completion counter");
|
||||
};
|
||||
outcomes.insert(outcome.value(), *count);
|
||||
}
|
||||
if key.key().name() == "rustfs_system_network_internode_rpc_active" {
|
||||
assert!(matches!(sample, DebugValue::Gauge(value) if value.into_inner() == 0.0));
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
outcomes,
|
||||
std::collections::HashMap::from([("success", 1), ("error", 2), ("cancelled", 1)])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_control_requires_explicit_durable_success_response() {
|
||||
use super::{DiskError, WriteAllResponse, format_control_write_response};
|
||||
format_control_write_response(WriteAllResponse {
|
||||
success: true,
|
||||
error: None,
|
||||
})
|
||||
.expect("explicit durable success");
|
||||
for response in [
|
||||
WriteAllResponse {
|
||||
success: false,
|
||||
error: None,
|
||||
},
|
||||
WriteAllResponse {
|
||||
success: true,
|
||||
error: Some(DiskError::FileNotFound.into()),
|
||||
},
|
||||
WriteAllResponse {
|
||||
success: false,
|
||||
error: Some(rustfs_protos::proto_gen::node_service::Error {
|
||||
code: u32::MAX,
|
||||
error_info: "unknown peer error".into(),
|
||||
}),
|
||||
},
|
||||
] {
|
||||
assert!(matches!(format_control_write_response(response), Err(DiskError::FileCorrupt)));
|
||||
}
|
||||
assert!(matches!(
|
||||
format_control_write_response(WriteAllResponse {
|
||||
success: false,
|
||||
error: Some(DiskError::FileNotFound.into())
|
||||
}),
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
use super::*;
|
||||
use crate::cluster::rpc::internode_data_transport::{InternodeDataTransportCapabilities, TcpHttpInternodeDataTransport};
|
||||
use crate::erasure::coding::{BitrotReader, Erasure, decode::ParallelReader};
|
||||
|
||||
@@ -6400,7 +6400,7 @@ pub(crate) struct PoolMetaWriteState {
|
||||
pool_meta_absent: bool,
|
||||
bootstrap_authority: PoolMetaBootstrapAuthority,
|
||||
/// First-hand bootstrap authority this process holds for each pool
|
||||
/// (index = pool index): `Fresh` only for pools it formatted itself,
|
||||
/// (index = pool index): `Fresh` for pools it formatted or resumed from a complete durable plan,
|
||||
/// `LegacyAdoption` only for pools whose migration it verified. Empty when
|
||||
/// the caller tracks deployment-wide authority only.
|
||||
pool_bootstrap_authorities: Vec<PoolMetaBootstrapAuthority>,
|
||||
@@ -6478,8 +6478,7 @@ impl PoolMetaWriteState {
|
||||
}
|
||||
|
||||
/// Startup state for a process that loaded every pool format itself and
|
||||
/// remembers, per pool, whether it created (or adopted) that pool
|
||||
/// first-hand. Deployment-wide authority is the conjunction across pools:
|
||||
/// remembers, per pool, whether it created, durably resumed, or adopted that pool. Deployment-wide authority is the conjunction across pools:
|
||||
/// any pool this process merely read yields `None`, exactly as before.
|
||||
pub(crate) fn for_startup_with_pool_bootstrap_authorities(
|
||||
cluster_id: uuid::Uuid,
|
||||
@@ -6510,7 +6509,7 @@ impl PoolMetaWriteState {
|
||||
.is_some_and(|authority| authority.is_proven())
|
||||
}
|
||||
|
||||
/// Pools this process formatted or adopted first-hand during this startup.
|
||||
/// Pools this process formatted, durably resumed, or adopted during startup.
|
||||
pub(crate) fn attested_pool_indices(&self) -> Vec<usize> {
|
||||
self.pool_bootstrap_authorities
|
||||
.iter()
|
||||
@@ -6527,7 +6526,7 @@ impl PoolMetaWriteState {
|
||||
/// [`PoolMetaIdentityWriteScope::Pools`]), so a complete, agreeing pending
|
||||
/// set proves that every pool joined this bootstrap fresh. A missing,
|
||||
/// corrupt, or disagreeing replica keeps the writer fail-closed, and a
|
||||
/// restart without first-hand proof never reopens bootstrap on its own.
|
||||
/// restart requires a complete durable format plan to recover creator proof.
|
||||
fn pending_identity_attested_by_every_pool(&self) -> bool {
|
||||
self.elected_bootstrap_writer == Some(true)
|
||||
&& self.pool_bootstrap_authority_proven(0)
|
||||
@@ -6759,6 +6758,27 @@ impl PoolMetaWriteState {
|
||||
self.expected_cluster_id.is_some() && (self.identity_needs_repair || self.identity_initialized != Some(true))
|
||||
}
|
||||
|
||||
/// A committed predecessor authorizes repairing an established identity
|
||||
/// before the next metadata write. A pending fresh identity must remain
|
||||
/// pending until its initial metadata commit is recoverable.
|
||||
pub(crate) fn startup_identity_precedes_metadata_write(&self) -> bool {
|
||||
!self.pool_meta_absent && !self.identity_is_pending()
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_startup_identity_initialized(&self) -> Result<()> {
|
||||
if self.expected_cluster_id.is_none()
|
||||
|| (self.identity_initialized == Some(true)
|
||||
&& !self.identity_needs_repair
|
||||
&& !self.identity_replicas_missing
|
||||
&& !self.identity_replicas_invalid)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
Err(Error::other(
|
||||
"pool metadata bootstrap pending: waiting for the elected writer to initialize the identity on every pool",
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_write_safe(&self, operation: &str) -> Result<()> {
|
||||
let (Some(mut context), _) = self.write_block() else {
|
||||
return Ok(());
|
||||
@@ -10146,6 +10166,8 @@ impl PoolMeta {
|
||||
self.save_for_startup_observing(pools, &mut write_state).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Commits a complete startup snapshot while leaving identity finalization
|
||||
/// to the caller, after durable format-bootstrap authority is retired.
|
||||
pub(crate) async fn save_for_startup_observing<S>(
|
||||
&self,
|
||||
pools: Vec<Arc<S>>,
|
||||
@@ -10160,6 +10182,8 @@ impl PoolMeta {
|
||||
Ok(outcome.into_committed())
|
||||
}
|
||||
|
||||
/// Commits a complete startup snapshot under the writer fence. The caller
|
||||
/// retires format-bootstrap authority before finalizing initialized identity.
|
||||
pub(crate) async fn save_for_startup_guarded<S>(
|
||||
&self,
|
||||
pools: Vec<Arc<S>>,
|
||||
@@ -10362,7 +10386,10 @@ impl PoolMeta {
|
||||
selection.revision.epoch, identity.epoch
|
||||
)));
|
||||
}
|
||||
if !selection.absent && write_state.identity_requires_repair() {
|
||||
// Full-snapshot saves belong to startup. Their caller retires
|
||||
// durable format authority before publishing initialized identity.
|
||||
// Runtime scoped saves retain their existing identity repair.
|
||||
if indices.is_some() && !selection.absent && write_state.identity_requires_repair() {
|
||||
let initialized = write_state.identity_initialized != Some(false) || selection.revision.is_generation_protocol();
|
||||
persist_pool_meta_identity(
|
||||
pools.clone(),
|
||||
@@ -10640,7 +10667,10 @@ impl PoolMeta {
|
||||
confirmed
|
||||
};
|
||||
if confirmed.revision == revision && confirmed.canonical.as_ref() == Some(&durable) {
|
||||
persist_pool_meta_identity(pools, write_state, true, PoolMetaIdentityWriteScope::All, fence, transaction_arm).await?;
|
||||
if indices.is_some() {
|
||||
persist_pool_meta_identity(pools, write_state, true, PoolMetaIdentityWriteScope::All, fence, transaction_arm)
|
||||
.await?;
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
startup_cas_test_observe(serde_json::json!({
|
||||
"kind": "confirmed", "object": POOL_META_NAME,
|
||||
|
||||
@@ -345,6 +345,7 @@ const DISK_OPERATION_NAMES: &[&str] = &[
|
||||
"read_parts",
|
||||
"read_multiple",
|
||||
"write_all",
|
||||
"write_format_control",
|
||||
"compare_and_update_file",
|
||||
"read_all",
|
||||
];
|
||||
@@ -3662,6 +3663,16 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_format_control(&self, path: &str, data: Bytes) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
"write_format_control",
|
||||
DiskMetricMutation::Write,
|
||||
|| async { self.disk.write_format_control(path, data).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn compare_and_update_file(
|
||||
&self,
|
||||
volume: &str,
|
||||
|
||||
@@ -3535,6 +3535,86 @@ pub struct FormatInfo {
|
||||
pub last_check: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
fn read_format_control_record(
|
||||
parent: &Path,
|
||||
name: &str,
|
||||
base: &Path,
|
||||
root: &os::PublicationRoot,
|
||||
pinned: &os::ExistingBaseDirectoryGuard,
|
||||
) -> std::io::Result<Option<FormatV3>> {
|
||||
#[cfg(unix)]
|
||||
let bytes = {
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
|
||||
let _ = (parent, base, root);
|
||||
let directory = pinned.last().ok_or(DiskError::InvalidPath)?;
|
||||
let file = match openat(
|
||||
directory,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
) {
|
||||
Ok(file) => std::fs::File::from(file),
|
||||
Err(rustix::io::Errno::NOENT) => return Ok(None),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
const MAX_BYTES: u64 = 16 * 1024 * 1024;
|
||||
let metadata = file.metadata()?;
|
||||
if !metadata.is_file() || metadata.len() > MAX_BYTES {
|
||||
return Err(DiskError::CorruptedFormat.into());
|
||||
}
|
||||
let mut bytes = Vec::with_capacity(usize::try_from(metadata.len()).map_err(|_| DiskError::CorruptedFormat)?);
|
||||
std::io::Read::read_to_end(&mut std::io::Read::take(file, MAX_BYTES + 1), &mut bytes)?;
|
||||
if u64::try_from(bytes.len()).map_err(|_| DiskError::CorruptedFormat)? > MAX_BYTES {
|
||||
return Err(DiskError::CorruptedFormat.into());
|
||||
}
|
||||
bytes
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let bytes = {
|
||||
let _ = pinned;
|
||||
match os::read_multipart_source_metadata(&parent.join(name), base, root) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None),
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
};
|
||||
FormatV3::try_from(bytes.as_slice())
|
||||
.map(Some)
|
||||
.map_err(|_| DiskError::CorruptedFormat.into())
|
||||
}
|
||||
|
||||
fn validate_format_control_publication(
|
||||
path: &Path,
|
||||
bytes: &[u8],
|
||||
base: &Path,
|
||||
root: &os::PublicationRoot,
|
||||
) -> std::io::Result<os::ExistingBaseDirectoryGuard> {
|
||||
let incoming = FormatV3::try_from(bytes).map_err(|_| DiskError::CorruptedFormat)?;
|
||||
incoming
|
||||
.find_disk_index_by_disk_id(incoming.erasure.this)
|
||||
.map_err(|_| DiskError::CorruptedFormat)?;
|
||||
let parent = path.parent().ok_or(DiskError::InvalidPath)?;
|
||||
let pinned = os::pin_multipart_source_directory(parent, base, root)?;
|
||||
let done = read_format_control_record(parent, super::FORMAT_BOOTSTRAP_DONE_FILE, base, root, &pinned)?;
|
||||
if path.file_name().is_some_and(|name| name == super::FORMAT_BOOTSTRAP_FILE) && done.is_some() {
|
||||
return Err(DiskError::MethodNotAllowed.into());
|
||||
}
|
||||
for existing in [
|
||||
read_format_control_record(parent, super::FORMAT_CONFIG_FILE, base, root, &pinned)?,
|
||||
read_format_control_record(parent, super::FORMAT_BOOTSTRAP_FILE, base, root, &pinned)?,
|
||||
done,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
if existing.shared_identity() != incoming.shared_identity() || existing.erasure.this != incoming.erasure.this {
|
||||
return Err(DiskError::CorruptedFormat.into());
|
||||
}
|
||||
}
|
||||
Ok(pinned)
|
||||
}
|
||||
|
||||
/// Durability mode for `write_all_internal`.
|
||||
///
|
||||
/// `FileOnly` is reserved for tmp files the caller immediately renames away.
|
||||
@@ -7716,7 +7796,10 @@ pub struct LocalDisk {
|
||||
/// disk I/O uses `io_format_path`, which is rooted at `mount_lease`.
|
||||
pub format_path: PathBuf,
|
||||
io_format_path: PathBuf,
|
||||
pub format_info: RwLock<FormatInfo>,
|
||||
pub format_info: Arc<RwLock<FormatInfo>>,
|
||||
// Registry-only key. Physical identity shares serialization across reopened
|
||||
// handles while separating replacement roots at the same configured path.
|
||||
format_control_key: PathBuf,
|
||||
pub endpoint: Endpoint,
|
||||
pub disk_info_cache: Arc<Cache<DiskInfo>>,
|
||||
pub scanning: Arc<AtomicU32>,
|
||||
@@ -8096,6 +8179,9 @@ impl LocalDisk {
|
||||
// subsequent path-based operation so retargeting the configured alias
|
||||
// cannot split ordinary IO from handle-relative publication.
|
||||
let root = publication_root.path().to_path_buf();
|
||||
let (format_device, format_file_id) = publication_root.physical_identity().map_err(DiskError::from)?;
|
||||
let format_control_key =
|
||||
PathBuf::from(format!(".rustfs-format-control/{format_device:x}/{}", Uuid::from_bytes(format_file_id)));
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let (mount_lease, io_root, mount_lease_mount_id) = (
|
||||
@@ -8266,7 +8352,8 @@ impl LocalDisk {
|
||||
endpoint: ep.clone(),
|
||||
format_path,
|
||||
io_format_path,
|
||||
format_info: RwLock::new(format_info),
|
||||
format_info: Arc::new(RwLock::new(format_info)),
|
||||
format_control_key,
|
||||
disk_info_cache: Arc::new(cache),
|
||||
scanning: Arc::new(AtomicU32::new(0)),
|
||||
rotational: Default::default(),
|
||||
@@ -11847,6 +11934,149 @@ impl LocalDisk {
|
||||
Ok(f)
|
||||
}
|
||||
|
||||
async fn retire_format_bootstrap(&self) -> Result<()> {
|
||||
let base = self.io_root().to_path_buf();
|
||||
let parent = base.join(RUSTFS_META_BUCKET);
|
||||
let root = self.publication_root.clone();
|
||||
let backend = Arc::clone(&self.io_backend);
|
||||
let owner = os::acquire_namespace_mutation_lease_with_owner(&self.format_control_key, None).await;
|
||||
self.blocking_queue
|
||||
.run_fsync_cancellable(move |is_cancelled| {
|
||||
let _owner = owner;
|
||||
if is_cancelled() {
|
||||
return Err(IoError::new(ErrorKind::Interrupted, "format bootstrap retirement cancelled"));
|
||||
}
|
||||
let invalidate = || {
|
||||
backend.invalidate_cached_fds_under(RUSTFS_META_BUCKET, super::FORMAT_BOOTSTRAP_FILE);
|
||||
backend.invalidate_cached_fds_under(RUSTFS_META_BUCKET, super::FORMAT_BOOTSTRAP_DONE_FILE);
|
||||
};
|
||||
invalidate();
|
||||
let retire = || -> std::io::Result<()> {
|
||||
os::record_native_publication(&parent.join(super::FORMAT_BOOTSTRAP_DONE_FILE));
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags, openat, renameat};
|
||||
|
||||
let mut parents = os::pin_multipart_source_directory(&parent, &base, &root)?;
|
||||
let directory = std::fs::File::from(parents.pop().ok_or(DiskError::InvalidPath)?);
|
||||
let source = openat(
|
||||
&directory,
|
||||
super::FORMAT_BOOTSTRAP_FILE,
|
||||
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
);
|
||||
match source {
|
||||
Ok(source) => {
|
||||
let source = std::fs::File::from(source);
|
||||
if !source.metadata()?.is_file() {
|
||||
return Err(DiskError::FileCorrupt.into());
|
||||
}
|
||||
renameat(
|
||||
&directory,
|
||||
super::FORMAT_BOOTSTRAP_FILE,
|
||||
&directory,
|
||||
super::FORMAT_BOOTSTRAP_DONE_FILE,
|
||||
)?;
|
||||
}
|
||||
Err(rustix::io::Errno::NOENT) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
// Retry also synchronizes an already absent source. A visible
|
||||
// rename whose prior fsync failed grants no durable proof.
|
||||
os::fsync_directory_handle_std(&parent, Some(&directory))
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let source = parent.join(super::FORMAT_BOOTSTRAP_FILE);
|
||||
let destination = parent.join(super::FORMAT_BOOTSTRAP_DONE_FILE);
|
||||
let _parents = os::pin_multipart_source_directory(&parent, &base, &root)?;
|
||||
match std::fs::symlink_metadata(&source) {
|
||||
Ok(metadata) if !metadata.is_file() => return Err(DiskError::FileCorrupt.into()),
|
||||
Ok(_) => {}
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => return os::fsync_dir_std(&parent),
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
match os::rename_all_with_precondition_and_parent_sync_std(
|
||||
&source,
|
||||
&destination,
|
||||
&base,
|
||||
&root,
|
||||
None,
|
||||
true,
|
||||
) {
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => os::fsync_dir_std(&parent),
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
};
|
||||
let result = retire();
|
||||
invalidate();
|
||||
result
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
|
||||
async fn open_format_bootstrap_reader(&self, offset: usize, length: usize) -> Result<FileReader> {
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let file_path = self.io_get_object_open_path(RUSTFS_META_BUCKET, super::FORMAT_BOOTSTRAP_FILE)?;
|
||||
let base = self.io_root().to_path_buf();
|
||||
let root = self.publication_root.clone();
|
||||
let offset = u64::try_from(offset).map_err(|_| DiskError::FileCorrupt)?;
|
||||
let length = u64::try_from(length).map_err(|_| DiskError::FileCorrupt)?;
|
||||
let end_offset = offset.checked_add(length).ok_or(DiskError::FileCorrupt)?;
|
||||
self.blocking_queue
|
||||
.run_cancellable(move |is_cancelled| {
|
||||
if is_cancelled() {
|
||||
return Err(IoError::new(ErrorKind::Interrupted, "format bootstrap read cancelled"));
|
||||
}
|
||||
#[cfg(unix)]
|
||||
let (reader, size) = {
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
|
||||
let parent = file_path.parent().ok_or(DiskError::InvalidPath)?;
|
||||
let name = file_path.file_name().ok_or(DiskError::InvalidPath)?;
|
||||
let parents = os::pin_multipart_source_directory(parent, &base, &root)?;
|
||||
let directory = parents.last().ok_or(DiskError::InvalidPath)?;
|
||||
// Validate the opened inode before returning a reader. NONBLOCK
|
||||
// keeps a corrupt FIFO from occupying a native worker forever.
|
||||
let mut file = std::fs::File::from(openat(
|
||||
directory,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
let metadata = file.metadata()?;
|
||||
if !metadata.is_file() {
|
||||
return Err(DiskError::FileCorrupt.into());
|
||||
}
|
||||
std::io::Seek::seek(&mut file, SeekFrom::Start(offset))?;
|
||||
(Box::new(File::from_std(file)) as FileReader, metadata.len())
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let (reader, size) = {
|
||||
let bytes = os::read_multipart_source_metadata(&file_path, &base, &root)?;
|
||||
let size = u64::try_from(bytes.len()).map_err(|_| DiskError::FileCorrupt)?;
|
||||
let mut cursor = std::io::Cursor::new(bytes);
|
||||
cursor.set_position(offset);
|
||||
(Box::new(cursor) as FileReader, size)
|
||||
};
|
||||
if size < end_offset {
|
||||
return Err(DiskError::FileCorrupt.into());
|
||||
}
|
||||
if length == 0 {
|
||||
Ok(reader)
|
||||
} else {
|
||||
Ok(Box::new(reader.take(length)) as FileReader)
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "MinIO-parity surface with no caller in this port (backlog#1823)")]
|
||||
fn get_metrics(&self) -> DiskMetrics {
|
||||
DiskMetrics::default()
|
||||
@@ -14289,6 +14519,61 @@ impl DiskAPI for LocalDisk {
|
||||
self.write_all_public(volume, path, data).await
|
||||
}
|
||||
|
||||
async fn write_format_control(&self, path: &str, data: Bytes) -> Result<()> {
|
||||
super::validate_format_control_path(path)?;
|
||||
let file_path = self.io_get_object_open_path(RUSTFS_META_BUCKET, path)?;
|
||||
let base = self.io_root().to_path_buf();
|
||||
let root = self.publication_root.clone();
|
||||
let backend = Arc::clone(&self.io_backend);
|
||||
let format_info = (path == super::FORMAT_CONFIG_FILE).then(|| Arc::clone(&self.format_info));
|
||||
let path = path.to_owned();
|
||||
let owner = os::acquire_namespace_mutation_lease_with_owner(&self.format_control_key, None).await;
|
||||
self.blocking_queue
|
||||
.run_fsync_cancellable(move |is_cancelled| {
|
||||
let _owner = owner;
|
||||
if is_cancelled() {
|
||||
return Err(IoError::new(ErrorKind::Interrupted, "format control publication cancelled"));
|
||||
}
|
||||
let invalidate = || {
|
||||
backend.invalidate_cached_fds_under(RUSTFS_META_BUCKET, &path);
|
||||
if let Some(format_info) = &format_info {
|
||||
let mut format_info = format_info.blocking_write();
|
||||
format_info.data = Bytes::new();
|
||||
format_info.id = None;
|
||||
format_info.file_info = None;
|
||||
format_info.last_check = None;
|
||||
}
|
||||
};
|
||||
// The native owner outlives a cancelled RPC. Both invalidations
|
||||
// run there, with cache guards released throughout filesystem I/O.
|
||||
invalidate();
|
||||
let publish = || {
|
||||
let pinned = validate_format_control_publication(&file_path, &data, &base, &root)?;
|
||||
#[cfg(unix)]
|
||||
let mut pinned = pinned;
|
||||
#[cfg(unix)]
|
||||
let leaf = pinned.pop().ok_or(DiskError::InvalidPath)?;
|
||||
#[cfg(not(unix))]
|
||||
let _pinned = pinned;
|
||||
os::replace_multipart_source_metadata_with_leaf_fd(
|
||||
&file_path,
|
||||
&data,
|
||||
&base,
|
||||
&root,
|
||||
#[cfg(unix)]
|
||||
Some(leaf),
|
||||
)
|
||||
};
|
||||
let result = publish();
|
||||
invalidate();
|
||||
crate::bucket::metadata_sys::bump_local_meta_mutation_gen();
|
||||
result
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
|
||||
async fn compare_and_update_file(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -15568,6 +15853,13 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
|
||||
if src_volume == RUSTFS_META_BUCKET
|
||||
&& dst_volume == RUSTFS_META_BUCKET
|
||||
&& src_path == super::FORMAT_BOOTSTRAP_FILE
|
||||
&& dst_path == super::FORMAT_BOOTSTRAP_DONE_FILE
|
||||
{
|
||||
return self.retire_format_bootstrap().await;
|
||||
}
|
||||
let source = self.io_get_object_path(src_volume, src_path)?;
|
||||
let destination = self.io_get_object_path(dst_volume, dst_path)?;
|
||||
check_path_length(source.to_string_lossy().as_ref())?;
|
||||
@@ -15614,12 +15906,18 @@ impl DiskAPI for LocalDisk {
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader> {
|
||||
crate::hp_guard!("LocalDisk::read_file");
|
||||
if volume == RUSTFS_META_BUCKET && path == super::FORMAT_BOOTSTRAP_FILE {
|
||||
return self.open_format_bootstrap_reader(0, 0).await;
|
||||
}
|
||||
self.io_backend.open_full_read(volume, path).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<FileReader> {
|
||||
crate::hp_guard!("LocalDisk::read_file_stream");
|
||||
if volume == RUSTFS_META_BUCKET && path == super::FORMAT_BOOTSTRAP_FILE {
|
||||
return self.open_format_bootstrap_reader(offset, length).await;
|
||||
}
|
||||
self.io_backend.open_read_stream(volume, path, offset, length).await
|
||||
}
|
||||
|
||||
@@ -19155,6 +19453,572 @@ mod test {
|
||||
assert_eq!(payload, b"read-only-payload");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_control_read_preserves_regular_payload_and_missing_error() {
|
||||
let directory = tempfile::tempdir().expect("bootstrap control fixture");
|
||||
let endpoint =
|
||||
Endpoint::try_from(directory.path().to_str().expect("fixture path is UTF-8")).expect("bootstrap control endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("bootstrap control disk");
|
||||
let path = "format.json.bootstrap";
|
||||
assert!(matches!(disk.read_file(RUSTFS_META_BUCKET, path).await, Err(DiskError::FileNotFound)));
|
||||
|
||||
disk.write_all(RUSTFS_META_BUCKET, path, Bytes::from_static(b"bootstrap-plan"))
|
||||
.await
|
||||
.expect("write regular bootstrap control");
|
||||
let mut reader = disk.read_file(RUSTFS_META_BUCKET, path).await.expect("open regular control");
|
||||
let mut payload = Vec::new();
|
||||
reader.read_to_end(&mut payload).await.expect("read regular control");
|
||||
assert_eq!(payload, b"bootstrap-plan");
|
||||
for (offset, length, expected) in [
|
||||
(0, 0, b"bootstrap-plan".as_slice()),
|
||||
(1, 2, b"oo".as_slice()),
|
||||
(1, 0, b"ootstrap-plan".as_slice()),
|
||||
] {
|
||||
let mut stream = disk
|
||||
.read_file_stream(RUSTFS_META_BUCKET, path, offset, length)
|
||||
.await
|
||||
.expect("open regular control stream");
|
||||
let mut payload = Vec::new();
|
||||
stream.read_to_end(&mut payload).await.expect("read control range");
|
||||
assert_eq!(payload, expected);
|
||||
}
|
||||
assert!(matches!(
|
||||
disk.read_file_stream(RUSTFS_META_BUCKET, path, 0, 100).await,
|
||||
Err(DiskError::FileCorrupt)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn bootstrap_control_read_rejects_nonregular_entries_without_blocking() {
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
|
||||
let directory = tempfile::tempdir().expect("nonregular bootstrap control fixture");
|
||||
let endpoint =
|
||||
Endpoint::try_from(directory.path().to_str().expect("fixture path is UTF-8")).expect("bootstrap control endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("bootstrap control disk");
|
||||
let meta = directory.path().join(RUSTFS_META_BUCKET);
|
||||
fs::create_dir_all(&meta).await.expect("bootstrap metadata directory");
|
||||
let path = meta.join("format.json.bootstrap");
|
||||
let target = directory.path().join("outside-control");
|
||||
fs::write(&target, b"foreign-bootstrap-plan").await.expect("symlink target");
|
||||
|
||||
for entry in ["directory", "symlink", "fifo"] {
|
||||
match entry {
|
||||
"directory" => std::fs::create_dir(&path).expect("directory control entry"),
|
||||
"symlink" => std::os::unix::fs::symlink(&target, &path).expect("symlink control entry"),
|
||||
"fifo" => assert!(
|
||||
std::process::Command::new("mkfifo")
|
||||
.arg(&path)
|
||||
.status()
|
||||
.expect("create POSIX FIFO control entry")
|
||||
.success()
|
||||
),
|
||||
_ => unreachable!("fixture entries are enumerated"),
|
||||
}
|
||||
for stream in [false, true] {
|
||||
let mut pending = Box::pin(async {
|
||||
if stream {
|
||||
disk.read_file_stream(RUSTFS_META_BUCKET, "format.json.bootstrap", 0, 0).await
|
||||
} else {
|
||||
disk.read_file(RUSTFS_META_BUCKET, "format.json.bootstrap").await
|
||||
}
|
||||
});
|
||||
let result = match timeout(Duration::from_secs(2), &mut pending).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => {
|
||||
// Release a regressed blocking FIFO open before failing.
|
||||
let _release = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.custom_flags(libc::O_NONBLOCK)
|
||||
.open(&path)
|
||||
.expect("release blocked control reader");
|
||||
let _ = pending.await;
|
||||
panic!("{entry} bootstrap control must fail without blocking");
|
||||
}
|
||||
};
|
||||
assert!(result.is_err(), "{entry} must never supply bootstrap authority");
|
||||
}
|
||||
if entry == "directory" {
|
||||
std::fs::remove_dir(&path).expect("remove directory entry");
|
||||
} else {
|
||||
std::fs::remove_file(&path).expect("remove nonregular entry");
|
||||
}
|
||||
}
|
||||
assert_eq!(fs::read(&target).await.expect("unchanged symlink target"), b"foreign-bootstrap-plan");
|
||||
}
|
||||
|
||||
fn format_control_test_plan() -> (FormatV3, Bytes) {
|
||||
let mut plan = FormatV3::new(1, 1);
|
||||
plan.erasure.this = plan.erasure.sets[0][0];
|
||||
let bytes = Bytes::from(plan.to_json().expect("valid control plan JSON"));
|
||||
(plan, bytes)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_control_publication_rejects_foreign_and_retired_plans() {
|
||||
let directory = tempfile::tempdir().expect("monotonic control fixture");
|
||||
let endpoint =
|
||||
Endpoint::try_from(directory.path().to_str().expect("fixture path is UTF-8")).expect("monotonic control endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("monotonic control disk");
|
||||
let (_, accepted) = format_control_test_plan();
|
||||
let (_, foreign) = format_control_test_plan();
|
||||
disk.write_format_control(super::super::FORMAT_BOOTSTRAP_FILE, accepted.clone())
|
||||
.await
|
||||
.expect("accept the first durable plan");
|
||||
disk.write_format_control(super::super::FORMAT_BOOTSTRAP_FILE, foreign.clone())
|
||||
.await
|
||||
.expect_err("a pending plan is immutable");
|
||||
disk.write_format_control(super::super::FORMAT_CONFIG_FILE, foreign.clone())
|
||||
.await
|
||||
.expect_err("format publication must agree with the accepted pending plan");
|
||||
disk.write_format_control(super::super::FORMAT_BOOTSTRAP_FILE, accepted.clone())
|
||||
.await
|
||||
.expect("an equivalent pending retry is idempotent");
|
||||
disk.write_format_control(super::super::FORMAT_CONFIG_FILE, accepted.clone())
|
||||
.await
|
||||
.expect("publish only the accepted format");
|
||||
disk.rename_file_durable(
|
||||
RUSTFS_META_BUCKET,
|
||||
super::super::FORMAT_BOOTSTRAP_FILE,
|
||||
RUSTFS_META_BUCKET,
|
||||
super::super::FORMAT_BOOTSTRAP_DONE_FILE,
|
||||
)
|
||||
.await
|
||||
.expect("retire accepted authority");
|
||||
for late in [accepted.clone(), foreign.clone()] {
|
||||
disk.write_format_control(super::super::FORMAT_BOOTSTRAP_FILE, late)
|
||||
.await
|
||||
.expect_err("retirement rejects even an equivalent delayed bootstrap write");
|
||||
}
|
||||
disk.write_format_control(super::super::FORMAT_CONFIG_FILE, foreign)
|
||||
.await
|
||||
.expect_err("a delayed foreign format cannot replace the established identity");
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, super::super::FORMAT_CONFIG_FILE)
|
||||
.await
|
||||
.expect("accepted format survives late publications"),
|
||||
accepted
|
||||
);
|
||||
assert!(matches!(
|
||||
disk.read_file(RUSTFS_META_BUCKET, super::super::FORMAT_BOOTSTRAP_FILE).await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_control_publication_rejects_corrupt_record_evidence() {
|
||||
for record in [
|
||||
super::super::FORMAT_CONFIG_FILE,
|
||||
super::super::FORMAT_BOOTSTRAP_FILE,
|
||||
super::super::FORMAT_BOOTSTRAP_DONE_FILE,
|
||||
] {
|
||||
let directory = tempfile::tempdir().expect("corrupt control fixture");
|
||||
let endpoint =
|
||||
Endpoint::try_from(directory.path().to_str().expect("fixture path is UTF-8")).expect("corrupt control endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("corrupt control disk");
|
||||
let evidence = directory.path().join(RUSTFS_META_BUCKET).join(record);
|
||||
fs::write(&evidence, b"corrupt-control")
|
||||
.await
|
||||
.expect("corrupt persisted evidence");
|
||||
let (_, incoming) = format_control_test_plan();
|
||||
disk.write_format_control(super::super::FORMAT_BOOTSTRAP_FILE, incoming)
|
||||
.await
|
||||
.expect_err("corrupt evidence cannot be interpreted as absence");
|
||||
assert_eq!(fs::read(&evidence).await.expect("retained corrupt evidence"), b"corrupt-control");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_control_publication_preserves_previous_bytes_before_commit() {
|
||||
use os::multipart_source_syscall_failures::{Stage, fail_once};
|
||||
|
||||
let directory = tempfile::tempdir().expect("atomic format fixture");
|
||||
let endpoint =
|
||||
Endpoint::try_from(directory.path().to_str().expect("fixture path is UTF-8")).expect("atomic format endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("atomic format disk");
|
||||
let (_, previous) = format_control_test_plan();
|
||||
let successor = Bytes::from(format!("{}\n", std::str::from_utf8(&previous).expect("JSON is UTF-8")));
|
||||
for path in [super::super::FORMAT_CONFIG_FILE, super::super::FORMAT_BOOTSTRAP_FILE] {
|
||||
disk.write_format_control(path, previous.clone())
|
||||
.await
|
||||
.expect("initial atomic control");
|
||||
let native = disk
|
||||
.io_get_object_open_path(RUSTFS_META_BUCKET, path)
|
||||
.expect("control I/O path");
|
||||
for stage in [Stage::Create, Stage::Write, Stage::FileSync, Stage::Rename] {
|
||||
let fault = fail_once(stage, &native, ErrorKind::Other);
|
||||
disk.write_format_control(path, successor.clone())
|
||||
.await
|
||||
.expect_err("failed preparation preserves existing control");
|
||||
assert!(fault.was_triggered(), "{stage:?} must exercise the real native publication");
|
||||
assert_eq!(
|
||||
fs::read(directory.path().join(RUSTFS_META_BUCKET).join(path))
|
||||
.await
|
||||
.expect("physical control"),
|
||||
previous
|
||||
);
|
||||
assert_eq!(disk.read_all(RUSTFS_META_BUCKET, path).await.expect("current control cache"), previous);
|
||||
}
|
||||
}
|
||||
assert!(matches!(
|
||||
disk.write_format_control("pool.bin", Bytes::new()).await,
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn format_control_publication_failure_after_rename_invalidates_format_cache() {
|
||||
let directory = tempfile::tempdir().expect("atomic format cache fixture");
|
||||
let endpoint =
|
||||
Endpoint::try_from(directory.path().to_str().expect("fixture path is UTF-8")).expect("atomic format endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("atomic format disk");
|
||||
let (_, previous) = format_control_test_plan();
|
||||
let successor = Bytes::from(format!("{}\n", std::str::from_utf8(&previous).expect("JSON is UTF-8")));
|
||||
disk.write_all(RUSTFS_META_BUCKET, super::super::FORMAT_CONFIG_FILE, previous)
|
||||
.await
|
||||
.expect("seed previous cached format");
|
||||
os::fsync_dir_recorder::set_failure(&directory.path().join(RUSTFS_META_BUCKET), ErrorKind::Other);
|
||||
disk.write_format_control(super::super::FORMAT_CONFIG_FILE, successor.clone())
|
||||
.await
|
||||
.expect_err("a failed directory barrier cannot acknowledge publication");
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, super::super::FORMAT_CONFIG_FILE)
|
||||
.await
|
||||
.expect("reload visible successor after ambiguous publication"),
|
||||
successor
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn format_control_cancelled_caller_cannot_retain_old_format_caches() {
|
||||
use os::prepared_publication_test_hooks::{Stage, install_at};
|
||||
|
||||
let directory = tempfile::tempdir().expect("cancelled format publication fixture");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(directory.path().to_str().expect("fixture path is UTF-8")).expect("format cancellation endpoint");
|
||||
endpoint.set_idx = 0;
|
||||
endpoint.disk_idx = 0;
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("format cancellation disk"));
|
||||
let mut previous = FormatV3::new(1, 1);
|
||||
previous.erasure.this = previous.erasure.sets[0][0];
|
||||
let previous_bytes = Bytes::from(previous.to_json().expect("previous format JSON"));
|
||||
disk.write_format_control(super::super::FORMAT_CONFIG_FILE, previous_bytes.clone())
|
||||
.await
|
||||
.expect("publish previous format");
|
||||
assert_eq!(disk.get_disk_id().await.expect("cache previous disk id"), Some(previous.erasure.this));
|
||||
|
||||
let successor = previous.clone();
|
||||
let successor_bytes = Bytes::from(format!("{}\n", previous.to_json().expect("successor format JSON")));
|
||||
let native = disk
|
||||
.io_get_object_open_path(RUSTFS_META_BUCKET, super::super::FORMAT_CONFIG_FILE)
|
||||
.expect("native format path");
|
||||
let parent = native.parent().expect("native format parent");
|
||||
let (before_rename_tx, before_rename_rx) = tokio::sync::oneshot::channel();
|
||||
let (rename_release_tx, rename_release_rx) = std::sync::mpsc::channel();
|
||||
let _rename = install_at(Stage::PreparedRename, &native, move || {
|
||||
let _ = before_rename_tx.send(());
|
||||
let _ = rename_release_rx.recv();
|
||||
});
|
||||
let (before_sync_tx, before_sync_rx) = tokio::sync::oneshot::channel();
|
||||
let (sync_release_tx, sync_release_rx) = std::sync::mpsc::channel();
|
||||
let _sync = install_at(Stage::DirFsync, parent, move || {
|
||||
let _ = before_sync_tx.send(());
|
||||
let _ = sync_release_rx.recv();
|
||||
});
|
||||
let caller = tokio::spawn({
|
||||
let disk = Arc::clone(&disk);
|
||||
let bytes = successor_bytes.clone();
|
||||
async move { disk.write_format_control(super::super::FORMAT_CONFIG_FILE, bytes).await }
|
||||
});
|
||||
timeout(Duration::from_secs(5), before_rename_rx)
|
||||
.await
|
||||
.expect("native publication reaches rename")
|
||||
.expect("rename barrier reports arrival");
|
||||
// A real reader can repopulate old caches while publication is staged.
|
||||
assert_eq!(
|
||||
disk.get_disk_id().await.expect("read previous disk while staging"),
|
||||
Some(previous.erasure.this)
|
||||
);
|
||||
assert_eq!(
|
||||
disk.read_file_mmap_copy(RUSTFS_META_BUCKET, super::super::FORMAT_CONFIG_FILE, 0, previous_bytes.len())
|
||||
.await
|
||||
.expect("cache previous file descriptor"),
|
||||
previous_bytes
|
||||
);
|
||||
rename_release_tx.send(()).expect("release physical rename");
|
||||
timeout(Duration::from_secs(5), before_sync_rx)
|
||||
.await
|
||||
.expect("native publication reaches post-rename sync")
|
||||
.expect("sync barrier reports arrival");
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("caller is cancelled at the barrier").is_cancelled());
|
||||
sync_release_tx
|
||||
.send(())
|
||||
.expect("allow detached native publication to complete");
|
||||
timeout(Duration::from_secs(5), async {
|
||||
while !disk.format_info.read().await.data.is_empty() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("native owner invalidates caches after cancelled caller");
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, super::super::FORMAT_CONFIG_FILE)
|
||||
.await
|
||||
.expect("read current published format"),
|
||||
successor_bytes
|
||||
);
|
||||
assert_eq!(disk.get_disk_id().await.expect("reload committed disk id"), Some(successor.erasure.this));
|
||||
assert_eq!(
|
||||
disk.read_file_mmap_copy(RUSTFS_META_BUCKET, super::super::FORMAT_CONFIG_FILE, 0, successor_bytes.len())
|
||||
.await
|
||||
.expect("read successor through descriptor path"),
|
||||
successor_bytes
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn format_control_cancelled_publication_serializes_reopened_writer_and_retirement() {
|
||||
use os::prepared_publication_test_hooks::{Stage, install_at};
|
||||
|
||||
let directory = tempfile::tempdir().expect("serialized format fixture");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(directory.path().to_str().expect("fixture path is UTF-8")).expect("serialized format endpoint");
|
||||
endpoint.pool_idx = 0;
|
||||
endpoint.set_idx = 0;
|
||||
endpoint.disk_idx = 0;
|
||||
let first = Arc::new(LocalDisk::new(&endpoint, false).await.expect("first format handle"));
|
||||
let mut old_plan = FormatV3::new(1, 1);
|
||||
old_plan.erasure.this = old_plan.erasure.sets[0][0];
|
||||
let intent = first
|
||||
.io_get_object_open_path(RUSTFS_META_BUCKET, super::super::FORMAT_BOOTSTRAP_FILE)
|
||||
.expect("native intent path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let _hook = install_at(Stage::PreparedRename, &intent, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
let old_writer = tokio::spawn({
|
||||
let disk = Arc::clone(&first);
|
||||
let bytes = Bytes::from(old_plan.to_json().expect("old intent JSON"));
|
||||
async move { disk.write_format_control(super::super::FORMAT_BOOTSTRAP_FILE, bytes).await }
|
||||
});
|
||||
timeout(Duration::from_secs(5), entered_rx)
|
||||
.await
|
||||
.expect("old physical publisher reaches rename")
|
||||
.expect("old publisher reports entry");
|
||||
old_writer.abort();
|
||||
assert!(old_writer.await.expect_err("cancel old caller").is_cancelled());
|
||||
|
||||
let reopened = LocalDisk::new(&endpoint, false).await.expect("reopen same physical root");
|
||||
assert_eq!(first.format_control_key, reopened.format_control_key);
|
||||
assert!(
|
||||
futures::FutureExt::now_or_never(
|
||||
os::acquire_namespace_mutation_lease_with_owner(&reopened.format_control_key, None,)
|
||||
)
|
||||
.is_none(),
|
||||
"the cancelled publisher retains the shared physical owner"
|
||||
);
|
||||
let new_plan = old_plan.clone();
|
||||
let new_bytes = Bytes::from(format!("{}\n", new_plan.to_json().expect("accepted intent JSON")));
|
||||
let mut replacement = std::pin::pin!(async {
|
||||
reopened
|
||||
.write_format_control(super::super::FORMAT_BOOTSTRAP_FILE, new_bytes.clone())
|
||||
.await?;
|
||||
reopened
|
||||
.write_format_control(super::super::FORMAT_CONFIG_FILE, new_bytes.clone())
|
||||
.await?;
|
||||
reopened
|
||||
.rename_file_durable(
|
||||
RUSTFS_META_BUCKET,
|
||||
super::super::FORMAT_BOOTSTRAP_FILE,
|
||||
RUSTFS_META_BUCKET,
|
||||
super::super::FORMAT_BOOTSTRAP_DONE_FILE,
|
||||
)
|
||||
.await
|
||||
});
|
||||
assert!(futures::FutureExt::now_or_never(&mut replacement).is_none());
|
||||
assert!(
|
||||
!directory
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(super::super::FORMAT_BOOTSTRAP_DONE_FILE)
|
||||
.exists()
|
||||
);
|
||||
release_tx.send(()).expect("release old owned publication");
|
||||
timeout(Duration::from_secs(5), &mut replacement)
|
||||
.await
|
||||
.expect("replacement progresses after old native owner exits")
|
||||
.expect("replacement and retirement commit");
|
||||
assert_eq!(
|
||||
fs::read(
|
||||
directory
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(super::super::FORMAT_BOOTSTRAP_DONE_FILE)
|
||||
)
|
||||
.await
|
||||
.expect("retired winning plan"),
|
||||
new_bytes
|
||||
);
|
||||
let restarted = crate::disk::new_disk(
|
||||
&endpoint,
|
||||
&crate::disk::DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("restart physical drive");
|
||||
let mut disks = vec![Some(restarted)];
|
||||
let loaded = crate::store::init_format::connect_load_init_formats_with_instance_ctx(
|
||||
&Arc::new(crate::runtime::instance::InstanceContext::new()),
|
||||
true,
|
||||
&mut disks,
|
||||
1,
|
||||
1,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("restart loads the winning format without stale intent resurrection");
|
||||
assert_eq!(loaded.format.shared_identity(), new_plan.shared_identity());
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, crate::core::pools::PoolMetaBootstrapAuthority::None);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn format_control_owners_separate_replacement_roots_at_the_same_endpoint() {
|
||||
let directory = tempfile::tempdir().expect("replacement root fixture");
|
||||
let root = directory.path().join("drive");
|
||||
fs::create_dir(&root).await.expect("initial drive root");
|
||||
let endpoint = Endpoint::try_from(root.to_str().expect("fixture path is UTF-8")).expect("drive endpoint");
|
||||
let original = LocalDisk::new(&endpoint, false).await.expect("original drive");
|
||||
let owner = os::acquire_namespace_mutation_lease_with_owner(&original.format_control_key, None).await;
|
||||
fs::rename(&root, directory.path().join("retired-drive"))
|
||||
.await
|
||||
.expect("retain original physical root at another name");
|
||||
fs::create_dir(&root).await.expect("replacement drive root");
|
||||
let replacement = LocalDisk::new(&endpoint, false).await.expect("replacement drive");
|
||||
assert_ne!(original.format_control_key, replacement.format_control_key);
|
||||
let (_, plan) = format_control_test_plan();
|
||||
timeout(
|
||||
Duration::from_secs(5),
|
||||
replacement.write_format_control(super::super::FORMAT_BOOTSTRAP_FILE, plan),
|
||||
)
|
||||
.await
|
||||
.expect("an old physical owner cannot block a replacement drive")
|
||||
.expect("replacement publication");
|
||||
drop(owner);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn format_control_retirement_retries_directory_sync_after_source_disappears() {
|
||||
let directory = tempfile::tempdir().expect("format retirement fixture");
|
||||
let endpoint =
|
||||
Endpoint::try_from(directory.path().to_str().expect("fixture path is UTF-8")).expect("format retirement endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("format retirement disk");
|
||||
let (_, plan) = format_control_test_plan();
|
||||
disk.write_format_control(super::super::FORMAT_BOOTSTRAP_FILE, plan.clone())
|
||||
.await
|
||||
.expect("durable bootstrap intent");
|
||||
let parent = directory.path().join(RUSTFS_META_BUCKET);
|
||||
for _ in 0..2 {
|
||||
os::fsync_dir_recorder::set_failure(&parent, ErrorKind::Other);
|
||||
disk.rename_file_durable(
|
||||
RUSTFS_META_BUCKET,
|
||||
super::super::FORMAT_BOOTSTRAP_FILE,
|
||||
RUSTFS_META_BUCKET,
|
||||
super::super::FORMAT_BOOTSTRAP_DONE_FILE,
|
||||
)
|
||||
.await
|
||||
.expect_err("retirement and already-absent retry must both synchronize the parent");
|
||||
assert!(!parent.join(super::super::FORMAT_BOOTSTRAP_FILE).exists());
|
||||
}
|
||||
disk.rename_file_durable(
|
||||
RUSTFS_META_BUCKET,
|
||||
super::super::FORMAT_BOOTSTRAP_FILE,
|
||||
RUSTFS_META_BUCKET,
|
||||
super::super::FORMAT_BOOTSTRAP_DONE_FILE,
|
||||
)
|
||||
.await
|
||||
.expect("a later successful barrier completes retirement");
|
||||
assert_eq!(
|
||||
fs::read(parent.join(super::super::FORMAT_BOOTSTRAP_DONE_FILE))
|
||||
.await
|
||||
.expect("retired intent"),
|
||||
plan
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_control_publication_keeps_metadata_reads_on_foreground_lane() {
|
||||
let directory = tempfile::tempdir().expect("format lane fixture");
|
||||
let endpoint =
|
||||
Endpoint::try_from(directory.path().to_str().expect("fixture path is UTF-8")).expect("format lane endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("format lane disk");
|
||||
let metadata = directory.path().join("existing-metadata");
|
||||
fs::write(&metadata, b"readable").await.expect("foreground metadata");
|
||||
let mut releases = Vec::new();
|
||||
let mut blockers = Vec::new();
|
||||
for _ in 0..os::MAX_FSYNC_WORKERS_PER_DISK {
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let queue = Arc::clone(&disk.blocking_queue);
|
||||
blockers.push(tokio::spawn(async move {
|
||||
queue
|
||||
.run_fsync_cancellable(move |_| {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.blocking_recv();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
}));
|
||||
started_rx.await.expect("fsync worker reached barrier");
|
||||
releases.push(release_tx);
|
||||
}
|
||||
let (_, plan) = format_control_test_plan();
|
||||
let mut publication = std::pin::pin!(disk.write_format_control(super::super::FORMAT_BOOTSTRAP_FILE, plan));
|
||||
let queued = futures::FutureExt::now_or_never(&mut publication).is_none();
|
||||
let read = timeout(
|
||||
Duration::from_secs(5),
|
||||
disk.run_metadata_read(None, metadata, |opened| {
|
||||
let mut file = opened?;
|
||||
let mut bytes = Vec::new();
|
||||
std::io::Read::read_to_end(&mut file, &mut bytes)?;
|
||||
Ok::<_, std::io::Error>(bytes)
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let published_while_blocked = directory
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(super::super::FORMAT_BOOTSTRAP_FILE)
|
||||
.exists();
|
||||
for release in releases {
|
||||
release.send(()).expect("release fsync worker");
|
||||
}
|
||||
for blocker in blockers {
|
||||
blocker.await.expect("join fsync blocker").expect("fsync blocker result");
|
||||
}
|
||||
publication
|
||||
.await
|
||||
.expect("atomic publication completes after fsync lane release");
|
||||
assert!(queued && !published_while_blocked, "format publication must use the fsync lane");
|
||||
assert_eq!(
|
||||
read.expect("foreground read progresses while fsync workers wait")
|
||||
.expect("metadata executor")
|
||||
.expect("metadata bytes"),
|
||||
b"readable"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_metadata_rejects_default_like_delete_marker() {
|
||||
use tempfile::tempdir;
|
||||
|
||||
@@ -58,6 +58,21 @@ pub const BUCKET_META_PREFIX: &str = "buckets";
|
||||
/// a user bucket of the same name, so bucket deletion must leave it in place.
|
||||
pub const HEAL_REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement";
|
||||
pub const FORMAT_CONFIG_FILE: &str = "format.json";
|
||||
pub(crate) const FORMAT_BOOTSTRAP_FILE: &str = "format.json.bootstrap";
|
||||
pub(crate) const FORMAT_BOOTSTRAP_DONE_FILE: &str = "format.json.bootstrap.done";
|
||||
|
||||
/// Validate an internal format control filename for atomic publication.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`DiskError::MethodNotAllowed`] for every other path.
|
||||
pub fn validate_format_control_path(path: &str) -> Result<()> {
|
||||
if matches!(path, FORMAT_CONFIG_FILE | FORMAT_BOOTSTRAP_FILE) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-disk marker present while an erasure-set heal is rebuilding this disk.
|
||||
/// `LocalDisk::disk_info` reports `healing = true` while the file exists.
|
||||
pub const HEALING_MARKER_PATH: &str = "healing.bin";
|
||||
@@ -1170,6 +1185,13 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_format_control(&self, path: &str, data: Bytes) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.write_format_control(path, data).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.write_format_control(path, data).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn compare_and_update_file(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1905,6 +1927,21 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>>;
|
||||
// CleanAbandonedData
|
||||
async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()>;
|
||||
/// Atomically publish a valid per-drive `FormatV3` control file in the system volume.
|
||||
/// Deployment, topology and member UUID must agree with existing control records.
|
||||
/// Pending intents admit equivalent plans, and retired intents reject bootstrap
|
||||
/// republication. Success requires a durable replacement inode and parent entry.
|
||||
/// Cancellation can leave the complete plan published. Retries revalidate the
|
||||
/// plan under the per-disk owner before publishing it durably.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`DiskError::MethodNotAllowed`] for unsupported implementations,
|
||||
/// paths or retired bootstrap intents, [`DiskError::CorruptedFormat`] for
|
||||
/// invalid or conflicting plans, and an I/O error when publication or its
|
||||
/// durability barrier fails.
|
||||
async fn write_format_control(&self, _path: &str, _data: Bytes) -> Result<()> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes>;
|
||||
/// Atomically replace or remove a small control file only when its current
|
||||
/// contents match `expected`. Implementations that cannot provide this
|
||||
@@ -2860,6 +2897,26 @@ pub fn count_part_not_success(part_errs: &[usize]) -> usize {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn format_control_paths_are_fixed_system_leaf_names() {
|
||||
for path in [super::FORMAT_CONFIG_FILE, super::FORMAT_BOOTSTRAP_FILE] {
|
||||
super::validate_format_control_path(path).expect("allowed format control leaf");
|
||||
}
|
||||
for path in [
|
||||
"",
|
||||
"../format.json",
|
||||
"format.json/",
|
||||
".rustfs.sys/format.json",
|
||||
"format.json.bootstrap.done",
|
||||
"pool.bin",
|
||||
] {
|
||||
assert!(matches!(
|
||||
super::validate_format_control_path(path),
|
||||
Err(super::DiskError::MethodNotAllowed)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::init_format::{
|
||||
check_format_erasure_values, formats_match_reference_slots, legacy_format_max_bytes, read_legacy_format_bytes,
|
||||
retain_format_quorum_members, write_format_file,
|
||||
};
|
||||
use crate::disk::{DiskAPI, DiskStore, RUSTFS_META_BUCKET, error::DiskError, format::FormatV3};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use futures::{StreamExt, future::join_all, stream};
|
||||
use std::sync::Arc;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::disk::FORMAT_BOOTSTRAP_DONE_FILE;
|
||||
pub(super) use crate::disk::FORMAT_BOOTSTRAP_FILE;
|
||||
const FORMAT_BOOTSTRAP_READ_CONCURRENCY: usize = 16;
|
||||
|
||||
pub(super) fn project_format_slots(reference: &FormatV3) -> Vec<Option<FormatV3>> {
|
||||
reference
|
||||
.erasure
|
||||
.sets
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|disk_id| {
|
||||
let mut format = reference.clone();
|
||||
format.erasure.this = *disk_id;
|
||||
Some(format)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Establishes restartable creator authority before any format is published.
|
||||
/// The first slot durably selects the plan before fanout. A retry completes
|
||||
/// that immutable plan while every configured disk remains unformatted.
|
||||
pub(super) async fn write_format_bootstrap_intents(disks: &[Option<DiskStore>], formats: &[Option<FormatV3>]) -> Result<()> {
|
||||
if disks.is_empty() || disks.len() != formats.len() || formats.iter().any(Option::is_none) {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
if disks.iter().any(Option::is_none) {
|
||||
return Err(DiskError::DiskNotFound.into());
|
||||
}
|
||||
let first = disks[0].as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
let anchor = formats[0].as_ref().ok_or(DiskError::CorruptedFormat)?;
|
||||
let set_drive_count = anchor.erasure.sets.first().map_or(0, Vec::len);
|
||||
check_format_erasure_values(formats, set_drive_count)?;
|
||||
if !formats_match_reference_slots(formats, anchor, 0) {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
let max_bytes = legacy_format_max_bytes(disks.len())?;
|
||||
match first.read_file(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE).await {
|
||||
Ok(reader) => {
|
||||
let data = read_legacy_format_bytes(reader, max_bytes).await?;
|
||||
let existing = FormatV3::try_from(data.as_slice())?;
|
||||
if existing.shared_identity() != anchor.shared_identity() || existing.erasure.this != anchor.erasure.this {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
}
|
||||
Err(DiskError::FileNotFound) => {
|
||||
// A missing anchor cannot authorize replacing another slot's plan.
|
||||
// This check is confined to fresh creation, so healthy restarts
|
||||
// retain the single-anchor absence fast path.
|
||||
let reads = stream::iter(disks.iter().skip(1).cloned().map(|disk| async move {
|
||||
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
match disk.read_file(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE).await {
|
||||
Err(DiskError::FileNotFound) => Ok(()),
|
||||
Ok(_) => Err(DiskError::CorruptedFormat),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}))
|
||||
.buffer_unordered(FORMAT_BOOTSTRAP_READ_CONCURRENCY);
|
||||
tokio::pin!(reads);
|
||||
while let Some(result) = reads.next().await {
|
||||
result?;
|
||||
}
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
first
|
||||
.write_format_control(FORMAT_BOOTSTRAP_FILE, anchor.to_json()?.into_bytes().into())
|
||||
.await?;
|
||||
let writes = disks.iter().zip(formats).skip(1).map(|(disk, format)| async move {
|
||||
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
let format = format.as_ref().ok_or(DiskError::CorruptedFormat)?;
|
||||
disk.write_format_control(FORMAT_BOOTSTRAP_FILE, format.to_json()?.into_bytes().into())
|
||||
.await
|
||||
});
|
||||
for result in join_all(writes).await {
|
||||
result?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Completes an anchored plan before its first format publication, or restores
|
||||
/// explicitly unformatted slots from a complete durable intent set. Partial
|
||||
/// intent sets grant no authority once any format has been published.
|
||||
pub(super) async fn resume_format_bootstrap(
|
||||
instance_ctx: &Arc<InstanceContext>,
|
||||
disks: &mut [Option<DiskStore>],
|
||||
formats: &[Option<FormatV3>],
|
||||
errors: &[Option<DiskError>],
|
||||
set_count: usize,
|
||||
set_drive_count: usize,
|
||||
deployment_id: Option<Uuid>,
|
||||
) -> Result<Option<FormatV3>> {
|
||||
if disks.is_empty()
|
||||
|| disks.len() != formats.len()
|
||||
|| disks.len() != errors.len()
|
||||
|| set_count.checked_mul(set_drive_count) != Some(disks.len())
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let max_bytes = legacy_format_max_bytes(disks.len())?;
|
||||
let Some(first) = disks.first().and_then(Option::as_ref) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let first_intent = async {
|
||||
let reader = first.read_file(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE).await?;
|
||||
let data = read_legacy_format_bytes(reader, max_bytes).await?;
|
||||
FormatV3::try_from(data.as_slice()).map_err(DiskError::from)
|
||||
}
|
||||
.await;
|
||||
let Ok(first_intent) = first_intent else {
|
||||
return Ok(None);
|
||||
};
|
||||
if first_intent.erasure.sets.len() != set_count
|
||||
|| first_intent.erasure.sets.iter().any(|set| set.len() != set_drive_count)
|
||||
|| deployment_id.is_some_and(|id| id != first_intent.id)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let projected = project_format_slots(&first_intent);
|
||||
if check_format_erasure_values(&projected, set_drive_count).is_err()
|
||||
|| !formats_match_reference_slots(&[Some(first_intent.clone())], &first_intent, 0)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let all_unformatted =
|
||||
formats.iter().all(Option::is_none) && errors.iter().all(|error| matches!(error, Some(DiskError::UnformattedDisk)));
|
||||
let intents = {
|
||||
let reads = stream::iter(disks.iter().cloned().enumerate().skip(1).map(|(index, disk)| async move {
|
||||
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
let reader = match disk.read_file(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE).await {
|
||||
Ok(reader) => reader,
|
||||
Err(DiskError::FileNotFound) => return Ok((index, None)),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let data = read_legacy_format_bytes(reader, max_bytes).await?;
|
||||
let format = FormatV3::try_from(data.as_slice())?;
|
||||
Ok::<_, DiskError>((index, Some(format)))
|
||||
}))
|
||||
.buffer_unordered(FORMAT_BOOTSTRAP_READ_CONCURRENCY);
|
||||
tokio::pin!(reads);
|
||||
let mut intents = vec![None; disks.len()];
|
||||
intents[0] = Some(first_intent);
|
||||
while let Some(result) = reads.next().await {
|
||||
let Ok((index, format)) = result else {
|
||||
return Ok(None);
|
||||
};
|
||||
intents[index] = format;
|
||||
}
|
||||
intents
|
||||
};
|
||||
let Some(reference) = intents.first().and_then(Option::as_ref) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !formats_match_reference_slots(&intents, reference, 0) {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
if intents.iter().any(Option::is_none) && !all_unformatted {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if !formats_match_reference_slots(formats, reference, 0) {
|
||||
return Err(Error::CorruptedFormat);
|
||||
}
|
||||
if formats.iter().zip(errors).any(|(format, error)| {
|
||||
if format.is_some() {
|
||||
error.is_some()
|
||||
} else {
|
||||
!matches!(error, Some(DiskError::UnformattedDisk))
|
||||
}
|
||||
}) {
|
||||
return Ok(None);
|
||||
}
|
||||
if all_unformatted {
|
||||
// Re-acknowledge every slot under native immutable-plan admission.
|
||||
// Merely observing a visible, unacknowledged intent supplies no barrier.
|
||||
write_format_bootstrap_intents(disks, &projected).await?;
|
||||
}
|
||||
|
||||
// Every intent and existing format has been checked before the first
|
||||
// mutation. A retry always publishes the same deployment and slot IDs.
|
||||
let writes = disks
|
||||
.iter()
|
||||
.zip(&projected)
|
||||
.zip(formats)
|
||||
.filter_map(|((disk, intent), existing)| {
|
||||
if existing.is_some() {
|
||||
return None;
|
||||
}
|
||||
Some(write_format_file(disk.as_ref()?, intent.as_ref()?))
|
||||
});
|
||||
for result in join_all(writes).await {
|
||||
result?;
|
||||
}
|
||||
let mut format = reference.clone();
|
||||
format.erasure.this = Uuid::nil();
|
||||
format.disk_info = None;
|
||||
let members = vec![true; disks.len()];
|
||||
retain_format_quorum_members(instance_ctx, disks, &format, &members, None, set_drive_count).await?;
|
||||
Ok(Some(format))
|
||||
}
|
||||
|
||||
/// Consumes this pool's creator authority before initialized identity publication.
|
||||
/// Replay requires every intent, so one durably absent slot is sufficient. Each
|
||||
/// attempt finishes before another begins, including its directory-sync barrier.
|
||||
pub(super) async fn retire_format_bootstrap_authority(disks: &[Option<DiskStore>]) -> Result<()> {
|
||||
let first = disks.first().and_then(Option::as_ref).into_iter();
|
||||
let local = disks.iter().skip(1).flatten().filter(|disk| disk.is_local());
|
||||
let remote = disks.iter().skip(1).flatten().filter(|disk| !disk.is_local());
|
||||
let mut failure = DiskError::DiskNotFound;
|
||||
for disk in first.chain(local).chain(remote) {
|
||||
match disk
|
||||
.rename_file_durable(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE, RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_DONE_FILE)
|
||||
.await
|
||||
{
|
||||
Ok(()) => return Ok(()),
|
||||
Err(error) => failure = error,
|
||||
}
|
||||
}
|
||||
Err(failure.into())
|
||||
}
|
||||
@@ -235,7 +235,22 @@ async fn persist_pool_meta_for_startup_if_safe<S>(
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
persist_pool_meta_for_startup_guarded(meta, pools, replica_state, write_state, topology_update, elected_writer, None).await
|
||||
let mut retirement = std::pin::pin!(std::future::ready(Ok(())));
|
||||
if write_state.startup_identity_precedes_metadata_write() {
|
||||
complete_pool_meta_for_startup(pools.clone(), write_state, elected_writer, None, retirement.as_mut()).await?;
|
||||
}
|
||||
let committed = persist_pool_meta_for_startup_guarded(
|
||||
meta,
|
||||
pools.clone(),
|
||||
replica_state,
|
||||
write_state,
|
||||
topology_update,
|
||||
elected_writer,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
complete_pool_meta_for_startup(pools, write_state, elected_writer, None, retirement.as_mut()).await?;
|
||||
Ok(committed)
|
||||
}
|
||||
|
||||
async fn persist_pool_meta_for_startup_guarded<S>(
|
||||
@@ -251,6 +266,11 @@ where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
if !elected_writer {
|
||||
if topology_update {
|
||||
return Err(Error::other(
|
||||
"pool metadata bootstrap pending: waiting for the elected writer to commit the configured topology",
|
||||
));
|
||||
}
|
||||
return Ok(meta.clone());
|
||||
}
|
||||
let should_write = topology_update || (replica_state.needs_repair && replica_state.repair_write_safe);
|
||||
@@ -275,12 +295,30 @@ where
|
||||
None => save_validated_pool_meta_for_startup(meta, pools.clone(), write_state).await?,
|
||||
};
|
||||
}
|
||||
if should_write || write_state.identity_requires_repair() {
|
||||
persist_startup_identity_with_guard(pools, write_state, true, writer.as_ref())
|
||||
Ok(committed)
|
||||
}
|
||||
|
||||
async fn complete_pool_meta_for_startup<S>(
|
||||
pools: Vec<Arc<S>>,
|
||||
write_state: &mut PoolMetaWriteState,
|
||||
elected_writer: bool,
|
||||
writer: Option<&Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
before_identity_commit: impl Future<Output = Result<()>>,
|
||||
) -> Result<()>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
if elected_writer && write_state.identity_requires_repair() {
|
||||
write_state.ensure_write_safe("store init failed during prepare_pool_meta_identity")?;
|
||||
// A committed predecessor or initial metadata commit makes retirement
|
||||
// recoverable. Every node requires the committed configured topology
|
||||
// and complete initialized identity before publication.
|
||||
before_identity_commit.await?;
|
||||
persist_startup_identity_with_guard(pools, write_state, true, writer)
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("store init failed during commit_pool_meta_identity: {err}")))?;
|
||||
}
|
||||
Ok(committed)
|
||||
write_state.ensure_startup_identity_initialized()
|
||||
}
|
||||
|
||||
async fn run_local_decommission_watchdog<F, Fut, W, Wait>(rx: CancellationToken, mut reconcile: F, mut wait_for_fleet_proof: W)
|
||||
@@ -613,8 +651,8 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
}?;
|
||||
// First-hand authority for this pool only: `Fresh` when this process
|
||||
// formatted it, `LegacyAdoption` when it verified the migration, and
|
||||
// Creator authority for this pool only: `Fresh` when this process
|
||||
// formatted it or resumed its durable plan, `LegacyAdoption` for migration, and
|
||||
// `None` when it merely read a format another node created.
|
||||
pool_meta_bootstrap_authorities.push(loaded_format.pool_meta_bootstrap_authority);
|
||||
let fm = loaded_format.format;
|
||||
@@ -797,15 +835,47 @@ impl ECStore {
|
||||
// distributed startup can race on the same lock and replay the prior init bug.
|
||||
{
|
||||
let write_state = &mut *startup_write_state;
|
||||
let persist = persist_pool_meta_for_startup_guarded(
|
||||
&installed_pool_meta,
|
||||
self.pools.clone(),
|
||||
pool_meta_replica_state,
|
||||
write_state,
|
||||
update,
|
||||
should_persist_pool_meta,
|
||||
startup_pool_meta_guard.clone(),
|
||||
);
|
||||
let persist = async {
|
||||
let mut retirement = std::pin::pin!(async {
|
||||
for pool in &self.pools {
|
||||
let mut disks = Vec::with_capacity(pool.set_count * pool.set_drive_count);
|
||||
for set in &pool.disk_set {
|
||||
disks.extend(set.disk_inventory().await);
|
||||
}
|
||||
super::format_bootstrap::retire_format_bootstrap_authority(&disks).await?;
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
if write_state.startup_identity_precedes_metadata_write() {
|
||||
complete_pool_meta_for_startup(
|
||||
self.pools.clone(),
|
||||
write_state,
|
||||
should_persist_pool_meta,
|
||||
startup_pool_meta_guard.as_ref(),
|
||||
retirement.as_mut(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let committed = persist_pool_meta_for_startup_guarded(
|
||||
&installed_pool_meta,
|
||||
self.pools.clone(),
|
||||
pool_meta_replica_state,
|
||||
write_state,
|
||||
update,
|
||||
should_persist_pool_meta,
|
||||
startup_pool_meta_guard.clone(),
|
||||
)
|
||||
.await?;
|
||||
complete_pool_meta_for_startup(
|
||||
self.pools.clone(),
|
||||
write_state,
|
||||
should_persist_pool_meta,
|
||||
startup_pool_meta_guard.as_ref(),
|
||||
retirement.as_mut(),
|
||||
)
|
||||
.await?;
|
||||
Ok::<_, Error>(committed)
|
||||
};
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let persist = crate::core::pools::startup_cas_test_scope(startup_attempt, "persist", &self.pools, persist);
|
||||
installed_pool_meta = persist.await?;
|
||||
@@ -988,11 +1058,12 @@ mod tests {
|
||||
include!("init/multipart_copy_heal_tests.rs");
|
||||
use super::{
|
||||
LOCAL_DECOMMISSION_RESUME_RETRY_DELAY, LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY, PoolMetaWriteState,
|
||||
establish_pool_meta_bootstrap_identity_if_proven, load_pool_meta_for_startup, local_decommission_watchdog_retry_delay,
|
||||
persist_pool_meta_for_startup_if_safe, pool_first_endpoint_is_local, pool_meta_has_active_decommission,
|
||||
preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with, resolve_store_init_stage_result,
|
||||
run_local_decommission_watchdog, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init,
|
||||
should_defer_rebalance_auto_start, should_retry_format_load, wait_for_local_decommission_resume_delay,
|
||||
complete_pool_meta_for_startup, establish_pool_meta_bootstrap_identity_if_proven, load_pool_meta_for_startup,
|
||||
local_decommission_watchdog_retry_delay, persist_pool_meta_for_startup_guarded, persist_pool_meta_for_startup_if_safe,
|
||||
pool_first_endpoint_is_local, pool_meta_has_active_decommission, preflight_startup_rpc_secret_with,
|
||||
resolve_startup_pool_defaults_with, resolve_store_init_stage_result, run_local_decommission_watchdog,
|
||||
save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init, should_defer_rebalance_auto_start,
|
||||
should_retry_format_load, wait_for_local_decommission_resume_delay,
|
||||
};
|
||||
use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
|
||||
use crate::core::pools::PoolMetaBootstrapAuthority;
|
||||
@@ -1570,6 +1641,184 @@ mod tests {
|
||||
.await
|
||||
.expect_err("wiping pool metadata after the crash must require recovery");
|
||||
assert!(err.to_string().contains("initialized cluster identity exists"));
|
||||
restarted
|
||||
.ensure_write_safe("metadata erased after failed identity repair")
|
||||
.expect_err("metadata loss cannot recreate creator authority");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_startup_identity_publication_waits_for_bootstrap_retirement() {
|
||||
let deployment_id = Uuid::new_v4();
|
||||
let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
|
||||
let pools = vec![storage.clone()];
|
||||
let mut state = PoolMetaWriteState::for_startup(deployment_id, true);
|
||||
establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut state, true)
|
||||
.await
|
||||
.expect("persist initial pending identity");
|
||||
let (_, replicas) = load_pool_meta_for_startup(pools.clone(), &mut state)
|
||||
.await
|
||||
.expect("load fresh bootstrap state");
|
||||
persist_pool_meta_for_startup_guarded(&init_test_pool_meta(None), pools.clone(), replicas, &mut state, true, true, None)
|
||||
.await
|
||||
.expect("commit initial pool metadata before consuming creator authority");
|
||||
let identity_writes = storage.identity_write_attempts.load(Ordering::SeqCst);
|
||||
|
||||
complete_pool_meta_for_startup(pools.clone(), &mut state, true, None, async {
|
||||
let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let metadata = objects
|
||||
.get(POOL_META_NAME)
|
||||
.expect("committed metadata must precede retirement");
|
||||
assert_eq!(
|
||||
pool_meta_v3_commit_state_for_test(metadata.0.clone()).expect("decode committed metadata"),
|
||||
(1, true)
|
||||
);
|
||||
let identity = objects
|
||||
.get(POOL_META_IDENTITY_NAME)
|
||||
.expect("pending identity must remain available");
|
||||
assert!(
|
||||
!pool_meta_identity_initialized_for_test(&identity.0).expect("decode pending identity"),
|
||||
"initialized publication must follow durable retirement"
|
||||
);
|
||||
Err(Error::Timeout)
|
||||
})
|
||||
.await
|
||||
.expect_err("failed retirement must prevent initialized identity publication");
|
||||
assert_eq!(storage.identity_write_attempts.load(Ordering::SeqCst), identity_writes);
|
||||
state
|
||||
.ensure_startup_identity_initialized()
|
||||
.expect_err("a failed retirement must keep startup unpublished");
|
||||
|
||||
// Restarting after retirement failed reuses the committed metadata,
|
||||
// without requiring the formatter's process-local bootstrap proof.
|
||||
let mut restarted = PoolMetaWriteState::for_startup(deployment_id, false);
|
||||
load_pool_meta_for_startup(pools.clone(), &mut restarted)
|
||||
.await
|
||||
.expect("committed metadata authorizes the retry");
|
||||
complete_pool_meta_for_startup(pools, &mut restarted, true, None, std::future::ready(Ok(())))
|
||||
.await
|
||||
.expect("successful retirement permits initialized identity publication");
|
||||
assert!(storage.identity_write_attempts.load(Ordering::SeqCst) > identity_writes);
|
||||
restarted
|
||||
.ensure_startup_identity_initialized()
|
||||
.expect("completed initialization permits startup");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_startup_followers_wait_for_initialized_identity_on_every_pool() {
|
||||
let deployment_id = Uuid::new_v4();
|
||||
let initialized = initialized_pool_meta_identity_for_test(deployment_id, 1).expect("encode initialized identity");
|
||||
let pending = pending_pool_meta_identity_for_test(deployment_id, 1, Uuid::new_v4()).expect("encode pending identity");
|
||||
for incomplete in [None, Some(pending), Some(vec![0, 1, 2])] {
|
||||
let payload = startup_pool_meta_payload(&init_test_pool_meta(None));
|
||||
let first = Arc::new(StartupPoolMetaStorage::new(payload.clone()));
|
||||
let second = Arc::new(StartupPoolMetaStorage::new(payload));
|
||||
first
|
||||
.objects
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(POOL_META_IDENTITY_NAME.to_owned(), (initialized.clone(), "identity-0".to_owned()));
|
||||
if let Some(identity) = incomplete {
|
||||
second
|
||||
.objects
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(POOL_META_IDENTITY_NAME.to_owned(), (identity, "identity-1".to_owned()));
|
||||
}
|
||||
let pools = vec![first.clone(), second.clone()];
|
||||
let mut state = PoolMetaWriteState::for_startup(deployment_id, false);
|
||||
load_pool_meta_for_startup(pools.clone(), &mut state)
|
||||
.await
|
||||
.expect("a readable committed snapshot can await identity convergence");
|
||||
complete_pool_meta_for_startup(pools.clone(), &mut state, false, None, async {
|
||||
panic!("a follower must never execute intent retirement")
|
||||
})
|
||||
.await
|
||||
.expect_err("an initialized replica cannot hide another pool's incomplete identity");
|
||||
state
|
||||
.ensure_write_safe("follower bootstrap retry")
|
||||
.expect("waiting must remain retryable");
|
||||
|
||||
second
|
||||
.objects
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(POOL_META_IDENTITY_NAME.to_owned(), (initialized.clone(), "identity-1-ready".to_owned()));
|
||||
load_pool_meta_for_startup(pools.clone(), &mut state)
|
||||
.await
|
||||
.expect("observe initialized identity on every pool");
|
||||
for elected_writer in [false, true] {
|
||||
complete_pool_meta_for_startup(pools.clone(), &mut state, elected_writer, None, async {
|
||||
panic!("established startup must perform no intent retirement I/O")
|
||||
})
|
||||
.await
|
||||
.expect("complete initialized identity permits read-only startup");
|
||||
}
|
||||
assert_eq!(first.identity_write_attempts.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(second.identity_write_attempts.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_startup_expansion_retires_before_identity_repair_and_waits_for_topology_commit() {
|
||||
let deployment_id = Uuid::new_v4();
|
||||
let first = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&init_test_pool_meta_with_pools(1))));
|
||||
let second = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
|
||||
inject_startup_object(
|
||||
&first,
|
||||
POOL_META_IDENTITY_NAME,
|
||||
initialized_pool_meta_identity_for_test(deployment_id, 1).expect("encode predecessor identity"),
|
||||
);
|
||||
let pools = vec![first.clone(), second.clone()];
|
||||
let mut state = PoolMetaWriteState::for_startup(deployment_id, false);
|
||||
let (_, replicas) = load_pool_meta_for_startup(pools.clone(), &mut state)
|
||||
.await
|
||||
.expect("load committed predecessor with a new pool");
|
||||
assert!(state.startup_identity_precedes_metadata_write());
|
||||
|
||||
complete_pool_meta_for_startup(pools.clone(), &mut state, true, None, async { Err(Error::Timeout) })
|
||||
.await
|
||||
.expect_err("failed retirement must stop identity repair before expansion");
|
||||
for pool in &pools {
|
||||
assert_eq!(pool.identity_write_attempts.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(pool.pool_meta_write_attempts.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
complete_pool_meta_for_startup(pools.clone(), &mut state, true, None, async {
|
||||
assert!(startup_object(&second, POOL_META_IDENTITY_NAME).is_none());
|
||||
assert!(startup_object(&first, POOL_META_NAME).is_some());
|
||||
assert!(startup_object(&second, POOL_META_NAME).is_none());
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.expect("retirement authorizes repairing initialized identities from the predecessor");
|
||||
state
|
||||
.ensure_startup_identity_initialized()
|
||||
.expect("every pool identity is repaired before native metadata publication");
|
||||
let expanded = init_test_pool_meta_with_pools(2);
|
||||
let identity_writes = pools
|
||||
.iter()
|
||||
.map(|pool| pool.identity_write_attempts.load(Ordering::SeqCst))
|
||||
.collect::<Vec<_>>();
|
||||
let error = persist_pool_meta_for_startup_guarded(&expanded, pools.clone(), replicas, &mut state, true, false, None)
|
||||
.await
|
||||
.expect_err("a follower cannot publish locally computed expanded topology");
|
||||
assert!(error.to_string().contains("commit the configured topology"));
|
||||
for (pool, identity_writes) in pools.iter().zip(identity_writes) {
|
||||
assert_eq!(pool.identity_write_attempts.load(Ordering::SeqCst), identity_writes);
|
||||
assert_eq!(pool.pool_meta_write_attempts.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
state
|
||||
.ensure_write_safe("waiting for topology commit")
|
||||
.expect("a follower retry must stay writable");
|
||||
|
||||
let committed = persist_pool_meta_for_startup_guarded(&expanded, pools.clone(), replicas, &mut state, true, true, None)
|
||||
.await
|
||||
.expect("the elected writer can commit expansion after identity repair");
|
||||
assert_eq!(committed.pools.len(), 2);
|
||||
complete_pool_meta_for_startup(pools, &mut state, true, None, async {
|
||||
panic!("the completed preflight retirement must not run twice")
|
||||
})
|
||||
.await
|
||||
.expect("completed expansion remains initialized");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3482,6 +3731,128 @@ mod tests {
|
||||
(instance_ctx, store, shutdown)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn interrupted_format_bootstrap_recovers_two_pools_through_store_init() {
|
||||
use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET};
|
||||
use crate::store::init_format;
|
||||
let directory = tempfile::tempdir().expect("create interrupted-bootstrap deployment");
|
||||
let deployment_id = Uuid::new_v4();
|
||||
let mut plans = Vec::new();
|
||||
for pool_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
for index in 0..4 {
|
||||
let path = directory.path().join(format!("pool{pool_index}/set0/disk{index}"));
|
||||
tokio::fs::create_dir_all(&path).await.expect("create bootstrap disk");
|
||||
let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("disk endpoint");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(index);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
let (disks, errors) = init_format::init_disks(
|
||||
&crate::runtime::instance::InstanceContext::new(),
|
||||
&Endpoints::from(endpoints),
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(errors.iter().all(Option::is_none), "open bootstrap disks: {errors:?}");
|
||||
let mut reference = crate::disk::format::FormatV3::new(1, 4);
|
||||
reference.id = deployment_id;
|
||||
let formats = (0..4)
|
||||
.map(|index| {
|
||||
let mut format = reference.clone();
|
||||
format.erasure.this = reference.erasure.sets[0][index];
|
||||
Some(format)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
super::super::format_bootstrap::write_format_bootstrap_intents(&disks, &formats)
|
||||
.await
|
||||
.expect("persist creator plan");
|
||||
for (disk, format) in disks.iter().zip(&formats).take(3) {
|
||||
disk.as_ref()
|
||||
.expect("fixture disk")
|
||||
.write_all(
|
||||
RUSTFS_META_BUCKET,
|
||||
FORMAT_CONFIG_FILE,
|
||||
format
|
||||
.as_ref()
|
||||
.expect("planned format")
|
||||
.to_json()
|
||||
.expect("encode format")
|
||||
.into_bytes()
|
||||
.into(),
|
||||
)
|
||||
.await
|
||||
.expect("publish first three formats");
|
||||
}
|
||||
plans.push(formats);
|
||||
}
|
||||
// A new context and new disk handles exercise the normal constructor's
|
||||
// format recovery, pool identity transaction, and publication barrier.
|
||||
let (_, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(directory.path(), "interrupted-bootstrap", &[4, 4])).await;
|
||||
assert_eq!(store.id, deployment_id);
|
||||
for (pool, expected) in store.pools.iter().zip(&plans) {
|
||||
let mut retired = 0;
|
||||
for set in &pool.disk_set {
|
||||
let disks = set.disk_inventory().await;
|
||||
let (formats, errors) = init_format::load_format_erasure_all(&disks, false).await;
|
||||
assert!(errors.iter().all(Option::is_none), "recovered formats: {errors:?}");
|
||||
assert_eq!(&formats, expected);
|
||||
for disk in disks.iter().flatten() {
|
||||
if matches!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, "format.json.bootstrap").await,
|
||||
Err(crate::disk::error::DiskError::FileNotFound)
|
||||
) {
|
||||
retired += 1;
|
||||
assert!(disk.read_all(RUSTFS_META_BUCKET, "format.json.bootstrap.done").await.is_ok());
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_eq!(retired, 1, "one durably absent intent consumes each pool's complete creator proof");
|
||||
}
|
||||
store
|
||||
.pool_meta_save_gate
|
||||
.read()
|
||||
.await
|
||||
.ensure_startup_identity_initialized()
|
||||
.expect("every pool identity is initialized before publication");
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket("bootstrap-recovered", &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket after recovered startup");
|
||||
store
|
||||
.put_object(
|
||||
"bootstrap-recovered",
|
||||
"payload",
|
||||
&mut PutObjReader::from_vec(b"preserved".to_vec()),
|
||||
&ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write after recovered startup");
|
||||
let info = store
|
||||
.get_object_info("bootstrap-recovered", "payload", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read committed object metadata");
|
||||
assert_eq!(info.size, 9);
|
||||
let mut reader = store
|
||||
.get_object_reader("bootstrap-recovered", "payload", None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read object after recovered startup");
|
||||
let mut payload = Vec::new();
|
||||
reader.stream.read_to_end(&mut payload).await.expect("read recovered payload");
|
||||
assert_eq!(payload, b"preserved");
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn direct_local_constructor_multipart_binds_complete_topology_without_registry_preinit() {
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(test)]
|
||||
use super::format_bootstrap::{FORMAT_BOOTSTRAP_FILE, retire_format_bootstrap_authority};
|
||||
use super::format_bootstrap::{project_format_slots, resume_format_bootstrap, write_format_bootstrap_intents};
|
||||
use crate::cluster::rpc::client::is_network_like_disk_error;
|
||||
use crate::config::storageclass;
|
||||
use crate::core::pools::PoolMetaBootstrapAuthority;
|
||||
@@ -120,6 +123,17 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
let (formats, errs, unformatted_bootstrap_routes) = load_format_erasure_all_observing_bootstrap_routes(disks, false).await;
|
||||
|
||||
check_disk_fatal_errs(&errs)?;
|
||||
// Only the elected pool creator may replay a complete durable format plan.
|
||||
// Absence of pool metadata or an orphaned format temp file grants no authority.
|
||||
if first_disk
|
||||
&& let Some(format) =
|
||||
resume_format_bootstrap(instance_ctx, disks, &formats, &errs, set_count, set_drive_count, deployment_id).await?
|
||||
{
|
||||
return Ok(LoadedFormat {
|
||||
format,
|
||||
pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::Fresh,
|
||||
});
|
||||
}
|
||||
|
||||
// Transient network errors still follow the first-node wait path so peers
|
||||
// can come online, but they are never fresh-cluster authority below.
|
||||
@@ -270,7 +284,7 @@ async fn verified_legacy_adoption_source(
|
||||
}
|
||||
}
|
||||
|
||||
async fn retain_format_quorum_members(
|
||||
pub(super) async fn retain_format_quorum_members(
|
||||
instance_ctx: &Arc<InstanceContext>,
|
||||
disks: &mut [Option<DiskStore>],
|
||||
format: &FormatV3,
|
||||
@@ -403,21 +417,13 @@ async fn init_format_erasure(
|
||||
set_drive_count: usize,
|
||||
deployment_id: Option<Uuid>,
|
||||
) -> Result<FormatV3> {
|
||||
let fm = FormatV3::new(set_count, set_drive_count);
|
||||
let mut fms = vec![None; disks.len()];
|
||||
for i in 0..set_count {
|
||||
for j in 0..set_drive_count {
|
||||
let idx = i * set_drive_count + j;
|
||||
let mut newfm = fm.clone();
|
||||
newfm.erasure.this = fm.erasure.sets[i][j];
|
||||
if let Some(id) = deployment_id {
|
||||
newfm.id = id;
|
||||
}
|
||||
|
||||
fms[idx] = Some(newfm);
|
||||
}
|
||||
let mut fm = FormatV3::new(set_count, set_drive_count);
|
||||
if let Some(id) = deployment_id {
|
||||
fm.id = id;
|
||||
}
|
||||
let fms = project_format_slots(&fm);
|
||||
|
||||
write_format_bootstrap_intents(disks, &fms).await?;
|
||||
write_format_file_all(disks, &fms).await?;
|
||||
let format = get_format_erasure_in_quorum(&fms, 0)?;
|
||||
let quorum_members = vec![true; disks.len()];
|
||||
@@ -602,14 +608,17 @@ fn select_persisted_migration_format(
|
||||
Err(write_error.map_or(quorum_error, Into::into))
|
||||
}
|
||||
|
||||
fn legacy_format_max_bytes(disk_count: usize) -> Result<usize> {
|
||||
pub(super) fn legacy_format_max_bytes(disk_count: usize) -> Result<usize> {
|
||||
disk_count
|
||||
.checked_mul(LEGACY_FORMAT_MAX_BYTES_PER_DISK)
|
||||
.and_then(|size| size.checked_add(LEGACY_FORMAT_BASE_MAX_BYTES))
|
||||
.ok_or_else(|| Error::other("legacy format size limit overflow"))
|
||||
}
|
||||
|
||||
async fn read_legacy_format_bytes(reader: impl tokio::io::AsyncRead + Unpin, max_bytes: usize) -> disk::error::Result<Vec<u8>> {
|
||||
pub(super) async fn read_legacy_format_bytes(
|
||||
reader: impl tokio::io::AsyncRead + Unpin,
|
||||
max_bytes: usize,
|
||||
) -> disk::error::Result<Vec<u8>> {
|
||||
let read_limit = max_bytes.checked_add(1).ok_or(DiskError::CorruptedFormat)?;
|
||||
let mut data = Vec::with_capacity(read_limit.min(64 * 1024));
|
||||
reader
|
||||
@@ -993,24 +1002,23 @@ pub async fn save_format_file(disk: &Option<DiskStore>, format: &Option<FormatV3
|
||||
return Err(DiskError::other("format is none"));
|
||||
};
|
||||
|
||||
write_format_file(disk, format).await?;
|
||||
// Replacement healing retains the interoperable write/rename contract
|
||||
// while configured peers are upgraded independently.
|
||||
let temporary = Uuid::new_v4().to_string();
|
||||
disk.write_all(RUSTFS_META_BUCKET, &temporary, format.to_json()?.into_bytes().into())
|
||||
.await?;
|
||||
disk.rename_file_durable(RUSTFS_META_BUCKET, &temporary, RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE)
|
||||
.await?;
|
||||
disk.set_disk_id(Some(format.erasure.this)).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn write_format_file(disk: &DiskStore, format: &FormatV3) -> disk::error::Result<()> {
|
||||
pub(super) async fn write_format_file(disk: &DiskStore, format: &FormatV3) -> disk::error::Result<()> {
|
||||
let json_data = format.to_json()?;
|
||||
|
||||
let tmpfile = Uuid::new_v4().to_string();
|
||||
|
||||
disk.write_all(RUSTFS_META_BUCKET, tmpfile.as_str(), json_data.into_bytes().into())
|
||||
.await?;
|
||||
|
||||
disk.rename_file(RUSTFS_META_BUCKET, tmpfile.as_str(), RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
disk.write_format_control(FORMAT_CONFIG_FILE, json_data.into_bytes().into())
|
||||
.await
|
||||
}
|
||||
|
||||
pub fn ec_drives_no_config(set_drive_count: usize) -> Result<usize> {
|
||||
@@ -1028,7 +1036,7 @@ mod tests {
|
||||
use crate::store::peer::{find_local_disk_by_ref, prewarm_local_disk_id_map_with_instance_ctx};
|
||||
use serial_test::serial;
|
||||
|
||||
async fn local_disks(count: usize) -> (tempfile::TempDir, Vec<Option<DiskStore>>) {
|
||||
pub(super) async fn local_disks(count: usize) -> (tempfile::TempDir, Vec<Option<DiskStore>>) {
|
||||
local_disks_with_set_width(count, count).await
|
||||
}
|
||||
|
||||
@@ -1995,7 +2003,7 @@ mod tests {
|
||||
.expect("format publication path");
|
||||
let (arrived, arrival) = tokio::sync::oneshot::channel();
|
||||
let (release, released) = std::sync::mpsc::channel();
|
||||
hooks.push(install_at(Stage::Rename, &destination, move || {
|
||||
hooks.push(install_at(Stage::PreparedRename, &destination, move || {
|
||||
let _ = arrived.send(());
|
||||
let _ = released.recv();
|
||||
}));
|
||||
@@ -2192,6 +2200,33 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_format_publication_keeps_interoperable_rename_contract() {
|
||||
use crate::disk::os::prepared_publication_test_hooks::{Stage, install_at};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
let (_directory, disks) = local_disks(1).await;
|
||||
let disk = disks[0].as_ref().expect("replacement fixture disk");
|
||||
let destination = disk.path().join(RUSTFS_META_BUCKET).join(FORMAT_CONFIG_FILE);
|
||||
let renamed = Arc::new(AtomicUsize::new(0));
|
||||
let observed = renamed.clone();
|
||||
let _hook = install_at(Stage::Rename, &destination, move || {
|
||||
observed.fetch_add(1, Ordering::SeqCst);
|
||||
});
|
||||
let mut format = FormatV3::new(1, 1);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
save_format_file(&disks[0], &Some(format.clone()))
|
||||
.await
|
||||
.expect("publish replacement through established RPCs");
|
||||
assert_eq!(
|
||||
renamed.load(Ordering::SeqCst),
|
||||
1,
|
||||
"replacement publication uses the interoperable durable rename"
|
||||
);
|
||||
assert_eq!(load_format_erasure(disk, false).await.expect("load replacement format"), format);
|
||||
assert_eq!(disk.get_disk_id().await.expect("load replacement identity"), Some(format.erasure.this));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fresh_format_load_initializes_all_disks() {
|
||||
let (_temp_dir, mut disks) = local_disks(3).await;
|
||||
@@ -2649,3 +2684,7 @@ mod tests {
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "init_format_bootstrap_tests.rs"]
|
||||
mod bootstrap_tests;
|
||||
|
||||
@@ -0,0 +1,649 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::tests::local_disks;
|
||||
use super::*;
|
||||
|
||||
async fn reopen_disks(disks: &[Option<DiskStore>]) -> Vec<Option<DiskStore>> {
|
||||
let endpoints = Endpoints::from(
|
||||
disks
|
||||
.iter()
|
||||
.map(|disk| disk.as_ref().expect("fixture disk exists").endpoint())
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
let (reopened, errors) = init_disks(
|
||||
&InstanceContext::new(),
|
||||
&endpoints,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(errors.iter().all(Option::is_none), "reopen persisted disks: {errors:?}");
|
||||
reopened
|
||||
}
|
||||
|
||||
fn bootstrap_plan() -> Vec<Option<FormatV3>> {
|
||||
let reference = FormatV3::new(1, 4);
|
||||
(0..4)
|
||||
.map(|index| {
|
||||
let mut format = reference.clone();
|
||||
format.erasure.this = reference.erasure.sets[0][index];
|
||||
Some(format)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn format_snapshot(disks: &[Option<DiskStore>]) -> Vec<Option<Vec<u8>>> {
|
||||
let mut snapshot = Vec::with_capacity(disks.len());
|
||||
for disk in disks {
|
||||
let path = disk
|
||||
.as_ref()
|
||||
.expect("snapshot disk exists")
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(FORMAT_CONFIG_FILE);
|
||||
match tokio::fs::read(path).await {
|
||||
Ok(bytes) => snapshot.push(Some(bytes)),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => snapshot.push(None),
|
||||
Err(error) => panic!("read persisted format: {error}"),
|
||||
}
|
||||
}
|
||||
snapshot
|
||||
}
|
||||
|
||||
async fn assert_plan_published(disks: &[Option<DiskStore>], expected: &[Option<FormatV3>]) {
|
||||
let (actual, errors) = load_format_erasure_all(disks, false).await;
|
||||
assert!(errors.iter().all(Option::is_none), "load recovered formats: {errors:?}");
|
||||
assert_eq!(actual, expected, "bootstrap recovery must preserve the deployment and every disk UUID");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_survives_formatter_restart() {
|
||||
let (_directory, mut disks) = local_disks(4).await;
|
||||
let first = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut disks, 1, 4, None)
|
||||
.await
|
||||
.expect("format fresh deployment");
|
||||
let before = format_snapshot(&disks).await;
|
||||
|
||||
let mut reopened = reopen_disks(&disks).await;
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None)
|
||||
.await
|
||||
.expect("resume bootstrap after process restart");
|
||||
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::Fresh);
|
||||
assert_eq!(loaded.format.shared_identity(), first.format.shared_identity());
|
||||
assert_eq!(
|
||||
format_snapshot(&reopened).await,
|
||||
before,
|
||||
"already published formats remain byte-identical"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_completes_interrupted_fourth_format_after_restart() {
|
||||
let (_directory, disks) = local_disks(4).await;
|
||||
let plan = bootstrap_plan();
|
||||
write_format_bootstrap_intents(&disks, &plan)
|
||||
.await
|
||||
.expect("persist bootstrap authority before formats");
|
||||
write_format_file_all(&disks[..3], &plan[..3])
|
||||
.await
|
||||
.expect("publish first three formats");
|
||||
let interrupted_disk = disks[3].as_ref().expect("fourth disk exists");
|
||||
let staged = Uuid::new_v4().to_string();
|
||||
interrupted_disk
|
||||
.write_all(
|
||||
RUSTFS_META_BUCKET,
|
||||
&staged,
|
||||
plan[3]
|
||||
.as_ref()
|
||||
.expect("fourth planned format")
|
||||
.to_json()
|
||||
.expect("encode planned format")
|
||||
.into_bytes()
|
||||
.into(),
|
||||
)
|
||||
.await
|
||||
.expect("stage the fourth format without its rename");
|
||||
let before = format_snapshot(&disks).await;
|
||||
assert!(before[3].is_none(), "reproducer has no fourth format.json");
|
||||
|
||||
let mut reopened = reopen_disks(&disks).await;
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None)
|
||||
.await
|
||||
.expect("finish interrupted format publication before identity reads");
|
||||
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::Fresh);
|
||||
assert_plan_published(&reopened, &plan).await;
|
||||
assert_eq!(
|
||||
format_snapshot(&reopened).await[..3],
|
||||
before[..3],
|
||||
"recovery only publishes the unformatted slot"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_reuses_plan_when_no_formats_were_published() {
|
||||
let (_directory, disks) = local_disks(4).await;
|
||||
let plan = bootstrap_plan();
|
||||
write_format_bootstrap_intents(&disks, &plan)
|
||||
.await
|
||||
.expect("persist the complete format plan");
|
||||
assert!(format_snapshot(&disks).await.iter().all(Option::is_none));
|
||||
|
||||
let mut reopened = reopen_disks(&disks).await;
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None)
|
||||
.await
|
||||
.expect("resume before first format publication");
|
||||
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::Fresh);
|
||||
assert_plan_published(&reopened, &plan).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_anchor_is_durable_before_fanout() {
|
||||
use crate::disk::os::prepared_publication_test_hooks::{Stage, install_at};
|
||||
|
||||
let (_directory, disks) = local_disks(4).await;
|
||||
let plan = bootstrap_plan();
|
||||
let anchor = disks[0]
|
||||
.as_ref()
|
||||
.expect("anchor disk")
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(FORMAT_BOOTSTRAP_FILE);
|
||||
let (arrived, arrival) = tokio::sync::oneshot::channel();
|
||||
let (release, released) = std::sync::mpsc::channel();
|
||||
let _hook = install_at(Stage::PreparedRename, &anchor, move || {
|
||||
let _ = arrived.send(());
|
||||
let _ = released.recv();
|
||||
});
|
||||
let publication = tokio::spawn({
|
||||
let disks = disks.clone();
|
||||
let plan = plan.clone();
|
||||
async move { write_format_bootstrap_intents(&disks, &plan).await }
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), arrival)
|
||||
.await
|
||||
.expect("anchor reaches native publication")
|
||||
.expect("anchor publication hook");
|
||||
for disk in disks.iter().skip(1).flatten() {
|
||||
assert!(matches!(
|
||||
disk.read_all(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE).await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
release.send(()).expect("release anchor publication");
|
||||
publication
|
||||
.await
|
||||
.expect("join anchor-first publication")
|
||||
.expect("publish the selected plan");
|
||||
assert!(format_snapshot(&disks).await.iter().all(Option::is_none));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_anchor_only_restart_preserves_selected_plan() {
|
||||
let (_directory, disks) = local_disks(4).await;
|
||||
let plan = bootstrap_plan();
|
||||
disks[0]
|
||||
.as_ref()
|
||||
.expect("anchor disk")
|
||||
.write_format_control(
|
||||
FORMAT_BOOTSTRAP_FILE,
|
||||
plan[0]
|
||||
.as_ref()
|
||||
.expect("anchor format")
|
||||
.to_json()
|
||||
.expect("encode selected plan")
|
||||
.into_bytes()
|
||||
.into(),
|
||||
)
|
||||
.await
|
||||
.expect("persist only the anchor before interruption");
|
||||
|
||||
let mut reopened = reopen_disks(&disks).await;
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None)
|
||||
.await
|
||||
.expect("complete fanout from the accepted anchor after restart");
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::Fresh);
|
||||
assert_plan_published(&reopened, &plan).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_missing_anchor_cannot_replace_partial_plan() {
|
||||
let (_directory, disks) = local_disks(4).await;
|
||||
let plan = bootstrap_plan();
|
||||
let existing = plan[1]
|
||||
.as_ref()
|
||||
.expect("second slot format")
|
||||
.to_json()
|
||||
.expect("encode partial plan");
|
||||
disks[1]
|
||||
.as_ref()
|
||||
.expect("second disk")
|
||||
.write_all(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE, existing.clone().into_bytes().into())
|
||||
.await
|
||||
.expect("persist a partial plan with no anchor");
|
||||
let mut reopened = reopen_disks(&disks).await;
|
||||
connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None)
|
||||
.await
|
||||
.err()
|
||||
.expect("partial plan cannot authorize selecting another anchor");
|
||||
assert!(format_snapshot(&disks).await.iter().all(Option::is_none));
|
||||
assert!(matches!(
|
||||
disks[0]
|
||||
.as_ref()
|
||||
.expect("anchor disk")
|
||||
.read_all(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE)
|
||||
.await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
assert_eq!(
|
||||
disks[1]
|
||||
.as_ref()
|
||||
.expect("second disk")
|
||||
.read_all(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE)
|
||||
.await
|
||||
.expect("read preserved partial plan")
|
||||
.as_ref(),
|
||||
existing.as_bytes()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_conflicting_fanout_cannot_change_anchor_or_formats() {
|
||||
let (_directory, disks) = local_disks(4).await;
|
||||
let selected = bootstrap_plan();
|
||||
let foreign = bootstrap_plan();
|
||||
for (index, plan) in [(0, &selected), (1, &foreign)] {
|
||||
disks[index]
|
||||
.as_ref()
|
||||
.expect("fixture disk")
|
||||
.write_all(
|
||||
RUSTFS_META_BUCKET,
|
||||
FORMAT_BOOTSTRAP_FILE,
|
||||
plan[index]
|
||||
.as_ref()
|
||||
.expect("planned slot")
|
||||
.to_json()
|
||||
.expect("encode planned slot")
|
||||
.into_bytes()
|
||||
.into(),
|
||||
)
|
||||
.await
|
||||
.expect("persist conflicting observed plans");
|
||||
}
|
||||
let mut reopened = reopen_disks(&disks).await;
|
||||
connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None)
|
||||
.await
|
||||
.err()
|
||||
.expect("a conflicting intent cannot be replaced by the selected anchor");
|
||||
assert!(format_snapshot(&disks).await.iter().all(Option::is_none));
|
||||
let anchor = disks[0]
|
||||
.as_ref()
|
||||
.expect("anchor disk")
|
||||
.read_all(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE)
|
||||
.await
|
||||
.expect("read preserved anchor");
|
||||
assert_eq!(
|
||||
FormatV3::try_from(anchor.as_ref()).expect("decode preserved anchor"),
|
||||
*selected[0].as_ref().expect("selected anchor")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_partial_proof_cannot_fill_a_partially_formatted_pool() {
|
||||
let (_directory, disks) = local_disks(4).await;
|
||||
let plan = bootstrap_plan();
|
||||
write_format_bootstrap_intents(&disks, &plan)
|
||||
.await
|
||||
.expect("persist the selected plan");
|
||||
write_format_file_all(&disks[..3], &plan[..3])
|
||||
.await
|
||||
.expect("publish only three formats");
|
||||
let missing = disks[3]
|
||||
.as_ref()
|
||||
.expect("fourth disk")
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(FORMAT_BOOTSTRAP_FILE);
|
||||
tokio::fs::remove_file(&missing).await.expect("remove one proof replica");
|
||||
let before = format_snapshot(&disks).await;
|
||||
let mut reopened = reopen_disks(&disks).await;
|
||||
if let Ok(loaded) =
|
||||
connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None).await
|
||||
{
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
}
|
||||
assert_eq!(format_snapshot(&disks).await, before);
|
||||
assert!(!missing.exists(), "partial format state cannot recreate missing proof");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_absence_does_not_reinitialize_existing_formats() {
|
||||
let (_directory, mut disks) = local_disks(4).await;
|
||||
let plan = bootstrap_plan();
|
||||
write_format_file_all(&disks, &plan)
|
||||
.await
|
||||
.expect("publish historical formats without an intent");
|
||||
let before = format_snapshot(&disks).await;
|
||||
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut disks, 1, 4, None)
|
||||
.await
|
||||
.expect("load existing deployment without granting bootstrap authority");
|
||||
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert_eq!(format_snapshot(&disks).await, before);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_incomplete_or_invalid_cannot_grant_authority() {
|
||||
for invalid_state in ["missing", "corrupt", "foreign", "wrong-slot"] {
|
||||
let (_directory, mut disks) = local_disks(4).await;
|
||||
let plan = bootstrap_plan();
|
||||
write_format_bootstrap_intents(&disks, &plan)
|
||||
.await
|
||||
.expect("persist complete authority");
|
||||
write_format_file_all(&disks, &plan).await.expect("publish expected formats");
|
||||
let intent_path = disks[3]
|
||||
.as_ref()
|
||||
.expect("fourth disk exists")
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(FORMAT_BOOTSTRAP_FILE);
|
||||
match invalid_state {
|
||||
"missing" => tokio::fs::remove_file(&intent_path).await.expect("remove one intent replica"),
|
||||
"corrupt" => tokio::fs::write(&intent_path, b"invalid JSON")
|
||||
.await
|
||||
.expect("corrupt one intent replica"),
|
||||
"foreign" => {
|
||||
let foreign = bootstrap_plan();
|
||||
tokio::fs::write(
|
||||
&intent_path,
|
||||
foreign[3]
|
||||
.as_ref()
|
||||
.expect("foreign fourth format")
|
||||
.to_json()
|
||||
.expect("encode foreign intent"),
|
||||
)
|
||||
.await
|
||||
.expect("replace one intent with another deployment");
|
||||
}
|
||||
"wrong-slot" => {
|
||||
tokio::fs::write(
|
||||
&intent_path,
|
||||
plan[2]
|
||||
.as_ref()
|
||||
.expect("third planned format")
|
||||
.to_json()
|
||||
.expect("encode wrong-slot intent"),
|
||||
)
|
||||
.await
|
||||
.expect("copy another slot's intent");
|
||||
}
|
||||
_ => unreachable!("all fixture states are enumerated"),
|
||||
}
|
||||
let before = format_snapshot(&disks).await;
|
||||
|
||||
let result =
|
||||
connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut disks, 1, 4, None).await;
|
||||
|
||||
if let Ok(loaded) = result {
|
||||
assert_eq!(
|
||||
loaded.pool_meta_bootstrap_authority,
|
||||
PoolMetaBootstrapAuthority::None,
|
||||
"{invalid_state} intent cannot authorize pool metadata initialization"
|
||||
);
|
||||
}
|
||||
assert_eq!(format_snapshot(&disks).await, before, "{invalid_state} intent cannot alter valid formats");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_retirement_prevents_future_rebootstrap() {
|
||||
let (_directory, mut disks) = local_disks(4).await;
|
||||
let first = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut disks, 1, 4, None)
|
||||
.await
|
||||
.expect("format fresh deployment");
|
||||
assert_eq!(first.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::Fresh);
|
||||
retire_format_bootstrap_authority(&disks)
|
||||
.await
|
||||
.expect("retire authority after committed pool metadata");
|
||||
retire_format_bootstrap_authority(&disks)
|
||||
.await
|
||||
.expect("authority retirement is idempotent");
|
||||
let before = format_snapshot(&disks).await;
|
||||
|
||||
let mut reopened = reopen_disks(&disks).await;
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None)
|
||||
.await
|
||||
.expect("load a deployment after authority retirement");
|
||||
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert_eq!(format_snapshot(&reopened).await, before);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_observer_cannot_resume_formatter_authority() {
|
||||
let (_directory, mut disks) = local_disks(4).await;
|
||||
let plan = bootstrap_plan();
|
||||
write_format_bootstrap_intents(&disks, &plan)
|
||||
.await
|
||||
.expect("persist complete format intent");
|
||||
write_format_file_all(&disks, &plan).await.expect("publish expected formats");
|
||||
let before = format_snapshot(&disks).await;
|
||||
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), false, &mut disks, 1, 4, None)
|
||||
.await
|
||||
.expect("observer may read the existing format");
|
||||
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
assert_eq!(format_snapshot(&disks).await, before, "an observer cannot republish storage formats");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_unavailable_disk_cannot_restore_authority() {
|
||||
let (_directory, mut disks) = local_disks(4).await;
|
||||
let plan = bootstrap_plan();
|
||||
write_format_bootstrap_intents(&disks, &plan)
|
||||
.await
|
||||
.expect("persist complete format intent");
|
||||
write_format_file_all(&disks, &plan).await.expect("publish expected formats");
|
||||
let physical_disks = disks.clone();
|
||||
let before = format_snapshot(&physical_disks).await;
|
||||
disks[3] = None;
|
||||
|
||||
let result =
|
||||
connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut disks, 1, 4, None).await;
|
||||
|
||||
if let Ok(loaded) = result {
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
}
|
||||
assert_eq!(
|
||||
format_snapshot(&physical_disks).await,
|
||||
before,
|
||||
"a missing configured disk cannot authorize format writes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_deployment_mismatch_cannot_publish_missing_format() {
|
||||
let (_directory, mut disks) = local_disks(4).await;
|
||||
let plan = bootstrap_plan();
|
||||
write_format_bootstrap_intents(&disks, &plan)
|
||||
.await
|
||||
.expect("persist complete format intent");
|
||||
write_format_file_all(&disks[..3], &plan[..3])
|
||||
.await
|
||||
.expect("publish three formats from the original deployment");
|
||||
let physical_disks = disks.clone();
|
||||
let before = format_snapshot(&physical_disks).await;
|
||||
|
||||
let result = connect_load_init_formats_with_instance_ctx(
|
||||
&Arc::new(InstanceContext::new()),
|
||||
true,
|
||||
&mut disks,
|
||||
1,
|
||||
4,
|
||||
Some(Uuid::new_v4()),
|
||||
)
|
||||
.await;
|
||||
|
||||
if let Ok(loaded) = result {
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
}
|
||||
assert_eq!(
|
||||
format_snapshot(&physical_disks).await,
|
||||
before,
|
||||
"an intent belonging to another deployment cannot publish the fourth format"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_conflicting_existing_format_cannot_be_overwritten() {
|
||||
let (_directory, mut disks) = local_disks(4).await;
|
||||
let plan = bootstrap_plan();
|
||||
write_format_bootstrap_intents(&disks, &plan)
|
||||
.await
|
||||
.expect("persist complete format intent");
|
||||
write_format_file_all(&disks[..3], &plan[..3])
|
||||
.await
|
||||
.expect("publish first three formats");
|
||||
let foreign = bootstrap_plan();
|
||||
disks[3]
|
||||
.as_ref()
|
||||
.expect("fourth disk")
|
||||
.write_all(
|
||||
RUSTFS_META_BUCKET,
|
||||
FORMAT_CONFIG_FILE,
|
||||
foreign[3]
|
||||
.as_ref()
|
||||
.expect("foreign fourth format")
|
||||
.to_json()
|
||||
.expect("encode foreign physical format")
|
||||
.into_bytes()
|
||||
.into(),
|
||||
)
|
||||
.await
|
||||
.expect("inject another deployment's physical format on the fourth disk");
|
||||
let physical_disks = disks.clone();
|
||||
let before = format_snapshot(&physical_disks).await;
|
||||
|
||||
let result =
|
||||
connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut disks, 1, 4, None).await;
|
||||
|
||||
if let Ok(loaded) = result {
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
}
|
||||
assert_eq!(
|
||||
format_snapshot(&physical_disks).await,
|
||||
before,
|
||||
"a complete intent cannot overwrite a conflicting persisted format"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_retirement_consumes_only_one_slot() {
|
||||
let (_directory, mut disks) = local_disks(4).await;
|
||||
let plan = bootstrap_plan();
|
||||
write_format_bootstrap_intents(&disks, &plan)
|
||||
.await
|
||||
.expect("persist complete format intent");
|
||||
write_format_file_all(&disks, &plan)
|
||||
.await
|
||||
.expect("publish every planned format");
|
||||
retire_format_bootstrap_authority(&disks)
|
||||
.await
|
||||
.expect("retire creator authority");
|
||||
retire_format_bootstrap_authority(&disks)
|
||||
.await
|
||||
.expect("repeat retirement after interrupted identity publication");
|
||||
|
||||
let mut absent = 0;
|
||||
for disk in disks.iter().flatten() {
|
||||
match disk.read_file(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE).await {
|
||||
Err(DiskError::FileNotFound) => absent += 1,
|
||||
Ok(_) => {}
|
||||
Err(error) => panic!("read intent after retirement: {error}"),
|
||||
}
|
||||
}
|
||||
assert_eq!(absent, 1, "one consumed slot is sufficient and avoids writes to sibling disks");
|
||||
let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut disks, 1, 4, None)
|
||||
.await
|
||||
.expect("load formats after creator authority consumption");
|
||||
assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_retirement_preserves_completion_symlink_target() {
|
||||
let (directory, disks) = local_disks(4).await;
|
||||
let plan = bootstrap_plan();
|
||||
write_format_bootstrap_intents(&disks, &plan)
|
||||
.await
|
||||
.expect("persist creator intent");
|
||||
let sentinel = directory.path().join("unrelated");
|
||||
tokio::fs::write(&sentinel, b"preserve").await.expect("write sentinel");
|
||||
let done = disks[0]
|
||||
.as_ref()
|
||||
.expect("fixture disk")
|
||||
.path()
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join("format.json.bootstrap.done");
|
||||
std::os::unix::fs::symlink(&sentinel, &done).expect("install completion symlink");
|
||||
retire_format_bootstrap_authority(&disks[..1])
|
||||
.await
|
||||
.expect("replace completion entry without following it");
|
||||
assert_eq!(tokio::fs::read(&sentinel).await.expect("read sentinel"), b"preserve");
|
||||
assert!(tokio::fs::symlink_metadata(done).await.expect("completion marker").is_file());
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "test-util"))]
|
||||
#[tokio::test]
|
||||
async fn bootstrap_intent_retirement_requires_successful_directory_sync() {
|
||||
let (_directory, disks) = local_disks(4).await;
|
||||
let disk = disks[0].as_ref().expect("fixture disk");
|
||||
crate::disk::os::set_local_dir_fsync_failure(disk, RUSTFS_META_BUCKET, "", std::io::ErrorKind::Other)
|
||||
.expect("inject absent-intent directory-sync failure");
|
||||
retire_format_bootstrap_authority(&disks[..1])
|
||||
.await
|
||||
.expect_err("visible absence requires a successful sync");
|
||||
retire_format_bootstrap_authority(&disks[..1])
|
||||
.await
|
||||
.expect("retirement sync retries safely");
|
||||
|
||||
let plan = bootstrap_plan();
|
||||
write_format_bootstrap_intents(&disks, &plan)
|
||||
.await
|
||||
.expect("persist creator intent");
|
||||
crate::disk::os::set_local_dir_fsync_failure(disk, RUSTFS_META_BUCKET, "", std::io::ErrorKind::Other)
|
||||
.expect("inject post-rename directory-sync failure");
|
||||
retire_format_bootstrap_authority(&disks[..1])
|
||||
.await
|
||||
.expect_err("rename visibility cannot replace durability");
|
||||
assert!(matches!(
|
||||
disk.read_file(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE).await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
crate::disk::os::set_local_dir_fsync_failure(disk, RUSTFS_META_BUCKET, "", std::io::ErrorKind::Other)
|
||||
.expect("inject retirement retry sync failure");
|
||||
retire_format_bootstrap_authority(&disks[..1])
|
||||
.await
|
||||
.expect_err("retry must repeat the durability barrier");
|
||||
retire_format_bootstrap_authority(&disks[..1])
|
||||
.await
|
||||
.expect("retry consumes authority durably");
|
||||
}
|
||||
@@ -435,6 +435,7 @@ pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult};
|
||||
pub(crate) use heal::{bucket_heal_scope, bucket_heal_scope_for_object};
|
||||
mod heal_walk;
|
||||
pub use heal_walk::HealWalkVersion;
|
||||
mod format_bootstrap;
|
||||
mod init;
|
||||
pub(crate) mod init_format;
|
||||
pub(crate) mod list_objects;
|
||||
|
||||
@@ -46,6 +46,15 @@ impl BootstrapLocalTarget {
|
||||
Arc::ptr_eq(&self.ctx, &store.ctx)
|
||||
}
|
||||
|
||||
/// Atomically publish a format control file on this bootstrap instance.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns a disk error for an unsupported path, unavailable local endpoint,
|
||||
/// publication failure, or failed durability barrier.
|
||||
pub async fn write_local_format_control(&self, disk_ref: &str, path: &str, data: bytes::Bytes) -> DiskResult<()> {
|
||||
write_local_format_control_with_ctx(&self.ctx, disk_ref, path, data).await
|
||||
}
|
||||
|
||||
pub async fn rename_local_data(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
@@ -85,6 +94,15 @@ fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Atomically publish a format control file on this storage instance.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns a disk error for an unsupported path, unavailable local endpoint,
|
||||
/// publication failure, or failed durability barrier.
|
||||
pub async fn write_local_format_control(&self, disk_ref: &str, path: &str, data: bytes::Bytes) -> DiskResult<()> {
|
||||
write_local_format_control_with_ctx(&self.ctx, disk_ref, path, data).await
|
||||
}
|
||||
|
||||
/// Resolve a quota lease against this request's captured storage instance.
|
||||
pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult<DiskStore> {
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
@@ -307,6 +325,18 @@ impl ECStore {
|
||||
}
|
||||
|
||||
// The optional ID is a cold lookup to cache only after final admission.
|
||||
async fn write_local_format_control_with_ctx(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk_ref: &str,
|
||||
path: &str,
|
||||
data: bytes::Bytes,
|
||||
) -> DiskResult<()> {
|
||||
crate::disk::validate_format_control_path(path)?;
|
||||
let (disk, id) = local_disk_candidate(ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(ctx, &disk, id, false).await?;
|
||||
disk.write_format_control(path, data).await
|
||||
}
|
||||
|
||||
async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<(DiskStore, Option<Uuid>)> {
|
||||
let map = ctx.local_disk_map();
|
||||
let path_disk = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned();
|
||||
|
||||
@@ -127,6 +127,15 @@ pub struct WriteAllRequest {
|
||||
pub data: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct WriteFormatControlRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub disk: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub path: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub data: ::prost::bytes::Bytes,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct WriteAllResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub success: bool,
|
||||
@@ -2221,6 +2230,21 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "WriteAll"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn write_format_control(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::WriteFormatControlRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::WriteAllResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WriteFormatControl");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "WriteFormatControl"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn compare_and_update_file(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::CompareAndUpdateFileRequest>,
|
||||
@@ -3801,6 +3825,10 @@ pub mod node_service_server {
|
||||
&self,
|
||||
request: tonic::Request<super::WriteAllRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::WriteAllResponse>, tonic::Status>;
|
||||
async fn write_format_control(
|
||||
&self,
|
||||
request: tonic::Request<super::WriteFormatControlRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::WriteAllResponse>, tonic::Status>;
|
||||
async fn compare_and_update_file(
|
||||
&self,
|
||||
request: tonic::Request<super::CompareAndUpdateFileRequest>,
|
||||
@@ -4548,6 +4576,34 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/WriteFormatControl" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct WriteFormatControlSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::WriteFormatControlRequest> for WriteFormatControlSvc<T> {
|
||||
type Response = super::WriteAllResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::WriteFormatControlRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::write_format_control(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = WriteFormatControlSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/CompareAndUpdateFile" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct CompareAndUpdateFileSvc<T: NodeService>(pub Arc<T>);
|
||||
|
||||
@@ -1433,6 +1433,20 @@ pub fn canonical_write_all_request_body(
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
/// Build the signed semantic body for atomic format-control publication.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns a length conversion error when a field exceeds the canonical wire limit.
|
||||
pub fn canonical_write_format_control_request_body(
|
||||
request: &proto_gen::node_service::WriteFormatControlRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-write-format-control-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.path)?;
|
||||
body.push_bytes(&request.data)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_compare_and_update_file_request_body(
|
||||
request: &proto_gen::node_service::CompareAndUpdateFileRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
@@ -3781,6 +3795,36 @@ mod tests {
|
||||
fields
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_control_canonical_body_binds_every_field_and_operation() {
|
||||
use proto_gen::node_service::{WriteAllRequest, WriteFormatControlRequest};
|
||||
let request = WriteFormatControlRequest {
|
||||
disk: "http://node-a:9000/data".into(),
|
||||
path: "format.json.bootstrap".into(),
|
||||
data: vec![1, 2, 3].into(),
|
||||
};
|
||||
let canonical = canonical_write_format_control_request_body(&request).expect("canonical format control");
|
||||
for mutate in [
|
||||
|r: &mut WriteFormatControlRequest| r.disk = "http://node-b:9000/data".into(),
|
||||
|r: &mut WriteFormatControlRequest| r.path = "format.json".into(),
|
||||
|r: &mut WriteFormatControlRequest| r.data = vec![1, 2, 4].into(),
|
||||
] {
|
||||
let mut changed = request.clone();
|
||||
mutate(&mut changed);
|
||||
assert_ne!(
|
||||
canonical_write_format_control_request_body(&changed).expect("changed canonical body"),
|
||||
canonical
|
||||
);
|
||||
}
|
||||
let ordinary = WriteAllRequest {
|
||||
disk: request.disk,
|
||||
volume: ".rustfs.sys".into(),
|
||||
path: request.path,
|
||||
data: request.data,
|
||||
};
|
||||
assert_ne!(canonical_write_all_request_body(&ordinary).expect("ordinary canonical body"), canonical);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_compat_send_site_manifest_covers_node_proto_bin_fields() {
|
||||
let mut manifest_fields = REQUEST_COMPAT_SEND_SITES
|
||||
|
||||
@@ -116,6 +116,12 @@ message WriteAllRequest {
|
||||
bytes data = 4;
|
||||
}
|
||||
|
||||
message WriteFormatControlRequest {
|
||||
string disk = 1;
|
||||
string path = 2;
|
||||
bytes data = 3;
|
||||
}
|
||||
|
||||
message WriteAllResponse {
|
||||
bool success = 1;
|
||||
optional Error error = 2;
|
||||
@@ -1308,6 +1314,7 @@ service NodeService {
|
||||
|
||||
rpc ReadAll(ReadAllRequest) returns (ReadAllResponse) {}; // auth-policy: read-only
|
||||
rpc WriteAll(WriteAllRequest) returns (WriteAllResponse) {}; // auth-policy: body-bound
|
||||
rpc WriteFormatControl(WriteFormatControlRequest) returns (WriteAllResponse) {}; // auth-policy: body-bound
|
||||
rpc CompareAndUpdateFile(CompareAndUpdateFileRequest) returns (CompareAndUpdateFileResponse) {}; // auth-policy: body-bound
|
||||
rpc Delete(DeleteRequest) returns (DeleteResponse) {}; // auth-policy: body-bound
|
||||
rpc DeleteAtIncarnation(DeleteRequest) returns (DeleteResponse) {}; // auth-policy: body-bound
|
||||
|
||||
@@ -92,4 +92,18 @@ If a block persists, inspect the first block and subsequent recovery phase, rest
|
||||
3. Restore storage formats and the `pool.bin.identity` marker from the same deployment before rejoining it.
|
||||
4. Start the node and wait for it to load the verified committed generation and repair its replicas before touching another node.
|
||||
|
||||
An initialized identity with every `pool.bin` missing is recovery required, as are existing storage formats with neither identity nor `pool.bin`. Format creation alone is not fresh-cluster proof: only the elected first topology node may create a durable `initialized=false` bootstrap identity with a fresh-bootstrap nonce, and only after every configured disk explicitly responds that it is unformatted. An unreachable peer, a non-elected distributed node, or an existing format is not sufficient proof. All-missing `pool.bin` replicas are accepted only by the same startup that proved the fresh topology and persisted that pending identity; when every `pool.bin` is missing, a later startup must recover even if the pending identity survived. This prevents a wiped or lagging node from rebuilding empty state and overwriting the cluster. Runtime reload, rebalance activation, and rebalance worker admission fail closed on this missing-authority condition; a clean probe alone cannot clear it.
|
||||
An initialized identity with every `pool.bin` missing requires metadata recovery. Existing storage formats with neither identity nor `pool.bin` also require recovery unless the pool retains a complete durable format-bootstrap intent. Runtime reload, rebalance activation, and rebalance worker admission preserve this authority check.
|
||||
|
||||
Fresh formation first requires every configured disk to explicitly respond that it is unformatted and the legacy-format adoption check to complete. The pool's elected creator durably selects its plan in the first-slot `.rustfs.sys/format.json.bootstrap` anchor before publishing intents to the other slots. Each intent contains the deployment UUID, complete erasure topology and assigned disk UUID. Every configured intent must be durably acknowledged before any format is published. After an interruption with all drives still explicitly unformatted, the creator can complete missing replicas of the validated anchor plan using the original UUIDs. Once any format exists, a complete consistent intent set is required before restoring explicitly unformatted slots and recovering creator authority. A missing anchor with partial replicas, corrupt or conflicting evidence, and unreachable configured drives withhold authority. A pending `pool.bin.identity` alone continues to require verified creator authority.
|
||||
|
||||
The elected metadata writer commits `pool.bin`, durably retires one bootstrap intent per pool to `format.json.bootstrap.done`, and then publishes initialized identity on every configured pool. Every node waits for that complete initialized-identity set before proceeding with startup. Since replay requires every intent, one durably consumed slot per pool is sufficient. A crash during retirement leaves committed metadata available for the next startup to finish identity publication. Established startup validates initialized identity through metadata reads, and creators probe the local first-slot intent before opening any replay-read fanout. The `.done` files are internal tombstones and should remain with the deployment.
|
||||
|
||||
An established cluster already has a committed metadata predecessor. During expansion or identity repair, the elected writer first retires bootstrap authority, repairs initialized identity on every physical participant, and then saves the updated topology. This preserves the native metadata writer's identity admission requirement. Followers wait for a committed snapshot matching the configured topology and initialized identity on every pool. The startup phase is derived from the observed metadata and identity state. An already complete identity skips retirement for both writers and followers.
|
||||
|
||||
Bootstrap format and intent publication uses the authenticated `WriteFormatControl` RPC, restricted to the two format control paths. Each operation performs its staged write, file sync, atomic rename and directory sync in one dedicated per-disk fsync-lane closure with a pinned parent. Unsupported peers return an error before control publication. These critical control writes always execute their durability barriers, including deployments using legacy full-off durability, because resumable creator authority requires persistent file and namespace state.
|
||||
|
||||
Format publication and retirement share a namespace owner keyed by physical-root identity, including reopened handles. The native task retains that owner through file and directory sync and cache invalidation after caller cancellation. Admission runs under the same owner: pending plans accept equivalent retries, terminal `.done` records reject bootstrap re-creation, and existing format or intent identities must agree with the incoming deployment, layout and exact drive slot. This prevents an older cancelled or delayed publication from replacing a newer accepted plan. The pinned parent used for evidence reads also supplies the publication directory handle.
|
||||
|
||||
Replacement-drive healing retains the existing staged `WriteAll` plus durable `RenameFile` sequence so configured peers can be upgraded independently. Bootstrap publication uses the new atomic operation directly and fails closed on unsupported peers.
|
||||
|
||||
Use the same protocol-capable version on all nodes for fresh formation. Finish initialization and intent retirement before rolling back to an older binary, which does not enforce the consumption barrier. Existing deployments retain their current format and identity encodings. Historical interrupted deployments created by older versions, including the format-plus-UUID-temporary-file state in issue #9270, require operator-confirmed recovery because their files contain no durable creator intent. Preserve the drives and follow the metadata-recovery procedure above when any existing data or topology history needs to be retained.
|
||||
|
||||
@@ -1585,6 +1585,13 @@ impl Node for NodeService {
|
||||
self.handle_write_all(request).await
|
||||
}
|
||||
|
||||
async fn write_format_control(
|
||||
&self,
|
||||
request: Request<WriteFormatControlRequest>,
|
||||
) -> Result<Response<WriteAllResponse>, Status> {
|
||||
self.handle_write_format_control(request).await
|
||||
}
|
||||
|
||||
async fn compare_and_update_file(
|
||||
&self,
|
||||
request: Request<CompareAndUpdateFileRequest>,
|
||||
@@ -3279,7 +3286,7 @@ mod tests {
|
||||
SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest,
|
||||
TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest,
|
||||
UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest,
|
||||
UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteFormatControlRequest, WriteMetadataRequest, WriteRequest,
|
||||
heal_control_service_client::HealControlServiceClient,
|
||||
heal_control_service_server::{HealControlService as _, HealControlServiceServer},
|
||||
node_service_client::NodeServiceClient,
|
||||
@@ -3297,7 +3304,7 @@ mod tests {
|
||||
use tonic::{Request, Response, Status};
|
||||
use uuid::Uuid;
|
||||
|
||||
const DISK_MUTATION_RPC_METHODS: [&str; 34] = [
|
||||
const DISK_MUTATION_RPC_METHODS: [&str; 35] = [
|
||||
"renamedata",
|
||||
"renamemultipartdata",
|
||||
"recovermultipartsource",
|
||||
@@ -3316,6 +3323,7 @@ mod tests {
|
||||
"writemetadata",
|
||||
"updatemetadata",
|
||||
"writeall",
|
||||
"writeformatcontrol",
|
||||
"compareandupdatefile",
|
||||
"delete",
|
||||
"deletepaths",
|
||||
@@ -5147,6 +5155,15 @@ mod tests {
|
||||
},
|
||||
rustfs_protos::canonical_write_all_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
write_format_control,
|
||||
WriteFormatControlRequest {
|
||||
disk: disk.clone(),
|
||||
path: "format.json.bootstrap".into(),
|
||||
data: vec![0x01, 0x02].into(),
|
||||
},
|
||||
rustfs_protos::canonical_write_format_control_request_body
|
||||
);
|
||||
assert_gated!(
|
||||
compare_and_update_file,
|
||||
CompareAndUpdateFileRequest {
|
||||
@@ -7799,6 +7816,146 @@ mod tests {
|
||||
assert!(matches!(disk.read_all(volume, &path).await, Err(DiskError::FileNotFound)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn format_control_rpc_publishes_unformatted_listener_target_in_one_call() {
|
||||
use crate::storage::storage_api::{init_local_disks_with_instance_ctx, new_instance_ctx};
|
||||
use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use tower_http::trace::TraceLayer;
|
||||
|
||||
let root = tempfile::tempdir().expect("format-control RPC disk");
|
||||
let mut endpoint = Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let instance = new_instance_ctx();
|
||||
let pool = PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 1,
|
||||
endpoints: Endpoints::from(vec![endpoint.clone()]),
|
||||
cmd_line: "format-control-rpc".into(),
|
||||
platform: "test".into(),
|
||||
};
|
||||
init_local_disks_with_instance_ctx(&instance, EndpointServerPools::from(vec![pool]))
|
||||
.await
|
||||
.expect("register unformatted listener target");
|
||||
assert!(!root.path().join(".rustfs.sys/format.json").exists());
|
||||
let slot = super::ServerContextSlot::with_instance_context(instance);
|
||||
let service = super::make_server_for_slot(slot);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("format-control listener");
|
||||
let address = listener.local_addr().expect("listener address");
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let recorded = calls.clone();
|
||||
let trace = TraceLayer::new_for_grpc().on_request(move |request: &http::Request<_>, _: &tracing::Span| {
|
||||
if request.uri().path() == "/node_service.NodeService/WriteFormatControl" {
|
||||
recorded.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
});
|
||||
let (shutdown, stopped) = tokio::sync::oneshot::channel();
|
||||
let server = tokio::spawn(async move {
|
||||
tonic::transport::Server::builder()
|
||||
.layer(trace)
|
||||
.add_service(NodeServiceServer::new(service))
|
||||
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
|
||||
let _ = stopped.await;
|
||||
})
|
||||
.await
|
||||
.expect("serve format-control target");
|
||||
});
|
||||
let mut client = NodeServiceClient::connect(format!("http://{address}"))
|
||||
.await
|
||||
.expect("connect real format-control client");
|
||||
let drive_id = Uuid::new_v4();
|
||||
let payload = Bytes::from(
|
||||
serde_json::to_vec(&serde_json::json!({
|
||||
"version": "1",
|
||||
"format": "xl-single",
|
||||
"id": Uuid::new_v4(),
|
||||
"xl": {
|
||||
"version": "3",
|
||||
"this": drive_id,
|
||||
"sets": [[drive_id]],
|
||||
"distributionAlgo": "SIPMOD+PARITY",
|
||||
},
|
||||
}))
|
||||
.expect("encode a valid single-drive format plan"),
|
||||
);
|
||||
let message = WriteFormatControlRequest {
|
||||
disk: endpoint.to_string(),
|
||||
path: "format.json.bootstrap".into(),
|
||||
data: payload.clone(),
|
||||
};
|
||||
let sign = |message: WriteFormatControlRequest| {
|
||||
let mut request = Request::new(message);
|
||||
let canonical =
|
||||
rustfs_protos::canonical_write_format_control_request_body(request.get_ref()).expect("canonical control request");
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical).expect("body digest");
|
||||
mark_v2_authenticated(&mut request);
|
||||
request
|
||||
};
|
||||
let metrics = rustfs_io_metrics::internode_metrics::global_internode_metrics();
|
||||
let before = metrics.snapshot();
|
||||
let response = super::timeout(Duration::from_secs(10), client.write_format_control(sign(message.clone())))
|
||||
.await
|
||||
.expect("bounded atomic control RPC")
|
||||
.expect("atomic control response")
|
||||
.into_inner();
|
||||
assert!(response.success, "{:?}", response.error);
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1, "one atomic publication uses one unary RPC");
|
||||
let successful = metrics.snapshot();
|
||||
assert_eq!(successful.incoming_requests_total - before.incoming_requests_total, 1);
|
||||
assert_eq!(
|
||||
successful.recv_bytes_total - before.recv_bytes_total,
|
||||
u64::try_from(payload.len()).expect("payload length")
|
||||
);
|
||||
assert_eq!(successful.errors_total, before.errors_total);
|
||||
assert_eq!(
|
||||
tokio::fs::read(root.path().join(".rustfs.sys/format.json.bootstrap"))
|
||||
.await
|
||||
.expect("published control"),
|
||||
payload
|
||||
);
|
||||
assert!(
|
||||
!root.path().join(".rustfs.sys/format.json").exists(),
|
||||
"an endpoint route needs no disk UUID"
|
||||
);
|
||||
|
||||
let mut invalid = message.clone();
|
||||
invalid.path = "../outside".into();
|
||||
let error = client
|
||||
.write_format_control(sign(invalid))
|
||||
.await
|
||||
.expect_err("invalid control path rejected before disk work");
|
||||
assert_eq!(error.code(), tonic::Code::InvalidArgument);
|
||||
let mut missing = message.clone();
|
||||
missing.disk = "unregistered-format-target".into();
|
||||
let response = client
|
||||
.write_format_control(sign(missing))
|
||||
.await
|
||||
.expect("typed missing-target response")
|
||||
.into_inner();
|
||||
assert!(!response.success);
|
||||
assert_eq!(response.error.expect("typed target failure").code, DiskError::DiskNotFound.to_u32());
|
||||
let error = client
|
||||
.write_format_control(Request::new(message))
|
||||
.await
|
||||
.expect_err("unsigned control mutation rejected");
|
||||
assert_eq!(error.code(), tonic::Code::PermissionDenied);
|
||||
let completed = metrics.snapshot();
|
||||
assert_eq!(completed.incoming_requests_total - before.incoming_requests_total, 2);
|
||||
assert_eq!(
|
||||
completed.recv_bytes_total - before.recv_bytes_total,
|
||||
2 * u64::try_from(payload.len()).expect("payload length")
|
||||
);
|
||||
assert_eq!(completed.errors_total - before.errors_total, 1);
|
||||
shutdown.send(()).expect("stop format-control server");
|
||||
super::timeout(Duration::from_secs(10), server)
|
||||
.await
|
||||
.expect("bounded server shutdown")
|
||||
.expect("server task");
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
async fn check_remote_bucket_creation(interrupted: bool) {
|
||||
use crate::storage::storage_api::{
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::storage::storage_api::ecstore_disk::{
|
||||
MultipartSourceOperation, MultipartSourceRequest, POOL_META_NATIVE_CAS_METADATA_SUFFIX,
|
||||
PoolMetaNativeRecoveryRequest as NativeRecoveryRequest, QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET,
|
||||
RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, is_quota_mutation_fence_path, require_source_condition,
|
||||
validate_format_control_path,
|
||||
};
|
||||
use crate::storage::storage_api::rpc_consumer::node_service::{
|
||||
BatchReadVersionReq, BatchReadVersionResp, ConditionalFileUpdate, DeleteOptions, DiskError, DiskInfoOptions,
|
||||
@@ -76,6 +77,14 @@ fn startup_cas_rename_observation(
|
||||
}
|
||||
|
||||
impl LocalMutationTarget {
|
||||
async fn write_local_format_control(&self, disk_ref: &str, path: &str, data: Bytes) -> Result<(), DiskError> {
|
||||
match self {
|
||||
Self::Ready(store) => store.write_local_format_control(disk_ref, path, data).await,
|
||||
Self::Bootstrap(target) => target.write_local_format_control(disk_ref, path, data).await,
|
||||
Self::Unbound => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}
|
||||
|
||||
async fn recover_local_multipart_source(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
@@ -2748,6 +2757,48 @@ impl NodeService {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn handle_write_format_control(
|
||||
&self,
|
||||
request: Request<WriteFormatControlRequest>,
|
||||
) -> Result<Response<WriteAllResponse>, Status> {
|
||||
let _duration = InternodeRpcDurationGuard::handler(INTERNODE_OPERATION_GRPC_WRITE_ALL);
|
||||
let canonical = rustfs_protos::canonical_write_format_control_request_body(request.get_ref())
|
||||
.map_err(|_| Status::invalid_argument("format control request length cannot be represented"))?;
|
||||
verify_tonic_canonical_body_digest(&request, &canonical)
|
||||
.map_err(|_| Status::permission_denied("format control requires a signed canonical body"))?;
|
||||
validate_format_control_path(&request.get_ref().path)
|
||||
.map_err(|_| Status::invalid_argument("unsupported format control filename"))?;
|
||||
let request = request.into_inner();
|
||||
let metrics = runtime_sources::current_internode_metrics();
|
||||
metrics.record_incoming_request_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
);
|
||||
metrics.record_recv_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
request.data.len(),
|
||||
);
|
||||
let target = self.local_mutation_target();
|
||||
let result = target
|
||||
.write_local_format_control(&request.disk, &request.path, request.data)
|
||||
.await;
|
||||
match result {
|
||||
Ok(()) => Ok(Response::new(WriteAllResponse {
|
||||
success: true,
|
||||
error: None,
|
||||
})),
|
||||
Err(error) => {
|
||||
metrics
|
||||
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
Ok(Response::new(WriteAllResponse {
|
||||
success: false,
|
||||
error: Some(error.into()),
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn handle_compare_and_update_file(
|
||||
&self,
|
||||
request: Request<CompareAndUpdateFileRequest>,
|
||||
@@ -3028,6 +3079,82 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn format_control_handler_metrics_count_admitted_payload_once() {
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use rustfs_protos::proto_gen::node_service::WriteFormatControlRequest;
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let enabled = rustfs_io_metrics::metrics_enabled();
|
||||
rustfs_io_metrics::set_metrics_enabled(true);
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("handler metrics runtime");
|
||||
runtime.block_on(async {
|
||||
let server = make_server();
|
||||
let message = WriteFormatControlRequest {
|
||||
disk: "missing-format-target".into(),
|
||||
path: "format.json.bootstrap".into(),
|
||||
data: bytes::Bytes::from_static(b"format-plan"),
|
||||
};
|
||||
let mut request = Request::new(message.clone());
|
||||
let canonical =
|
||||
rustfs_protos::canonical_write_format_control_request_body(request.get_ref()).expect("canonical control");
|
||||
crate::storage::storage_api::set_tonic_canonical_body_digest(&mut request, &canonical).expect("control digest");
|
||||
request
|
||||
.metadata_mut()
|
||||
.insert("x-rustfs-rpc-auth-version", "2".parse().expect("v2 header"));
|
||||
let response = server
|
||||
.handle_write_format_control(request)
|
||||
.await
|
||||
.expect("typed target failure")
|
||||
.into_inner();
|
||||
assert!(!response.success);
|
||||
let error = server
|
||||
.handle_write_format_control(Request::new(message))
|
||||
.await
|
||||
.expect_err("unsigned control rejection");
|
||||
assert_eq!(error.code(), tonic::Code::PermissionDenied);
|
||||
});
|
||||
});
|
||||
rustfs_io_metrics::set_metrics_enabled(enabled);
|
||||
let entries = snapshotter.snapshot().into_vec();
|
||||
let value = |name: &str| {
|
||||
&entries
|
||||
.iter()
|
||||
.find(|(key, _, _, _)| {
|
||||
key.key().name() == name
|
||||
&& key
|
||||
.key()
|
||||
.labels()
|
||||
.any(|label| label.key() == "operation" && label.value() == "grpc_write_all")
|
||||
})
|
||||
.expect("format-control handler metric")
|
||||
.3
|
||||
};
|
||||
for (name, expected) in [
|
||||
("rustfs_system_network_internode_operation_requests_incoming_total", 1),
|
||||
("rustfs_system_network_internode_operation_recv_bytes_total", 11),
|
||||
("rustfs_system_network_internode_operation_errors_total", 1),
|
||||
] {
|
||||
assert!(matches!(value(name), DebugValue::Counter(count) if *count == expected), "{name}");
|
||||
}
|
||||
assert!(
|
||||
matches!(value("rustfs_system_network_internode_operation_server_duration_ms"), DebugValue::Histogram(samples) if samples.len() == 2)
|
||||
);
|
||||
assert!(
|
||||
matches!(value("rustfs_system_network_internode_rpc_active"), DebugValue::Gauge(active) if active.into_inner() == 0.0)
|
||||
);
|
||||
assert!(
|
||||
entries
|
||||
.iter()
|
||||
.all(|(key, _, _, _)| key.key().name() != "rustfs_system_network_internode_operation_duration_ms")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn handlers_record_duration_on_missing_disk_errors() {
|
||||
|
||||
@@ -554,7 +554,7 @@ pub(crate) mod ecstore_disk {
|
||||
RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteBackgroundIo,
|
||||
RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout,
|
||||
is_quota_mutation_fence_path, require_source_condition, validate_batch_read_version_item_count,
|
||||
with_remote_background_io,
|
||||
validate_format_control_path, with_remote_background_io,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
|
||||
Reference in New Issue
Block a user