gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-29 16:37:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(scanner): fence movement generation publication (#6461)

Browse Source 
* feat(scanner): add movement generation fencing

* fix(scanner): prioritize unverified cycle deferral

* feat(ecstore): add scanner publication lease fence

* feat(rpc): add scanner publication lease protocol

* feat(scanner): hold remote leases through usage publish

* test(scanner): cover publication lease fencing

* fix(scanner): fence remote leases across restart and delay

* feat(rpc): fence scanner publication rename writes

* fix(scanner): fence observed cleanup deletes

* fix(proto): qualify lease release test types

* fix(scanner): pin movement notifications

* fix(scanner): clean publication imports

* fix(ecstore): satisfy scanner fence clippy

* refactor(scanner): group wait and publication options

* fix(scanner): satisfy final lint and facade guards

* fix(rpc): resolve facade export conflicts

* fix(ci): remove unused decommission and healing facades

* fix(ci): cfg-gate test-only usage overlay import

* fix(scanner): wake on remote scanner restart
This commit is contained in:
cxymds
2026-08-24 14:17:35 +08:00
committed by GitHub
parent 1c5c28842a
commit eec0e0e056
30 changed files with 2926 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/core/io_primitives.rs
+129 -11
View File
@@ -3499,6 +3499,26 @@ pub(in crate::set_disk) struct RenameDataCommit {
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<()>>,
}
/// Options shared by the normal and early-ack rename fanouts. Keeping the
/// quorum and optional scanner lease map together avoids widening either
/// fanout helper's argument list while preserving the fence semantics.
pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
write_quorum: usize,
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
}
impl<'a> RenameDataFenceOptions<'a> {
pub(in crate::set_disk) fn new(
write_quorum: usize,
scanner_publication_lease_tokens: Option<&'a HashMap<String, Uuid>>,
) -> Self {
Self {
write_quorum,
scanner_publication_lease_tokens,
}
}
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
type RenameDataLegacyTuple = (
Vec<Option<DiskStore>>,
@@ -3690,16 +3710,51 @@ impl SetDisks {
.map(RenameDataCommit::into_legacy_tuple)
}
#[tracing::instrument(level = "debug", skip(disks, file_infos))]
async fn rename_data_owned_early_ack(
pub(in crate::set_disk) fn scanner_publication_lease_token_for_disk(
disk: Option<&DiskStore>,
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
) -> disk::error::Result<Option<Uuid>> {
let Some(disk) = disk else {
return Ok(None);
};
if disk.is_local() {
return Ok(None);
}
let Some(tokens) = scanner_publication_lease_tokens else {
return Ok(None);
};
let host = disk.endpoint().grid_host();
tokens
.get(&host)
.copied()
.ok_or_else(|| DiskError::other(format!("scanner publication lease token missing for remote disk host {host}")))
.map(Some)
}
pub(in crate::set_disk) fn scanner_publication_lease_tokens_for_disks(
disks: &[Option<DiskStore>],
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
) -> disk::error::Result<Vec<Option<Uuid>>> {
disks
.iter()
.map(|disk| Self::scanner_publication_lease_token_for_disk(disk.as_ref(), scanner_publication_lease_tokens))
.collect()
}
#[tracing::instrument(level = "debug", skip(disks, file_infos, fence_options))]
async fn rename_data_owned_early_ack_with_fence(
disks: &[Option<DiskStore>],
src_bucket: &str,
src_object: &str,
file_infos: Vec<FileInfo>,
dst_bucket: &str,
dst_object: &str,
write_quorum: usize,
fence_options: RenameDataFenceOptions<'_>,
) -> disk::error::Result<RenameDataCommit> {
let RenameDataFenceOptions {
write_quorum,
scanner_publication_lease_tokens,
} = fence_options;
if let Some(file_info) = disks
.iter()
.zip(file_infos.iter())
@@ -3714,6 +3769,7 @@ impl SetDisks {
let disk_count = disks.len();
let fanout_disks = disks.to_vec();
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
let coordinator_disks = fanout_disks.clone();
let src_bucket = Arc::new(src_bucket.to_string());
let src_object = Arc::new(src_object.to_string());
@@ -3730,7 +3786,12 @@ impl SetDisks {
let successful_rename_completion_rank =
rustfs_io_metrics::put_stage_metrics_enabled().then(|| Arc::new(AtomicUsize::new(0)));
let mut tasks = JoinSet::new();
for (i, (disk, file_info)) in fanout_disks.into_iter().zip(file_infos.iter()).enumerate() {
for (i, ((disk, file_info), scanner_publication_lease_token)) in fanout_disks
.into_iter()
.zip(file_infos.iter())
.zip(fanout_fence_tokens.into_iter())
.enumerate()
{
let src_bucket = fanout_src_bucket.clone();
let src_object = fanout_src_object.clone();
let dst_bucket = fanout_dst_bucket.clone();
@@ -3767,7 +3828,14 @@ impl SetDisks {
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
let result = disk
.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
.rename_data_borrowed_with_fence(
&src_bucket,
&src_object,
file_info,
&dst_bucket,
&dst_object,
scanner_publication_lease_token,
)
.await;
if let Some(disk_wait_started) = disk_wait_started {
let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0;
@@ -3967,19 +4035,45 @@ impl SetDisks {
dst_bucket: &str,
dst_object: &str,
write_quorum: usize,
) -> disk::error::Result<RenameDataCommit> {
Self::rename_data_owned_with_fence(
disks,
src_bucket,
src_object,
file_infos,
dst_bucket,
dst_object,
RenameDataFenceOptions::new(write_quorum, None),
)
.await
}
#[tracing::instrument(level = "debug", skip(disks, file_infos, fence_options))]
pub(in crate::set_disk) async fn rename_data_owned_with_fence(
disks: &[Option<DiskStore>],
src_bucket: &str,
src_object: &str,
file_infos: Vec<FileInfo>,
dst_bucket: &str,
dst_object: &str,
fence_options: RenameDataFenceOptions<'_>,
) -> disk::error::Result<RenameDataCommit> {
if put_rename_early_ack_enabled() {
return Self::rename_data_owned_early_ack(
return Self::rename_data_owned_early_ack_with_fence(
disks,
src_bucket,
src_object,
file_infos,
dst_bucket,
dst_object,
write_quorum,
fence_options,
)
.await;
}
let RenameDataFenceOptions {
write_quorum,
scanner_publication_lease_tokens,
} = fence_options;
if let Some(file_info) = disks
.iter()
.zip(file_infos.iter())
@@ -4005,6 +4099,7 @@ impl SetDisks {
let disk_count = disks.len();
let fanout_disks = disks.to_vec();
let fanout_file_infos = file_infos;
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
let fanout_src_bucket = src_bucket.clone();
let fanout_src_object = src_object.clone();
let fanout_dst_bucket = dst_bucket.clone();
@@ -4019,8 +4114,9 @@ impl SetDisks {
let futures = fanout_disks
.into_iter()
.zip(fanout_file_infos.iter())
.zip(fanout_fence_tokens.into_iter())
.enumerate()
.map(|(i, (disk, file_info))| {
.map(|(i, ((disk, file_info), scanner_publication_lease_token))| {
let src_bucket = fanout_src_bucket.clone();
let src_object = fanout_src_object.clone();
let dst_object = fanout_dst_object.clone();
@@ -4060,7 +4156,14 @@ impl SetDisks {
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
let result = disk
.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
.rename_data_borrowed_with_fence(
&src_bucket,
&src_object,
file_info,
&dst_bucket,
&dst_object,
scanner_publication_lease_token,
)
.await;
if let Some(disk_wait_started) = disk_wait_started {
let duration_ms = disk_wait_started.elapsed().as_secs_f64() * 1000.0;
@@ -5426,18 +5529,32 @@ impl SetDisks {
Ok(())
}
#[cfg(test)]
pub(in crate::set_disk) async fn delete_prefix(&self, bucket: &str, prefix: &str) -> disk::error::Result<()> {
self.delete_prefix_with_scanner_publication_lease(bucket, prefix, None).await
}
/// Delete a prefix with an optional per-remote-disk scanner publication
/// lease fence. The fence is transient and never reaches disk metadata;
/// remote deletes without a matching token fail closed before the RPC.
pub(in crate::set_disk) async fn delete_prefix_with_scanner_publication_lease(
&self,
bucket: &str,
prefix: &str,
scanner_publication_lease_tokens: Option<&HashMap<String, Uuid>>,
) -> disk::error::Result<()> {
let disks = self.get_disks_internal().await;
let write_quorum = disks.len() / 2 + 1;
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(&disks, scanner_publication_lease_tokens)?;
let mut futures = Vec::with_capacity(disks.len());
for disk_op in disks.iter() {
for (disk_op, scanner_publication_lease_token) in disks.iter().zip(fanout_fence_tokens) {
let bucket = bucket.to_string();
let prefix = prefix.to_string();
futures.push(async move {
if let Some(disk) = disk_op {
disk.delete(
disk.delete_with_scanner_publication_lease(
&bucket,
&prefix,
DeleteOptions {
@@ -5445,6 +5562,7 @@ impl SetDisks {
immediate: true,
..Default::default()
},
scanner_publication_lease_token,
)
.await
} else {
crates/ecstore/src/set_disk/ops/object.rs
+67 -4
View File
@@ -49,8 +49,8 @@ use crate::data_usage::quota_object_size;
use crate::diagnostics::get::GetObjectFailureReason;
use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
use crate::error::is_err_invalid_upload_id;
use crate::object_api::NamespaceLockFence;
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
use crate::object_api::{NamespaceLockFence, SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY};
use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
use crate::services::tier::tier::{TierConfigMgr, TierOperationLease};
use crate::store::ECStore;
@@ -63,6 +63,60 @@ use std::sync::OnceLock;
use tokio_util::sync::CancellationToken;
const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json";
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES: usize = 64 * 1024;
const SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES: usize = 256;
fn take_scanner_publication_lease_tokens(user_defined: &mut HashMap<String, String>) -> Result<Option<HashMap<String, Uuid>>> {
let Some(encoded) = user_defined.remove(SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY) else {
return Ok(None);
};
if encoded.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_BYTES {
return Err(Error::other("scanner publication lease fence is too large"));
}
let encoded_tokens = serde_json::from_str::<HashMap<String, String>>(&encoded)
.map_err(|err| Error::other(format!("invalid scanner publication lease fence: {err}")))?;
if encoded_tokens.len() > SCANNER_PUBLICATION_LEASE_FENCE_MAX_ENTRIES {
return Err(Error::other("scanner publication lease fence has too many entries"));
}
let mut tokens = HashMap::with_capacity(encoded_tokens.len());
for (host, token) in encoded_tokens {
if host.is_empty() || host.len() > 1024 {
return Err(Error::other("invalid scanner publication lease fence host"));
}
let token = Uuid::parse_str(&token)
.map_err(|err| Error::other(format!("invalid scanner publication lease fence token: {err}")))?;
if token.is_nil() {
return Err(Error::other("invalid scanner publication lease fence token"));
}
tokens.insert(host, token);
}
Ok(Some(tokens))
}
#[cfg(test)]
mod scanner_publication_lease_fence_tests {
use super::*;
#[test]
fn scanner_publication_lease_fence_is_transient_and_validated() {
let token = Uuid::new_v4();
let mut metadata = HashMap::from([(
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
serde_json::json!({"http://node-a:9000": token.to_string()}).to_string(),
)]);
let parsed = take_scanner_publication_lease_tokens(&mut metadata)
.expect("valid scanner publication lease fence should parse")
.expect("scanner publication lease fence should be present");
assert_eq!(parsed.get("http://node-a:9000"), Some(&token));
assert!(!metadata.contains_key(SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY));
let mut malformed = HashMap::from([(
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY.to_string(),
r#"{"http://node-a:9000":"not-a-uuid"}"#.to_string(),
)]);
assert!(take_scanner_publication_lease_tokens(&mut malformed).is_err());
}
}
struct PutObjectCommitCancellation {
token: CancellationToken,
@@ -2177,6 +2231,7 @@ impl SetDisks {
user_defined.insert(key.clone(), value.clone());
}
}
let scanner_publication_lease_tokens = take_scanner_publication_lease_tokens(&mut user_defined)?;
if replication_lww_applicable(opts) {
// Object Lock evaluation stamps category timestamps with this
// receiver's clock. Pin them back to the source-authored times
@@ -2896,6 +2951,7 @@ impl SetDisks {
let commit_bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone();
let commit_capacity_scope_token = opts.capacity_scope_token;
let commit_replication_state = replication_state_to_filemeta(&opts.put_replication_state());
let commit_scanner_publication_lease_tokens = scanner_publication_lease_tokens;
tmp_cleanup_owned = true;
let commit = move |cancellation: Option<CancellationToken>| async move {
@@ -3015,14 +3071,17 @@ impl SetDisks {
}
Self::assign_rename_data_indexes(&mut parts_metadatas);
let rename_result = SetDisks::rename_data_owned(
let rename_result = SetDisks::rename_data_owned_with_fence(
&commit_disks,
RUSTFS_META_TMP_BUCKET,
commit_tmp_dir.as_str(),
parts_metadatas,
&commit_bucket,
&commit_object,
write_quorum,
crate::set_disk::core::io_primitives::RenameDataFenceOptions::new(
write_quorum,
commit_scanner_publication_lease_tokens.as_ref(),
),
)
.await;
if quota_mutation_fence {
@@ -6418,6 +6477,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
#[tracing::instrument(skip(self))]
async fn delete_object(&self, bucket: &str, object: &str, mut opts: ObjectOptions) -> Result<ObjectInfo> {
// Scanner cleanup carries the per-peer lease fence as transient
// request metadata. Consume it before any delete-prefix fanout so it
// cannot be persisted or treated as user metadata.
let scanner_publication_lease_tokens = take_scanner_publication_lease_tokens(&mut opts.user_defined)?;
let preserve_delete_replication_state = should_preserve_delete_replication_state(&opts);
let delete_config_snapshot = if opts.delete_prefix || opts.transition.expire_restored || preserve_delete_replication_state
{
@@ -6544,7 +6607,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
}
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
self.delete_prefix(bucket, object)
self.delete_prefix_with_scanner_publication_lease(bucket, object, scanner_publication_lease_tokens.as_ref())
.await
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;