fix(rpc): bind local mutations to the listener instance

This commit is contained in:
overtrue
2026-09-06 10:58:11 +08:00
parent 7f631ec378
commit 11c9fd64ce
15 changed files with 1408 additions and 157 deletions
+4 -2
View File
@@ -366,6 +366,8 @@ pub mod data_usage {
pub mod disk {
pub use crate::disk::disk_store::get_object_disk_read_timeout;
pub use crate::disk::local::ScanGuard;
#[cfg(all(feature = "test-util", not(windows)))]
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
pub use crate::disk::{
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
CheckPartsResp, ConditionalFileUpdate, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
@@ -542,8 +544,8 @@ pub mod storage {
pub use crate::core::pools::HealLifecycleExpiryContext;
pub use crate::store::HealWalkVersion;
pub use crate::store::{
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path,
find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
};
}
+52 -5
View File
@@ -243,7 +243,7 @@ pub(crate) mod fsync_dir_recorder {
}
/// Pause a real namespace mutation inside its physical executor.
#[cfg(all(test, not(windows)))]
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
pub(crate) mod prepared_publication_test_hooks {
use super::*;
@@ -252,6 +252,7 @@ pub(crate) mod prepared_publication_test_hooks {
PreparedRename,
Rename,
Remove,
#[cfg(test)]
Rollback,
DirFsync,
}
@@ -268,6 +269,7 @@ pub(crate) mod prepared_publication_test_hooks {
}
}
#[cfg(test)]
pub(crate) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
install_at(Stage::PreparedRename, path, hook)
}
@@ -286,6 +288,51 @@ pub(crate) mod prepared_publication_test_hooks {
}
}
/// Controlled application-test pause at an existing physical executor boundary.
#[cfg(all(feature = "test-util", not(windows)))]
pub struct LocalPublicationPause {
_hook: prepared_publication_test_hooks::Guard,
entered: oneshot::Receiver<()>,
_release: std::sync::mpsc::Sender<()>,
}
#[cfg(all(feature = "test-util", not(windows)))]
#[derive(Clone, Copy)]
pub enum LocalPublicationStage {
PreparedRename,
Rename,
Remove,
}
#[cfg(all(feature = "test-util", not(windows)))]
impl LocalPublicationPause {
pub fn install(disk: &crate::disk::Disk, volume: &str, path: &str, stage: LocalPublicationStage) -> Result<Self> {
let path = disk
.get_object_path_for_io_if_local(volume, path)
.ok_or(DiskError::DiskNotFound)??;
let stage = match stage {
LocalPublicationStage::PreparedRename => prepared_publication_test_hooks::Stage::PreparedRename,
LocalPublicationStage::Rename => prepared_publication_test_hooks::Stage::Rename,
LocalPublicationStage::Remove => prepared_publication_test_hooks::Stage::Remove,
};
let (entered_tx, entered) = oneshot::channel();
let (release, release_rx) = std::sync::mpsc::channel::<()>();
let hook = prepared_publication_test_hooks::install_at(stage, &path, move || {
let _ = entered_tx.send(());
let _ = release_rx.recv();
});
Ok(Self {
_hook: hook,
entered,
_release: release,
})
}
pub async fn entered(&mut self) -> std::result::Result<(), oneshot::error::RecvError> {
(&mut self.entered).await
}
}
#[cfg(all(test, windows))]
pub(crate) mod windows_rename_test_hooks {
use super::*;
@@ -776,7 +823,7 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Ve
fsync_spawn_blocking(move || {
// The batch worker may be cancelled while this syscall is still running.
let _namespace_owners = namespace_owners;
#[cfg(all(test, not(windows)))]
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::DirFsync, &dir);
#[cfg(test)]
{
@@ -1912,7 +1959,7 @@ pub(crate) async fn remove_file_with_owner(
let path = path.as_ref().to_path_buf();
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
run_blocking_namespace_operation(lease, move || {
#[cfg(all(test, not(windows)))]
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path);
std::fs::remove_file(path)
})
@@ -2136,7 +2183,7 @@ pub(crate) async fn rename_all_with_prepared_source(
move || {
validate_prepared_rename_source(&prepared_source, &src_file_path)?;
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
#[cfg(test)]
#[cfg(any(test, feature = "test-util"))]
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path);
rename_prepared(&src_file_path, &dst_file_path, &preparation)
}
@@ -2267,7 +2314,7 @@ async fn reliable_rename_inner_with_lease(
let base_dir = base_dir.clone();
move || {
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
#[cfg(all(test, not(windows)))]
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
{
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src_file_path);
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst_file_path);
+2 -2
View File
@@ -442,7 +442,7 @@ pub(crate) mod utils;
use peer::init_local_peer;
pub use peer::{
all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
prewarm_local_disk_id_map_with_instance_ctx,
};
@@ -1787,7 +1787,7 @@ mod tests {
// Build a minimal ECStore carrying an explicit instance context. Empty
// pools/disks are sufficient: the Phase 5 accessors read only `self.ctx`.
fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
pub(super) fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
let endpoint_pools = EndpointServerPools::default();
Arc::new(ECStore {
id: uuid::Uuid::new_v4(),
+717 -1
View File
@@ -13,7 +13,10 @@
// limitations under the License.
use super::*;
use crate::runtime::instance::InstanceContext;
use crate::bucket::utils::has_bad_path_component;
use crate::disk::error::{DiskError, Result as DiskResult};
use crate::disk::{DeleteOptions, Disk, RenameDataGuards, RenameDataResp};
use crate::runtime::instance::{InstanceContext, NamespaceCommitGuard};
use crate::runtime::sources as runtime_sources;
use tracing::{debug, error};
@@ -22,6 +25,203 @@ const LOG_SUBSYSTEM_DISK_STARTUP: &str = "disk_startup";
const EVENT_LOCAL_DISK_ID_PREWARM_SKIPPED: &str = "local_disk_id_prewarm_skipped";
const EVENT_LOCK_CLIENT_INITIALIZATION_FAILED: &str = "lock_client_initialization_failed";
/// An instance-bound capability for internal writes before ECStore/IAM startup.
/// Its private context and volume checks cannot be replaced by a caller guard.
#[derive(Clone)]
pub struct BootstrapLocalTarget {
ctx: Arc<InstanceContext>,
}
impl BootstrapLocalTarget {
pub fn new(ctx: Arc<InstanceContext>) -> Self {
Self { ctx }
}
pub fn is_for_store(&self, store: &ECStore) -> bool {
Arc::ptr_eq(&self.ctx, &store.ctx)
}
pub async fn rename_local_data(
&self,
disk_ref: &str,
source: (&str, &str),
fi: &FileInfo,
destination: (&str, &str),
scanner_token: Option<Uuid>,
) -> DiskResult<RenameDataResp> {
if scanner_token.is_some() {
return Err(DiskError::other("bootstrap rename cannot use a scanner publication lease"));
}
validate_bootstrap_volume(source.0)?;
validate_bootstrap_volume(destination.0)?;
rename_local_data_with_ctx(&self.ctx, disk_ref, source, fi, destination, RenameDataGuards::default()).await
}
pub async fn undo_local_write(
&self,
disk_ref: &str,
volume: &str,
path: &str,
fi: FileInfo,
opts: DeleteOptions,
) -> DiskResult<()> {
validate_bootstrap_volume(volume)?;
undo_local_write_with_ctx(&self.ctx, disk_ref, volume, path, fi, opts).await
}
}
fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
// Prefix membership alone permits aliases such as .rustfs.sys/../bucket.
// Validate both raw rename volumes before any disk lookup or admission.
if has_bad_path_component(volume) || !is_meta_bucketname(volume) {
return Err(DiskError::FileAccessDenied);
}
Ok(())
}
impl ECStore {
/// Execute on this instance's active local disk through the physical owner.
pub async fn rename_local_data(
&self,
disk_ref: &str,
source: (&str, &str),
fi: &FileInfo,
destination: (&str, &str),
scanner_token: Option<Uuid>,
) -> DiskResult<RenameDataResp> {
let external_guard: Option<Arc<dyn Send + Sync>> = if let Some(token) = scanner_token {
Some(Arc::new(
self.acquire_scanner_publication_lease_guard(token)
.await
.map_err(|err| DiskError::other(err.to_string()))?,
))
} else {
None
};
rename_local_data_with_ctx(
&self.ctx,
disk_ref,
source,
fi,
destination,
RenameDataGuards {
scanner_publication_lease_token: scanner_token,
external_guard,
namespace_owner: None,
},
)
.await
}
pub async fn undo_local_write(
&self,
disk_ref: &str,
volume: &str,
path: &str,
fi: FileInfo,
opts: DeleteOptions,
) -> DiskResult<()> {
undo_local_write_with_ctx(&self.ctx, disk_ref, volume, path, fi, opts).await
}
}
// The optional ID is a cold lookup to cache only after final admission.
async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<(DiskStore, Option<Uuid>)> {
let map = ctx.local_disk_map();
if let Some(disk) = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned() {
return Ok((disk, None));
}
let disk_id = Uuid::parse_str(disk_ref).map_err(|_| DiskError::DiskNotFound)?;
let cached_path = ctx.local_disk_id_map().read().await.get(&disk_id).cloned();
if let Some(path) = cached_path {
let cached_disk = map.read().await.get(&path).and_then(Option::as_ref).cloned();
if let Some(disk) = cached_disk
&& matches!(disk.as_ref(), Disk::Local(_))
&& disk.get_disk_id().await? == Some(disk_id)
{
return Ok((disk, None));
}
}
let disks: Vec<_> = map.read().await.values().filter_map(Clone::clone).collect();
// Disk identity may perform format I/O. No registry guard spans this await.
for disk in disks {
if matches!(disk.as_ref(), Disk::Local(_)) && disk.get_disk_id().await.ok().flatten() == Some(disk_id) {
return Ok((disk, Some(disk_id)));
}
}
Err(DiskError::DiskNotFound)
}
async fn admit_local_disk(
ctx: &Arc<InstanceContext>,
disk: &DiskStore,
disk_id: Option<Uuid>,
volume: &str,
) -> DiskResult<Option<Arc<NamespaceCommitGuard>>> {
if !matches!(disk.as_ref(), Disk::Local(_)) {
return Err(DiskError::DiskNotFound);
}
let map = ctx.local_disk_map();
let active = map.read().await;
if !active
.get(&disk.endpoint().to_string())
.and_then(Option::as_ref)
.is_some_and(|current| Arc::ptr_eq(current, disk))
{
return Err(DiskError::DiskNotFound);
}
// Preserve registry -> ID-cache lock order; no filesystem I/O under either.
if let Some(disk_id) = disk_id {
ctx.local_disk_id_map()
.write()
.await
.insert(disk_id, disk.endpoint().to_string());
}
// Admission linearizes under the registry read: replacement/quarantine
// before this point rejects; later changes do not revoke physical I/O.
Ok((!is_meta_bucketname(volume)).then(|| ctx.begin_namespace_commit()))
}
async fn rename_local_data_with_ctx(
ctx: &Arc<InstanceContext>,
disk_ref: &str,
source: (&str, &str),
fi: &FileInfo,
destination: (&str, &str),
mut guards: RenameDataGuards,
) -> DiskResult<RenameDataResp> {
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
let owner = admit_local_disk(ctx, &disk, disk_id, destination.0).await?;
guards.namespace_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
let result = disk
.rename_data_borrowed_with_fence_observed(source.0, source.1, fi, destination.0, destination.1, guards)
.await
.result;
drop(owner);
result
}
async fn undo_local_write_with_ctx(
ctx: &Arc<InstanceContext>,
disk_ref: &str,
volume: &str,
path: &str,
fi: FileInfo,
opts: DeleteOptions,
) -> DiskResult<()> {
if !opts.undo_write {
return Err(DiskError::other("target undo requires undo_write"));
}
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
let owner = admit_local_disk(ctx, &disk, disk_id, volume).await?;
let physical_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
let result = disk
.undo_write_with_namespace_owner(volume, path, fi, opts, physical_owner)
.await;
drop(owner);
result
}
async fn remember_local_disk_id(disk: &DiskStore) -> Option<Uuid> {
remember_local_disk_id_with_instance_ctx(&crate::runtime::global::current_ctx(), disk).await
}
@@ -265,6 +465,522 @@ mod tests {
}])
}
async fn target_disk(ctx: &Arc<InstanceContext>, root: &std::path::Path, id: Uuid) -> DiskStore {
let mut format = crate::layout::format::FormatV3::new(1, 1);
format.erasure.this = id;
format.erasure.sets[0][0] = id;
let meta = root.join(crate::disk::RUSTFS_META_BUCKET);
tokio::fs::create_dir_all(&meta).await.expect("create format volume");
tokio::fs::write(
meta.join(crate::disk::FORMAT_CONFIG_FILE),
serde_json::to_vec(&format).expect("encode format"),
)
.await
.expect("write real disk identity");
let mut endpoint = Endpoint::try_from(root.to_str().expect("UTF-8 root")).expect("endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("open real local disk");
assert_eq!(disk.get_disk_id().await.expect("read disk format identity"), Some(id));
ctx.local_disk_map()
.write()
.await
.insert(disk.endpoint().to_string(), Some(disk.clone()));
disk
}
fn target_file_info(object: &str, version: Uuid, body: &'static [u8]) -> FileInfo {
let mut fi = FileInfo::new(object, 1, 0);
fi.erasure.index = 1;
fi.version_id = Some(version);
fi.mod_time = Some(OffsetDateTime::now_utc());
fi.size = i64::try_from(body.len()).expect("fixture length");
fi.parts = vec![rustfs_filemeta::ObjectPartInfo {
number: 1,
size: body.len(),
actual_size: fi.size,
..Default::default()
}];
fi.data = Some(bytes::Bytes::from_static(body));
fi.set_inline_data();
fi
}
async fn seed_target(disk: &DiskStore, volume: &str, object: &str, fi: FileInfo) -> Vec<u8> {
let dir = disk.path().join(volume);
tokio::fs::create_dir_all(&dir).await.expect("real fixture volume");
disk.write_metadata(volume, volume, object, fi.clone())
.await
.expect("seed real metadata");
let read = disk
.read_version(
volume,
volume,
object,
&fi.version_id.expect("fixture version").to_string(),
&crate::disk::ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("read fixture before mutation");
assert_eq!(read.data, fi.data, "fixture must contain readable inline bytes");
tokio::fs::read(dir.join(object).join(crate::disk::STORAGE_FORMAT_FILE))
.await
.expect("seeded metadata bytes")
}
#[tokio::test]
async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
for warm in [false, true] {
let ctx_a = Arc::new(InstanceContext::new());
let ctx_b = Arc::new(InstanceContext::new());
let a = tempfile::tempdir().expect("A root");
let b = tempfile::tempdir().expect("B root");
let id = Uuid::new_v4();
let disk_a = target_disk(&ctx_a, a.path(), id).await;
let disk_b = target_disk(&ctx_b, b.path(), id).await;
if warm {
assert!(record_local_disk_id_if_active(&ctx_a, &disk_a, id).await);
assert!(record_local_disk_id_if_active(&ctx_b, &disk_b, id).await);
}
let version = Uuid::new_v4();
let fi = target_file_info("destination", version, b"new-A");
for disk in [&disk_a, &disk_b] {
seed_target(disk, "target-bucket", "staged", fi.clone()).await;
}
let b_before = seed_target(
&disk_b,
"target-bucket",
"destination",
target_file_info("destination", version, b"old-B"),
)
.await;
let store = super::super::tests::build_store_with_ctx(ctx_a.clone());
store
.rename_local_data(&id.to_string(), ("target-bucket", "staged"), &fi, ("target-bucket", "destination"), None)
.await
.expect("rename on A");
let read = disk_a
.read_version(
"target-bucket",
"target-bucket",
"destination",
&version.to_string(),
&crate::disk::ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("read committed A");
assert_eq!(read.data, fi.data, "warm={warm}");
assert_eq!(
tokio::fs::read(b.path().join("target-bucket/destination/xl.meta"))
.await
.expect("B metadata"),
b_before
);
assert!(b.path().join("target-bucket/staged/xl.meta").exists());
assert!(ctx_a.namespace_commit_generation() > 0);
assert_eq!(ctx_b.namespace_commit_generation(), 0);
assert!(!ctx_a.namespace_commits_pending());
assert!(!ctx_b.namespace_commits_pending());
assert_eq!(ctx_a.local_disk_id_map().read().await.get(&id), Some(&disk_a.endpoint().to_string()));
}
}
#[tokio::test]
async fn target_admission_rejects_removed_quarantined_and_replaced_arcs() {
let ctx = Arc::new(InstanceContext::new());
let root = tempfile::tempdir().expect("root");
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
let endpoint = disk.endpoint().to_string();
for state in ["removed", "quarantined", "replaced"] {
let replacement = new_disk(
&disk.endpoint(),
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("separate active Arc");
let map = ctx.local_disk_map();
let mut entries = map.write().await;
match state {
"removed" => {
entries.remove(&endpoint);
}
"quarantined" => {
entries.insert(endpoint.clone(), None);
}
_ => {
entries.insert(endpoint.clone(), Some(replacement));
}
}
drop(entries);
assert!(
matches!(admit_local_disk(&ctx, &disk, None, "target-bucket").await, Err(DiskError::DiskNotFound)),
"{state}"
);
assert!(!ctx.namespace_commits_pending());
assert_eq!(ctx.namespace_commit_generation(), 0);
}
}
#[tokio::test]
async fn target_uuid_cache_cannot_admit_a_different_format_at_the_same_path() {
let ctx = Arc::new(InstanceContext::new());
let root = tempfile::tempdir().expect("root");
let old_id = Uuid::new_v4();
let old = target_disk(&ctx, root.path(), old_id).await;
assert!(record_local_disk_id_if_active(&ctx, &old, old_id).await);
let replacement_id = Uuid::new_v4();
let replacement = target_disk(&ctx, root.path(), replacement_id).await;
assert!(!Arc::ptr_eq(&old, &replacement));
assert!(matches!(
local_disk_candidate(&ctx, &old_id.to_string()).await,
Err(DiskError::DiskNotFound)
));
let (candidate, verified) = local_disk_candidate(&ctx, &replacement_id.to_string())
.await
.expect("replacement UUID");
assert!(Arc::ptr_eq(&candidate, &replacement));
assert_eq!(verified, Some(replacement_id));
assert!(!ctx.namespace_commits_pending());
}
#[tokio::test]
async fn bootstrap_rejects_user_volumes_aliases_and_scanner_tokens_without_mutation() {
let ctx = Arc::new(InstanceContext::new());
let root = tempfile::tempdir().expect("root");
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
let target = BootstrapLocalTarget::new(ctx.clone());
let fi = target_file_info("destination", Uuid::new_v4(), b"body");
let user_before = seed_target(&disk, "victim", "staged", fi.clone()).await;
let meta_before = seed_target(&disk, ".rustfs.sys/tmp", "staged", fi.clone()).await;
for invalid in [
"victim",
".rustfs.sys/../victim",
".rustfs.sys/./tmp",
".rustfs.sys/ .. /victim",
".rustfs.sys\\..\\victim",
".minio.sys/../victim",
] {
for (src, dst) in [(invalid, ".rustfs.sys/tmp"), (".rustfs.sys/tmp", invalid)] {
assert!(
target
.rename_local_data(&disk.endpoint().to_string(), (src, "staged"), &fi, (dst, "destination"), None)
.await
.is_err(),
"src={src}, dst={dst}"
);
}
assert!(
target
.undo_local_write(
&disk.endpoint().to_string(),
invalid,
"staged",
fi.clone(),
DeleteOptions {
undo_write: true,
..Default::default()
}
)
.await
.is_err(),
"{invalid}"
);
}
assert!(
target
.rename_local_data(
&disk.endpoint().to_string(),
(".rustfs.sys/tmp", "staged"),
&fi,
(".rustfs.sys/tmp", "destination"),
Some(Uuid::new_v4())
)
.await
.is_err()
);
assert_eq!(
tokio::fs::read(root.path().join("victim/staged/xl.meta"))
.await
.expect("user source"),
user_before
);
assert_eq!(
tokio::fs::read(root.path().join(".rustfs.sys/tmp/staged/xl.meta"))
.await
.expect("metadata source"),
meta_before
);
assert!(!root.path().join("victim/destination").exists());
assert!(!root.path().join(".rustfs.sys/tmp/destination").exists());
assert_eq!(ctx.namespace_commit_generation(), 0);
assert!(!ctx.namespace_commits_pending());
}
#[tokio::test]
async fn bootstrap_allows_internal_multisegment_rename_without_namespace_owner() {
for volume in [".rustfs.sys/tmp", ".rustfs.sys/multipart", ".minio.sys/config"] {
let ctx = Arc::new(InstanceContext::new());
let root = tempfile::tempdir().expect("root");
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
let fi = target_file_info("destination", Uuid::new_v4(), b"internal-CAS-body");
seed_target(&disk, volume, "staged", fi.clone()).await;
BootstrapLocalTarget::new(ctx.clone())
.rename_local_data(&disk.endpoint().to_string(), (volume, "staged"), &fi, (volume, "destination"), None)
.await
.expect("legitimate bootstrap metadata write");
let read = disk
.read_version(
volume,
volume,
"destination",
&fi.version_id.expect("version").to_string(),
&crate::disk::ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("read bootstrap result");
assert_eq!(read.data, fi.data);
assert_eq!(ctx.namespace_commit_generation(), 0);
assert!(!ctx.namespace_commits_pending());
}
}
#[cfg(not(windows))]
#[tokio::test]
async fn target_rename_cancellation_retains_real_namespace_and_scanner_owners() {
use crate::disk::os::prepared_publication_test_hooks as hooks;
let ctx = Arc::new(InstanceContext::new());
let sibling = Arc::new(InstanceContext::new());
let root = tempfile::tempdir().expect("root");
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
let store = super::super::tests::build_store_with_ctx(ctx.clone());
let fi = target_file_info("destination", Uuid::new_v4(), b"physically-owned");
seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
let (token, _) = store
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
.await
.expect("real scanner token in A");
let destination = disk
.get_object_path_for_io_if_local("target-bucket", "destination/xl.meta")
.expect("local disk")
.expect("destination IO path");
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
let _hook = hooks::install(&destination, move || {
let _ = entered_tx.send(());
let _ = release_rx.recv();
});
let disk_ref = disk.endpoint().to_string();
let mut rename = Box::pin(store.rename_local_data(
&disk_ref,
("target-bucket", "staged"),
&fi,
("target-bucket", "destination"),
Some(token),
));
tokio::time::timeout(std::time::Duration::from_secs(10), async {
tokio::select! {
result = &mut rename => panic!("rename completed before physical pause: {result:?}"),
entered = entered_rx => entered.expect("physical rename entered"),
}
})
.await
.expect("bounded physical entry");
drop(rename);
assert!(store.scanner_data_usage_publication_blocked().await);
assert!(ctx.namespace_commits_pending());
assert!(!sibling.namespace_commits_pending());
assert!(
store
.rename_local_data(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "another"), Some(token))
.await
.is_err(),
"real pending rename blocks another scanner publication"
);
assert!(store.release_scanner_publication_lease(token).await, "remove registered token");
let gate = ctx.data_movement_operation_gate();
assert!(
gate.clone().try_write_owned().is_err(),
"physical operation still owns the scanner read guard"
);
drop(release_tx);
let _drained = tokio::time::timeout(std::time::Duration::from_secs(10), gate.write_owned())
.await
.expect("physical tail must release scanner guard");
tokio::time::timeout(std::time::Duration::from_secs(10), async {
while ctx.namespace_commits_pending() {
tokio::task::yield_now().await;
}
})
.await
.expect("namespace owner drains");
let read = disk
.read_version(
"target-bucket",
"target-bucket",
"destination",
&fi.version_id.expect("version").to_string(),
&crate::disk::ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("read actual late commit");
assert_eq!(read.data, fi.data);
assert!(ctx.namespace_commit_generation() >= 2);
assert_eq!(sibling.namespace_commit_generation(), 0);
}
#[tokio::test]
async fn target_ready_rejects_unknown_foreign_released_and_expired_scanner_tokens() {
let ctx = Arc::new(InstanceContext::new());
let other = Arc::new(InstanceContext::new());
let store = super::super::tests::build_store_with_ctx(ctx.clone());
let other_store = super::super::tests::build_store_with_ctx(other);
let root = tempfile::tempdir().expect("root");
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
let fi = target_file_info("destination", Uuid::new_v4(), b"unchanged");
let before = seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
let ttl = crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL;
let (foreign, _) = other_store.acquire_scanner_publication_lease(0, ttl).await.expect("B token");
let (released, _) = store.acquire_scanner_publication_lease(0, ttl).await.expect("A token");
assert!(store.release_scanner_publication_lease(released).await);
let (valid, _) = store.acquire_scanner_publication_lease(0, ttl).await.expect("new A token");
for token in [Uuid::new_v4(), foreign, released] {
assert!(
store
.rename_local_data(
&disk.endpoint().to_string(),
("target-bucket", "staged"),
&fi,
("target-bucket", "destination"),
Some(token)
)
.await
.is_err()
);
}
tokio::time::pause();
tokio::time::advance(ttl + std::time::Duration::from_secs(1)).await;
tokio::time::resume();
assert!(
store
.rename_local_data(
&disk.endpoint().to_string(),
("target-bucket", "staged"),
&fi,
("target-bucket", "destination"),
Some(valid)
)
.await
.is_err(),
"expired real token"
);
let _ = other_store.release_scanner_publication_lease(foreign).await;
assert_eq!(
tokio::fs::read(root.path().join("target-bucket/staged/xl.meta"))
.await
.expect("source bytes"),
before
);
assert!(!root.path().join("target-bucket/destination").exists());
assert!(!ctx.namespace_commits_pending());
}
#[cfg(not(windows))]
#[tokio::test]
#[serial_test::serial]
async fn target_ordinary_timeout_keeps_its_physical_namespace_owner() {
use crate::disk::os::prepared_publication_test_hooks as hooks;
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("1"))], async {
let ctx = Arc::new(InstanceContext::new());
let store = super::super::tests::build_store_with_ctx(ctx.clone());
let root = tempfile::tempdir().expect("root");
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
let fi = target_file_info("destination", Uuid::new_v4(), b"timed-out-physical-commit");
seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
let path = disk
.get_object_path_for_io_if_local("target-bucket", "destination/xl.meta")
.expect("local")
.expect("destination IO path");
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
let _hook = hooks::install(&path, move || {
let _ = entered_tx.send(());
let _ = release_rx.recv();
});
let disk_ref = disk.endpoint().to_string();
let mut rename = Box::pin(store.rename_local_data(
&disk_ref,
("target-bucket", "staged"),
&fi,
("target-bucket", "destination"),
None,
));
tokio::time::timeout(std::time::Duration::from_secs(10), async {
tokio::select! {
result = &mut rename => panic!("completed before physical pause: {result:?}"),
entered = entered_rx => entered.expect("physical entry"),
}
})
.await
.expect("bounded entry");
tokio::time::pause();
tokio::time::advance(std::time::Duration::from_secs(2)).await;
tokio::time::resume();
let result = tokio::time::timeout(std::time::Duration::from_secs(5), &mut rename)
.await
.expect("ordinary deadline remains enabled");
assert!(matches!(result, Err(DiskError::Timeout)), "{result:?}");
drop(rename);
assert!(ctx.namespace_commits_pending(), "timeout is not a physical drain");
drop(release_tx);
tokio::time::timeout(std::time::Duration::from_secs(10), async {
while ctx.namespace_commits_pending() {
tokio::task::yield_now().await;
}
})
.await
.expect("late physical owner drains");
let read = disk
.read_version(
"target-bucket",
"target-bucket",
"destination",
&fi.version_id.expect("version").to_string(),
&crate::disk::ReadOptions {
read_data: true,
..Default::default()
},
)
.await
.expect("read actual timeout tail");
assert_eq!(read.data, fi.data);
})
.await;
}
#[test]
fn endpoint_rpc_authority_preserves_port_and_ipv6_brackets() {
let endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint");