mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-07 20:46:11 +00:00
fix(rpc): bind local mutations to the listener instance
This commit is contained in:
@@ -366,6 +366,8 @@ pub mod data_usage {
|
|||||||
pub mod disk {
|
pub mod disk {
|
||||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||||
pub use crate::disk::local::ScanGuard;
|
pub use crate::disk::local::ScanGuard;
|
||||||
|
#[cfg(all(feature = "test-util", not(windows)))]
|
||||||
|
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
|
||||||
pub use crate::disk::{
|
pub use crate::disk::{
|
||||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||||
CheckPartsResp, ConditionalFileUpdate, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
|
CheckPartsResp, ConditionalFileUpdate, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
|
||||||
@@ -542,8 +544,8 @@ pub mod storage {
|
|||||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||||
pub use crate::store::HealWalkVersion;
|
pub use crate::store::HealWalkVersion;
|
||||||
pub use crate::store::{
|
pub use crate::store::{
|
||||||
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path,
|
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||||
find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||||
prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
|
prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -243,7 +243,7 @@ pub(crate) mod fsync_dir_recorder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Pause a real namespace mutation inside its physical executor.
|
/// Pause a real namespace mutation inside its physical executor.
|
||||||
#[cfg(all(test, not(windows)))]
|
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||||
pub(crate) mod prepared_publication_test_hooks {
|
pub(crate) mod prepared_publication_test_hooks {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
@@ -252,6 +252,7 @@ pub(crate) mod prepared_publication_test_hooks {
|
|||||||
PreparedRename,
|
PreparedRename,
|
||||||
Rename,
|
Rename,
|
||||||
Remove,
|
Remove,
|
||||||
|
#[cfg(test)]
|
||||||
Rollback,
|
Rollback,
|
||||||
DirFsync,
|
DirFsync,
|
||||||
}
|
}
|
||||||
@@ -268,6 +269,7 @@ pub(crate) mod prepared_publication_test_hooks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
pub(crate) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
|
pub(crate) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard {
|
||||||
install_at(Stage::PreparedRename, path, hook)
|
install_at(Stage::PreparedRename, path, hook)
|
||||||
}
|
}
|
||||||
@@ -286,6 +288,51 @@ pub(crate) mod prepared_publication_test_hooks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Controlled application-test pause at an existing physical executor boundary.
|
||||||
|
#[cfg(all(feature = "test-util", not(windows)))]
|
||||||
|
pub struct LocalPublicationPause {
|
||||||
|
_hook: prepared_publication_test_hooks::Guard,
|
||||||
|
entered: oneshot::Receiver<()>,
|
||||||
|
_release: std::sync::mpsc::Sender<()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(feature = "test-util", not(windows)))]
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
pub enum LocalPublicationStage {
|
||||||
|
PreparedRename,
|
||||||
|
Rename,
|
||||||
|
Remove,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(feature = "test-util", not(windows)))]
|
||||||
|
impl LocalPublicationPause {
|
||||||
|
pub fn install(disk: &crate::disk::Disk, volume: &str, path: &str, stage: LocalPublicationStage) -> Result<Self> {
|
||||||
|
let path = disk
|
||||||
|
.get_object_path_for_io_if_local(volume, path)
|
||||||
|
.ok_or(DiskError::DiskNotFound)??;
|
||||||
|
let stage = match stage {
|
||||||
|
LocalPublicationStage::PreparedRename => prepared_publication_test_hooks::Stage::PreparedRename,
|
||||||
|
LocalPublicationStage::Rename => prepared_publication_test_hooks::Stage::Rename,
|
||||||
|
LocalPublicationStage::Remove => prepared_publication_test_hooks::Stage::Remove,
|
||||||
|
};
|
||||||
|
let (entered_tx, entered) = oneshot::channel();
|
||||||
|
let (release, release_rx) = std::sync::mpsc::channel::<()>();
|
||||||
|
let hook = prepared_publication_test_hooks::install_at(stage, &path, move || {
|
||||||
|
let _ = entered_tx.send(());
|
||||||
|
let _ = release_rx.recv();
|
||||||
|
});
|
||||||
|
Ok(Self {
|
||||||
|
_hook: hook,
|
||||||
|
entered,
|
||||||
|
_release: release,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn entered(&mut self) -> std::result::Result<(), oneshot::error::RecvError> {
|
||||||
|
(&mut self.entered).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(all(test, windows))]
|
#[cfg(all(test, windows))]
|
||||||
pub(crate) mod windows_rename_test_hooks {
|
pub(crate) mod windows_rename_test_hooks {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -776,7 +823,7 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup, namespace_owners: Ve
|
|||||||
fsync_spawn_blocking(move || {
|
fsync_spawn_blocking(move || {
|
||||||
// The batch worker may be cancelled while this syscall is still running.
|
// The batch worker may be cancelled while this syscall is still running.
|
||||||
let _namespace_owners = namespace_owners;
|
let _namespace_owners = namespace_owners;
|
||||||
#[cfg(all(test, not(windows)))]
|
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::DirFsync, &dir);
|
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::DirFsync, &dir);
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
{
|
{
|
||||||
@@ -1912,7 +1959,7 @@ pub(crate) async fn remove_file_with_owner(
|
|||||||
let path = path.as_ref().to_path_buf();
|
let path = path.as_ref().to_path_buf();
|
||||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||||
run_blocking_namespace_operation(lease, move || {
|
run_blocking_namespace_operation(lease, move || {
|
||||||
#[cfg(all(test, not(windows)))]
|
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path);
|
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path);
|
||||||
std::fs::remove_file(path)
|
std::fs::remove_file(path)
|
||||||
})
|
})
|
||||||
@@ -2136,7 +2183,7 @@ pub(crate) async fn rename_all_with_prepared_source(
|
|||||||
move || {
|
move || {
|
||||||
validate_prepared_rename_source(&prepared_source, &src_file_path)?;
|
validate_prepared_rename_source(&prepared_source, &src_file_path)?;
|
||||||
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
|
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
|
||||||
#[cfg(test)]
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path);
|
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path);
|
||||||
rename_prepared(&src_file_path, &dst_file_path, &preparation)
|
rename_prepared(&src_file_path, &dst_file_path, &preparation)
|
||||||
}
|
}
|
||||||
@@ -2267,7 +2314,7 @@ async fn reliable_rename_inner_with_lease(
|
|||||||
let base_dir = base_dir.clone();
|
let base_dir = base_dir.clone();
|
||||||
move || {
|
move || {
|
||||||
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
|
let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?;
|
||||||
#[cfg(all(test, not(windows)))]
|
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||||
{
|
{
|
||||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src_file_path);
|
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src_file_path);
|
||||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst_file_path);
|
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst_file_path);
|
||||||
|
|||||||
@@ -442,7 +442,7 @@ pub(crate) mod utils;
|
|||||||
|
|
||||||
use peer::init_local_peer;
|
use peer::init_local_peer;
|
||||||
pub use peer::{
|
pub use peer::{
|
||||||
all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
|
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
|
||||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||||
prewarm_local_disk_id_map_with_instance_ctx,
|
prewarm_local_disk_id_map_with_instance_ctx,
|
||||||
};
|
};
|
||||||
@@ -1787,7 +1787,7 @@ mod tests {
|
|||||||
|
|
||||||
// Build a minimal ECStore carrying an explicit instance context. Empty
|
// Build a minimal ECStore carrying an explicit instance context. Empty
|
||||||
// pools/disks are sufficient: the Phase 5 accessors read only `self.ctx`.
|
// pools/disks are sufficient: the Phase 5 accessors read only `self.ctx`.
|
||||||
fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
|
pub(super) fn build_store_with_ctx(ctx: Arc<InstanceContext>) -> Arc<ECStore> {
|
||||||
let endpoint_pools = EndpointServerPools::default();
|
let endpoint_pools = EndpointServerPools::default();
|
||||||
Arc::new(ECStore {
|
Arc::new(ECStore {
|
||||||
id: uuid::Uuid::new_v4(),
|
id: uuid::Uuid::new_v4(),
|
||||||
|
|||||||
@@ -13,7 +13,10 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::runtime::instance::InstanceContext;
|
use crate::bucket::utils::has_bad_path_component;
|
||||||
|
use crate::disk::error::{DiskError, Result as DiskResult};
|
||||||
|
use crate::disk::{DeleteOptions, Disk, RenameDataGuards, RenameDataResp};
|
||||||
|
use crate::runtime::instance::{InstanceContext, NamespaceCommitGuard};
|
||||||
use crate::runtime::sources as runtime_sources;
|
use crate::runtime::sources as runtime_sources;
|
||||||
use tracing::{debug, error};
|
use tracing::{debug, error};
|
||||||
|
|
||||||
@@ -22,6 +25,203 @@ const LOG_SUBSYSTEM_DISK_STARTUP: &str = "disk_startup";
|
|||||||
const EVENT_LOCAL_DISK_ID_PREWARM_SKIPPED: &str = "local_disk_id_prewarm_skipped";
|
const EVENT_LOCAL_DISK_ID_PREWARM_SKIPPED: &str = "local_disk_id_prewarm_skipped";
|
||||||
const EVENT_LOCK_CLIENT_INITIALIZATION_FAILED: &str = "lock_client_initialization_failed";
|
const EVENT_LOCK_CLIENT_INITIALIZATION_FAILED: &str = "lock_client_initialization_failed";
|
||||||
|
|
||||||
|
/// An instance-bound capability for internal writes before ECStore/IAM startup.
|
||||||
|
/// Its private context and volume checks cannot be replaced by a caller guard.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct BootstrapLocalTarget {
|
||||||
|
ctx: Arc<InstanceContext>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BootstrapLocalTarget {
|
||||||
|
pub fn new(ctx: Arc<InstanceContext>) -> Self {
|
||||||
|
Self { ctx }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_for_store(&self, store: &ECStore) -> bool {
|
||||||
|
Arc::ptr_eq(&self.ctx, &store.ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn rename_local_data(
|
||||||
|
&self,
|
||||||
|
disk_ref: &str,
|
||||||
|
source: (&str, &str),
|
||||||
|
fi: &FileInfo,
|
||||||
|
destination: (&str, &str),
|
||||||
|
scanner_token: Option<Uuid>,
|
||||||
|
) -> DiskResult<RenameDataResp> {
|
||||||
|
if scanner_token.is_some() {
|
||||||
|
return Err(DiskError::other("bootstrap rename cannot use a scanner publication lease"));
|
||||||
|
}
|
||||||
|
validate_bootstrap_volume(source.0)?;
|
||||||
|
validate_bootstrap_volume(destination.0)?;
|
||||||
|
rename_local_data_with_ctx(&self.ctx, disk_ref, source, fi, destination, RenameDataGuards::default()).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn undo_local_write(
|
||||||
|
&self,
|
||||||
|
disk_ref: &str,
|
||||||
|
volume: &str,
|
||||||
|
path: &str,
|
||||||
|
fi: FileInfo,
|
||||||
|
opts: DeleteOptions,
|
||||||
|
) -> DiskResult<()> {
|
||||||
|
validate_bootstrap_volume(volume)?;
|
||||||
|
undo_local_write_with_ctx(&self.ctx, disk_ref, volume, path, fi, opts).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
|
||||||
|
// Prefix membership alone permits aliases such as .rustfs.sys/../bucket.
|
||||||
|
// Validate both raw rename volumes before any disk lookup or admission.
|
||||||
|
if has_bad_path_component(volume) || !is_meta_bucketname(volume) {
|
||||||
|
return Err(DiskError::FileAccessDenied);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ECStore {
|
||||||
|
/// Execute on this instance's active local disk through the physical owner.
|
||||||
|
pub async fn rename_local_data(
|
||||||
|
&self,
|
||||||
|
disk_ref: &str,
|
||||||
|
source: (&str, &str),
|
||||||
|
fi: &FileInfo,
|
||||||
|
destination: (&str, &str),
|
||||||
|
scanner_token: Option<Uuid>,
|
||||||
|
) -> DiskResult<RenameDataResp> {
|
||||||
|
let external_guard: Option<Arc<dyn Send + Sync>> = if let Some(token) = scanner_token {
|
||||||
|
Some(Arc::new(
|
||||||
|
self.acquire_scanner_publication_lease_guard(token)
|
||||||
|
.await
|
||||||
|
.map_err(|err| DiskError::other(err.to_string()))?,
|
||||||
|
))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
rename_local_data_with_ctx(
|
||||||
|
&self.ctx,
|
||||||
|
disk_ref,
|
||||||
|
source,
|
||||||
|
fi,
|
||||||
|
destination,
|
||||||
|
RenameDataGuards {
|
||||||
|
scanner_publication_lease_token: scanner_token,
|
||||||
|
external_guard,
|
||||||
|
namespace_owner: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn undo_local_write(
|
||||||
|
&self,
|
||||||
|
disk_ref: &str,
|
||||||
|
volume: &str,
|
||||||
|
path: &str,
|
||||||
|
fi: FileInfo,
|
||||||
|
opts: DeleteOptions,
|
||||||
|
) -> DiskResult<()> {
|
||||||
|
undo_local_write_with_ctx(&self.ctx, disk_ref, volume, path, fi, opts).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The optional ID is a cold lookup to cache only after final admission.
|
||||||
|
async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<(DiskStore, Option<Uuid>)> {
|
||||||
|
let map = ctx.local_disk_map();
|
||||||
|
if let Some(disk) = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned() {
|
||||||
|
return Ok((disk, None));
|
||||||
|
}
|
||||||
|
let disk_id = Uuid::parse_str(disk_ref).map_err(|_| DiskError::DiskNotFound)?;
|
||||||
|
let cached_path = ctx.local_disk_id_map().read().await.get(&disk_id).cloned();
|
||||||
|
if let Some(path) = cached_path {
|
||||||
|
let cached_disk = map.read().await.get(&path).and_then(Option::as_ref).cloned();
|
||||||
|
if let Some(disk) = cached_disk
|
||||||
|
&& matches!(disk.as_ref(), Disk::Local(_))
|
||||||
|
&& disk.get_disk_id().await? == Some(disk_id)
|
||||||
|
{
|
||||||
|
return Ok((disk, None));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let disks: Vec<_> = map.read().await.values().filter_map(Clone::clone).collect();
|
||||||
|
// Disk identity may perform format I/O. No registry guard spans this await.
|
||||||
|
for disk in disks {
|
||||||
|
if matches!(disk.as_ref(), Disk::Local(_)) && disk.get_disk_id().await.ok().flatten() == Some(disk_id) {
|
||||||
|
return Ok((disk, Some(disk_id)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(DiskError::DiskNotFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn admit_local_disk(
|
||||||
|
ctx: &Arc<InstanceContext>,
|
||||||
|
disk: &DiskStore,
|
||||||
|
disk_id: Option<Uuid>,
|
||||||
|
volume: &str,
|
||||||
|
) -> DiskResult<Option<Arc<NamespaceCommitGuard>>> {
|
||||||
|
if !matches!(disk.as_ref(), Disk::Local(_)) {
|
||||||
|
return Err(DiskError::DiskNotFound);
|
||||||
|
}
|
||||||
|
let map = ctx.local_disk_map();
|
||||||
|
let active = map.read().await;
|
||||||
|
if !active
|
||||||
|
.get(&disk.endpoint().to_string())
|
||||||
|
.and_then(Option::as_ref)
|
||||||
|
.is_some_and(|current| Arc::ptr_eq(current, disk))
|
||||||
|
{
|
||||||
|
return Err(DiskError::DiskNotFound);
|
||||||
|
}
|
||||||
|
// Preserve registry -> ID-cache lock order; no filesystem I/O under either.
|
||||||
|
if let Some(disk_id) = disk_id {
|
||||||
|
ctx.local_disk_id_map()
|
||||||
|
.write()
|
||||||
|
.await
|
||||||
|
.insert(disk_id, disk.endpoint().to_string());
|
||||||
|
}
|
||||||
|
// Admission linearizes under the registry read: replacement/quarantine
|
||||||
|
// before this point rejects; later changes do not revoke physical I/O.
|
||||||
|
Ok((!is_meta_bucketname(volume)).then(|| ctx.begin_namespace_commit()))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn rename_local_data_with_ctx(
|
||||||
|
ctx: &Arc<InstanceContext>,
|
||||||
|
disk_ref: &str,
|
||||||
|
source: (&str, &str),
|
||||||
|
fi: &FileInfo,
|
||||||
|
destination: (&str, &str),
|
||||||
|
mut guards: RenameDataGuards,
|
||||||
|
) -> DiskResult<RenameDataResp> {
|
||||||
|
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
|
||||||
|
let owner = admit_local_disk(ctx, &disk, disk_id, destination.0).await?;
|
||||||
|
guards.namespace_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
|
||||||
|
let result = disk
|
||||||
|
.rename_data_borrowed_with_fence_observed(source.0, source.1, fi, destination.0, destination.1, guards)
|
||||||
|
.await
|
||||||
|
.result;
|
||||||
|
drop(owner);
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn undo_local_write_with_ctx(
|
||||||
|
ctx: &Arc<InstanceContext>,
|
||||||
|
disk_ref: &str,
|
||||||
|
volume: &str,
|
||||||
|
path: &str,
|
||||||
|
fi: FileInfo,
|
||||||
|
opts: DeleteOptions,
|
||||||
|
) -> DiskResult<()> {
|
||||||
|
if !opts.undo_write {
|
||||||
|
return Err(DiskError::other("target undo requires undo_write"));
|
||||||
|
}
|
||||||
|
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
|
||||||
|
let owner = admit_local_disk(ctx, &disk, disk_id, volume).await?;
|
||||||
|
let physical_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
|
||||||
|
let result = disk
|
||||||
|
.undo_write_with_namespace_owner(volume, path, fi, opts, physical_owner)
|
||||||
|
.await;
|
||||||
|
drop(owner);
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
async fn remember_local_disk_id(disk: &DiskStore) -> Option<Uuid> {
|
async fn remember_local_disk_id(disk: &DiskStore) -> Option<Uuid> {
|
||||||
remember_local_disk_id_with_instance_ctx(&crate::runtime::global::current_ctx(), disk).await
|
remember_local_disk_id_with_instance_ctx(&crate::runtime::global::current_ctx(), disk).await
|
||||||
}
|
}
|
||||||
@@ -265,6 +465,522 @@ mod tests {
|
|||||||
}])
|
}])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn target_disk(ctx: &Arc<InstanceContext>, root: &std::path::Path, id: Uuid) -> DiskStore {
|
||||||
|
let mut format = crate::layout::format::FormatV3::new(1, 1);
|
||||||
|
format.erasure.this = id;
|
||||||
|
format.erasure.sets[0][0] = id;
|
||||||
|
let meta = root.join(crate::disk::RUSTFS_META_BUCKET);
|
||||||
|
tokio::fs::create_dir_all(&meta).await.expect("create format volume");
|
||||||
|
tokio::fs::write(
|
||||||
|
meta.join(crate::disk::FORMAT_CONFIG_FILE),
|
||||||
|
serde_json::to_vec(&format).expect("encode format"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("write real disk identity");
|
||||||
|
let mut endpoint = Endpoint::try_from(root.to_str().expect("UTF-8 root")).expect("endpoint");
|
||||||
|
endpoint.set_pool_index(0);
|
||||||
|
endpoint.set_set_index(0);
|
||||||
|
endpoint.set_disk_index(0);
|
||||||
|
let disk = new_disk(
|
||||||
|
&endpoint,
|
||||||
|
&DiskOption {
|
||||||
|
cleanup: false,
|
||||||
|
health_check: false,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("open real local disk");
|
||||||
|
assert_eq!(disk.get_disk_id().await.expect("read disk format identity"), Some(id));
|
||||||
|
ctx.local_disk_map()
|
||||||
|
.write()
|
||||||
|
.await
|
||||||
|
.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||||
|
disk
|
||||||
|
}
|
||||||
|
|
||||||
|
fn target_file_info(object: &str, version: Uuid, body: &'static [u8]) -> FileInfo {
|
||||||
|
let mut fi = FileInfo::new(object, 1, 0);
|
||||||
|
fi.erasure.index = 1;
|
||||||
|
fi.version_id = Some(version);
|
||||||
|
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||||
|
fi.size = i64::try_from(body.len()).expect("fixture length");
|
||||||
|
fi.parts = vec![rustfs_filemeta::ObjectPartInfo {
|
||||||
|
number: 1,
|
||||||
|
size: body.len(),
|
||||||
|
actual_size: fi.size,
|
||||||
|
..Default::default()
|
||||||
|
}];
|
||||||
|
fi.data = Some(bytes::Bytes::from_static(body));
|
||||||
|
fi.set_inline_data();
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn seed_target(disk: &DiskStore, volume: &str, object: &str, fi: FileInfo) -> Vec<u8> {
|
||||||
|
let dir = disk.path().join(volume);
|
||||||
|
tokio::fs::create_dir_all(&dir).await.expect("real fixture volume");
|
||||||
|
disk.write_metadata(volume, volume, object, fi.clone())
|
||||||
|
.await
|
||||||
|
.expect("seed real metadata");
|
||||||
|
let read = disk
|
||||||
|
.read_version(
|
||||||
|
volume,
|
||||||
|
volume,
|
||||||
|
object,
|
||||||
|
&fi.version_id.expect("fixture version").to_string(),
|
||||||
|
&crate::disk::ReadOptions {
|
||||||
|
read_data: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("read fixture before mutation");
|
||||||
|
assert_eq!(read.data, fi.data, "fixture must contain readable inline bytes");
|
||||||
|
tokio::fs::read(dir.join(object).join(crate::disk::STORAGE_FORMAT_FILE))
|
||||||
|
.await
|
||||||
|
.expect("seeded metadata bytes")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
|
||||||
|
for warm in [false, true] {
|
||||||
|
let ctx_a = Arc::new(InstanceContext::new());
|
||||||
|
let ctx_b = Arc::new(InstanceContext::new());
|
||||||
|
let a = tempfile::tempdir().expect("A root");
|
||||||
|
let b = tempfile::tempdir().expect("B root");
|
||||||
|
let id = Uuid::new_v4();
|
||||||
|
let disk_a = target_disk(&ctx_a, a.path(), id).await;
|
||||||
|
let disk_b = target_disk(&ctx_b, b.path(), id).await;
|
||||||
|
if warm {
|
||||||
|
assert!(record_local_disk_id_if_active(&ctx_a, &disk_a, id).await);
|
||||||
|
assert!(record_local_disk_id_if_active(&ctx_b, &disk_b, id).await);
|
||||||
|
}
|
||||||
|
let version = Uuid::new_v4();
|
||||||
|
let fi = target_file_info("destination", version, b"new-A");
|
||||||
|
for disk in [&disk_a, &disk_b] {
|
||||||
|
seed_target(disk, "target-bucket", "staged", fi.clone()).await;
|
||||||
|
}
|
||||||
|
let b_before = seed_target(
|
||||||
|
&disk_b,
|
||||||
|
"target-bucket",
|
||||||
|
"destination",
|
||||||
|
target_file_info("destination", version, b"old-B"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let store = super::super::tests::build_store_with_ctx(ctx_a.clone());
|
||||||
|
store
|
||||||
|
.rename_local_data(&id.to_string(), ("target-bucket", "staged"), &fi, ("target-bucket", "destination"), None)
|
||||||
|
.await
|
||||||
|
.expect("rename on A");
|
||||||
|
let read = disk_a
|
||||||
|
.read_version(
|
||||||
|
"target-bucket",
|
||||||
|
"target-bucket",
|
||||||
|
"destination",
|
||||||
|
&version.to_string(),
|
||||||
|
&crate::disk::ReadOptions {
|
||||||
|
read_data: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("read committed A");
|
||||||
|
assert_eq!(read.data, fi.data, "warm={warm}");
|
||||||
|
assert_eq!(
|
||||||
|
tokio::fs::read(b.path().join("target-bucket/destination/xl.meta"))
|
||||||
|
.await
|
||||||
|
.expect("B metadata"),
|
||||||
|
b_before
|
||||||
|
);
|
||||||
|
assert!(b.path().join("target-bucket/staged/xl.meta").exists());
|
||||||
|
assert!(ctx_a.namespace_commit_generation() > 0);
|
||||||
|
assert_eq!(ctx_b.namespace_commit_generation(), 0);
|
||||||
|
assert!(!ctx_a.namespace_commits_pending());
|
||||||
|
assert!(!ctx_b.namespace_commits_pending());
|
||||||
|
assert_eq!(ctx_a.local_disk_id_map().read().await.get(&id), Some(&disk_a.endpoint().to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn target_admission_rejects_removed_quarantined_and_replaced_arcs() {
|
||||||
|
let ctx = Arc::new(InstanceContext::new());
|
||||||
|
let root = tempfile::tempdir().expect("root");
|
||||||
|
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||||
|
let endpoint = disk.endpoint().to_string();
|
||||||
|
for state in ["removed", "quarantined", "replaced"] {
|
||||||
|
let replacement = new_disk(
|
||||||
|
&disk.endpoint(),
|
||||||
|
&DiskOption {
|
||||||
|
cleanup: false,
|
||||||
|
health_check: false,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("separate active Arc");
|
||||||
|
let map = ctx.local_disk_map();
|
||||||
|
let mut entries = map.write().await;
|
||||||
|
match state {
|
||||||
|
"removed" => {
|
||||||
|
entries.remove(&endpoint);
|
||||||
|
}
|
||||||
|
"quarantined" => {
|
||||||
|
entries.insert(endpoint.clone(), None);
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
entries.insert(endpoint.clone(), Some(replacement));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
drop(entries);
|
||||||
|
assert!(
|
||||||
|
matches!(admit_local_disk(&ctx, &disk, None, "target-bucket").await, Err(DiskError::DiskNotFound)),
|
||||||
|
"{state}"
|
||||||
|
);
|
||||||
|
assert!(!ctx.namespace_commits_pending());
|
||||||
|
assert_eq!(ctx.namespace_commit_generation(), 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn target_uuid_cache_cannot_admit_a_different_format_at_the_same_path() {
|
||||||
|
let ctx = Arc::new(InstanceContext::new());
|
||||||
|
let root = tempfile::tempdir().expect("root");
|
||||||
|
let old_id = Uuid::new_v4();
|
||||||
|
let old = target_disk(&ctx, root.path(), old_id).await;
|
||||||
|
assert!(record_local_disk_id_if_active(&ctx, &old, old_id).await);
|
||||||
|
let replacement_id = Uuid::new_v4();
|
||||||
|
let replacement = target_disk(&ctx, root.path(), replacement_id).await;
|
||||||
|
assert!(!Arc::ptr_eq(&old, &replacement));
|
||||||
|
assert!(matches!(
|
||||||
|
local_disk_candidate(&ctx, &old_id.to_string()).await,
|
||||||
|
Err(DiskError::DiskNotFound)
|
||||||
|
));
|
||||||
|
let (candidate, verified) = local_disk_candidate(&ctx, &replacement_id.to_string())
|
||||||
|
.await
|
||||||
|
.expect("replacement UUID");
|
||||||
|
assert!(Arc::ptr_eq(&candidate, &replacement));
|
||||||
|
assert_eq!(verified, Some(replacement_id));
|
||||||
|
assert!(!ctx.namespace_commits_pending());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bootstrap_rejects_user_volumes_aliases_and_scanner_tokens_without_mutation() {
|
||||||
|
let ctx = Arc::new(InstanceContext::new());
|
||||||
|
let root = tempfile::tempdir().expect("root");
|
||||||
|
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||||
|
let target = BootstrapLocalTarget::new(ctx.clone());
|
||||||
|
let fi = target_file_info("destination", Uuid::new_v4(), b"body");
|
||||||
|
let user_before = seed_target(&disk, "victim", "staged", fi.clone()).await;
|
||||||
|
let meta_before = seed_target(&disk, ".rustfs.sys/tmp", "staged", fi.clone()).await;
|
||||||
|
for invalid in [
|
||||||
|
"victim",
|
||||||
|
".rustfs.sys/../victim",
|
||||||
|
".rustfs.sys/./tmp",
|
||||||
|
".rustfs.sys/ .. /victim",
|
||||||
|
".rustfs.sys\\..\\victim",
|
||||||
|
".minio.sys/../victim",
|
||||||
|
] {
|
||||||
|
for (src, dst) in [(invalid, ".rustfs.sys/tmp"), (".rustfs.sys/tmp", invalid)] {
|
||||||
|
assert!(
|
||||||
|
target
|
||||||
|
.rename_local_data(&disk.endpoint().to_string(), (src, "staged"), &fi, (dst, "destination"), None)
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"src={src}, dst={dst}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
target
|
||||||
|
.undo_local_write(
|
||||||
|
&disk.endpoint().to_string(),
|
||||||
|
invalid,
|
||||||
|
"staged",
|
||||||
|
fi.clone(),
|
||||||
|
DeleteOptions {
|
||||||
|
undo_write: true,
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"{invalid}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
target
|
||||||
|
.rename_local_data(
|
||||||
|
&disk.endpoint().to_string(),
|
||||||
|
(".rustfs.sys/tmp", "staged"),
|
||||||
|
&fi,
|
||||||
|
(".rustfs.sys/tmp", "destination"),
|
||||||
|
Some(Uuid::new_v4())
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
tokio::fs::read(root.path().join("victim/staged/xl.meta"))
|
||||||
|
.await
|
||||||
|
.expect("user source"),
|
||||||
|
user_before
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
tokio::fs::read(root.path().join(".rustfs.sys/tmp/staged/xl.meta"))
|
||||||
|
.await
|
||||||
|
.expect("metadata source"),
|
||||||
|
meta_before
|
||||||
|
);
|
||||||
|
assert!(!root.path().join("victim/destination").exists());
|
||||||
|
assert!(!root.path().join(".rustfs.sys/tmp/destination").exists());
|
||||||
|
assert_eq!(ctx.namespace_commit_generation(), 0);
|
||||||
|
assert!(!ctx.namespace_commits_pending());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn bootstrap_allows_internal_multisegment_rename_without_namespace_owner() {
|
||||||
|
for volume in [".rustfs.sys/tmp", ".rustfs.sys/multipart", ".minio.sys/config"] {
|
||||||
|
let ctx = Arc::new(InstanceContext::new());
|
||||||
|
let root = tempfile::tempdir().expect("root");
|
||||||
|
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||||
|
let fi = target_file_info("destination", Uuid::new_v4(), b"internal-CAS-body");
|
||||||
|
seed_target(&disk, volume, "staged", fi.clone()).await;
|
||||||
|
BootstrapLocalTarget::new(ctx.clone())
|
||||||
|
.rename_local_data(&disk.endpoint().to_string(), (volume, "staged"), &fi, (volume, "destination"), None)
|
||||||
|
.await
|
||||||
|
.expect("legitimate bootstrap metadata write");
|
||||||
|
let read = disk
|
||||||
|
.read_version(
|
||||||
|
volume,
|
||||||
|
volume,
|
||||||
|
"destination",
|
||||||
|
&fi.version_id.expect("version").to_string(),
|
||||||
|
&crate::disk::ReadOptions {
|
||||||
|
read_data: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("read bootstrap result");
|
||||||
|
assert_eq!(read.data, fi.data);
|
||||||
|
assert_eq!(ctx.namespace_commit_generation(), 0);
|
||||||
|
assert!(!ctx.namespace_commits_pending());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn target_rename_cancellation_retains_real_namespace_and_scanner_owners() {
|
||||||
|
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||||
|
let ctx = Arc::new(InstanceContext::new());
|
||||||
|
let sibling = Arc::new(InstanceContext::new());
|
||||||
|
let root = tempfile::tempdir().expect("root");
|
||||||
|
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||||
|
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||||
|
let fi = target_file_info("destination", Uuid::new_v4(), b"physically-owned");
|
||||||
|
seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
|
||||||
|
let (token, _) = store
|
||||||
|
.acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL)
|
||||||
|
.await
|
||||||
|
.expect("real scanner token in A");
|
||||||
|
let destination = disk
|
||||||
|
.get_object_path_for_io_if_local("target-bucket", "destination/xl.meta")
|
||||||
|
.expect("local disk")
|
||||||
|
.expect("destination IO path");
|
||||||
|
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||||
|
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||||
|
let _hook = hooks::install(&destination, move || {
|
||||||
|
let _ = entered_tx.send(());
|
||||||
|
let _ = release_rx.recv();
|
||||||
|
});
|
||||||
|
let disk_ref = disk.endpoint().to_string();
|
||||||
|
let mut rename = Box::pin(store.rename_local_data(
|
||||||
|
&disk_ref,
|
||||||
|
("target-bucket", "staged"),
|
||||||
|
&fi,
|
||||||
|
("target-bucket", "destination"),
|
||||||
|
Some(token),
|
||||||
|
));
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||||
|
tokio::select! {
|
||||||
|
result = &mut rename => panic!("rename completed before physical pause: {result:?}"),
|
||||||
|
entered = entered_rx => entered.expect("physical rename entered"),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded physical entry");
|
||||||
|
drop(rename);
|
||||||
|
assert!(store.scanner_data_usage_publication_blocked().await);
|
||||||
|
assert!(ctx.namespace_commits_pending());
|
||||||
|
assert!(!sibling.namespace_commits_pending());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.rename_local_data(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "another"), Some(token))
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"real pending rename blocks another scanner publication"
|
||||||
|
);
|
||||||
|
assert!(store.release_scanner_publication_lease(token).await, "remove registered token");
|
||||||
|
let gate = ctx.data_movement_operation_gate();
|
||||||
|
assert!(
|
||||||
|
gate.clone().try_write_owned().is_err(),
|
||||||
|
"physical operation still owns the scanner read guard"
|
||||||
|
);
|
||||||
|
drop(release_tx);
|
||||||
|
let _drained = tokio::time::timeout(std::time::Duration::from_secs(10), gate.write_owned())
|
||||||
|
.await
|
||||||
|
.expect("physical tail must release scanner guard");
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||||
|
while ctx.namespace_commits_pending() {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("namespace owner drains");
|
||||||
|
let read = disk
|
||||||
|
.read_version(
|
||||||
|
"target-bucket",
|
||||||
|
"target-bucket",
|
||||||
|
"destination",
|
||||||
|
&fi.version_id.expect("version").to_string(),
|
||||||
|
&crate::disk::ReadOptions {
|
||||||
|
read_data: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("read actual late commit");
|
||||||
|
assert_eq!(read.data, fi.data);
|
||||||
|
assert!(ctx.namespace_commit_generation() >= 2);
|
||||||
|
assert_eq!(sibling.namespace_commit_generation(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn target_ready_rejects_unknown_foreign_released_and_expired_scanner_tokens() {
|
||||||
|
let ctx = Arc::new(InstanceContext::new());
|
||||||
|
let other = Arc::new(InstanceContext::new());
|
||||||
|
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||||
|
let other_store = super::super::tests::build_store_with_ctx(other);
|
||||||
|
let root = tempfile::tempdir().expect("root");
|
||||||
|
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||||
|
let fi = target_file_info("destination", Uuid::new_v4(), b"unchanged");
|
||||||
|
let before = seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
|
||||||
|
let ttl = crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL;
|
||||||
|
let (foreign, _) = other_store.acquire_scanner_publication_lease(0, ttl).await.expect("B token");
|
||||||
|
let (released, _) = store.acquire_scanner_publication_lease(0, ttl).await.expect("A token");
|
||||||
|
assert!(store.release_scanner_publication_lease(released).await);
|
||||||
|
let (valid, _) = store.acquire_scanner_publication_lease(0, ttl).await.expect("new A token");
|
||||||
|
for token in [Uuid::new_v4(), foreign, released] {
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.rename_local_data(
|
||||||
|
&disk.endpoint().to_string(),
|
||||||
|
("target-bucket", "staged"),
|
||||||
|
&fi,
|
||||||
|
("target-bucket", "destination"),
|
||||||
|
Some(token)
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
tokio::time::pause();
|
||||||
|
tokio::time::advance(ttl + std::time::Duration::from_secs(1)).await;
|
||||||
|
tokio::time::resume();
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.rename_local_data(
|
||||||
|
&disk.endpoint().to_string(),
|
||||||
|
("target-bucket", "staged"),
|
||||||
|
&fi,
|
||||||
|
("target-bucket", "destination"),
|
||||||
|
Some(valid)
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"expired real token"
|
||||||
|
);
|
||||||
|
let _ = other_store.release_scanner_publication_lease(foreign).await;
|
||||||
|
assert_eq!(
|
||||||
|
tokio::fs::read(root.path().join("target-bucket/staged/xl.meta"))
|
||||||
|
.await
|
||||||
|
.expect("source bytes"),
|
||||||
|
before
|
||||||
|
);
|
||||||
|
assert!(!root.path().join("target-bucket/destination").exists());
|
||||||
|
assert!(!ctx.namespace_commits_pending());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn target_ordinary_timeout_keeps_its_physical_namespace_owner() {
|
||||||
|
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||||
|
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("1"))], async {
|
||||||
|
let ctx = Arc::new(InstanceContext::new());
|
||||||
|
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||||
|
let root = tempfile::tempdir().expect("root");
|
||||||
|
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||||
|
let fi = target_file_info("destination", Uuid::new_v4(), b"timed-out-physical-commit");
|
||||||
|
seed_target(&disk, "target-bucket", "staged", fi.clone()).await;
|
||||||
|
let path = disk
|
||||||
|
.get_object_path_for_io_if_local("target-bucket", "destination/xl.meta")
|
||||||
|
.expect("local")
|
||||||
|
.expect("destination IO path");
|
||||||
|
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||||
|
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||||
|
let _hook = hooks::install(&path, move || {
|
||||||
|
let _ = entered_tx.send(());
|
||||||
|
let _ = release_rx.recv();
|
||||||
|
});
|
||||||
|
let disk_ref = disk.endpoint().to_string();
|
||||||
|
let mut rename = Box::pin(store.rename_local_data(
|
||||||
|
&disk_ref,
|
||||||
|
("target-bucket", "staged"),
|
||||||
|
&fi,
|
||||||
|
("target-bucket", "destination"),
|
||||||
|
None,
|
||||||
|
));
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||||
|
tokio::select! {
|
||||||
|
result = &mut rename => panic!("completed before physical pause: {result:?}"),
|
||||||
|
entered = entered_rx => entered.expect("physical entry"),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded entry");
|
||||||
|
tokio::time::pause();
|
||||||
|
tokio::time::advance(std::time::Duration::from_secs(2)).await;
|
||||||
|
tokio::time::resume();
|
||||||
|
let result = tokio::time::timeout(std::time::Duration::from_secs(5), &mut rename)
|
||||||
|
.await
|
||||||
|
.expect("ordinary deadline remains enabled");
|
||||||
|
assert!(matches!(result, Err(DiskError::Timeout)), "{result:?}");
|
||||||
|
drop(rename);
|
||||||
|
assert!(ctx.namespace_commits_pending(), "timeout is not a physical drain");
|
||||||
|
drop(release_tx);
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||||
|
while ctx.namespace_commits_pending() {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("late physical owner drains");
|
||||||
|
let read = disk
|
||||||
|
.read_version(
|
||||||
|
"target-bucket",
|
||||||
|
"target-bucket",
|
||||||
|
"destination",
|
||||||
|
&fi.version_id.expect("version").to_string(),
|
||||||
|
&crate::disk::ReadOptions {
|
||||||
|
read_data: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("read actual timeout tail");
|
||||||
|
assert_eq!(read.data, fi.data);
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn endpoint_rpc_authority_preserves_port_and_ipv6_brackets() {
|
fn endpoint_rpc_authority_preserves_port_and_ipv6_brackets() {
|
||||||
let endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint");
|
let endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint");
|
||||||
|
|||||||
@@ -26,13 +26,14 @@
|
|||||||
//! server is not ready rather than that another server's global context applies.
|
//! server is not ready rather than that another server's global context applies.
|
||||||
|
|
||||||
use super::global::{AppContext, get_global_app_context};
|
use super::global::{AppContext, get_global_app_context};
|
||||||
use crate::app::storage_api::context::ECStore;
|
use crate::app::storage_api::context::{BootstrapLocalTarget, ECStore, InstanceContext};
|
||||||
use std::sync::{Arc, OnceLock};
|
use std::sync::{Arc, OnceLock};
|
||||||
|
|
||||||
/// Late-bound, per-server handle to the application context.
|
/// Late-bound, per-server handle to the application context.
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
pub struct ServerContextSlot {
|
pub struct ServerContextSlot {
|
||||||
app_context: OnceLock<Arc<AppContext>>,
|
app_context: OnceLock<Arc<AppContext>>,
|
||||||
|
bootstrap_target: Option<BootstrapLocalTarget>,
|
||||||
heal_topology_fingerprint: Arc<tokio::sync::OnceCell<String>>,
|
heal_topology_fingerprint: Arc<tokio::sync::OnceCell<String>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -50,15 +51,47 @@ impl ServerContextSlot {
|
|||||||
pub fn new() -> Arc<Self> {
|
pub fn new() -> Arc<Self> {
|
||||||
Arc::new(Self {
|
Arc::new(Self {
|
||||||
app_context: OnceLock::new(),
|
app_context: OnceLock::new(),
|
||||||
|
bootstrap_target: None,
|
||||||
heal_topology_fingerprint: Arc::new(tokio::sync::OnceCell::new()),
|
heal_topology_fingerprint: Arc::new(tokio::sync::OnceCell::new()),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Bind the listener to its foundation before it can accept requests.
|
||||||
|
pub fn with_instance_context(ctx: Arc<InstanceContext>) -> Arc<Self> {
|
||||||
|
Arc::new(Self {
|
||||||
|
bootstrap_target: Some(BootstrapLocalTarget::new(ctx)),
|
||||||
|
..Self::default()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Install this server's application context (once). Returns `false` if
|
/// Install this server's application context (once). Returns `false` if
|
||||||
/// the slot was already installed; the first installation wins, matching
|
/// the slot was already installed; the first installation wins, matching
|
||||||
/// the process-global singleton's `get_or_init` semantics.
|
/// the process-global singleton's `get_or_init` semantics.
|
||||||
pub fn install(&self, context: Arc<AppContext>) -> bool {
|
pub fn install(&self, context: Arc<AppContext>) -> bool {
|
||||||
self.app_context.set(context).is_ok()
|
self.try_install(context).is_ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Claim the slot before any process-global application publication.
|
||||||
|
/// Repeated installation, even of the same Arc, is an explicit conflict.
|
||||||
|
pub fn try_install(&self, context: Arc<AppContext>) -> std::io::Result<()> {
|
||||||
|
if self
|
||||||
|
.bootstrap_target
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|target| !target.is_for_store(&context.object_store()))
|
||||||
|
{
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidInput,
|
||||||
|
"application context does not belong to this server foundation",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
self.app_context.set(context).map_err(|_| {
|
||||||
|
std::io::Error::new(std::io::ErrorKind::AlreadyExists, "server application context is already installed")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Immutable, restricted startup capability; never resolves an ambient store.
|
||||||
|
pub fn bootstrap_target(&self) -> Option<BootstrapLocalTarget> {
|
||||||
|
self.bootstrap_target.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// This server's installed application context, if startup has completed.
|
/// This server's installed application context, if startup has completed.
|
||||||
|
|||||||
@@ -37,8 +37,8 @@ impl AppContext {
|
|||||||
// also publishes to the process default (first server wins) so legacy
|
// also publishes to the process default (first server wins) so legacy
|
||||||
// free-function readers keep resolving the first server's context.
|
// free-function readers keep resolving the first server's context.
|
||||||
let context = Arc::new(AppContext::with_default_interfaces(store, iam, kms_interface));
|
let context = Arc::new(AppContext::with_default_interfaces(store, iam, kms_interface));
|
||||||
publish_global_app_context(context.clone());
|
server_ctx.try_install(context.clone())?;
|
||||||
let _ = server_ctx.install(context);
|
publish_global_app_context(context);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1259,7 +1259,7 @@ pub(crate) mod context {
|
|||||||
pub(crate) use super::EndpointServerPools;
|
pub(crate) use super::EndpointServerPools;
|
||||||
pub(crate) use super::bucket;
|
pub(crate) use super::bucket;
|
||||||
pub(crate) use super::runtime;
|
pub(crate) use super::runtime;
|
||||||
pub(crate) use crate::storage::storage_api::{ECStore, EndpointServerPools};
|
pub(crate) use crate::storage::storage_api::{BootstrapLocalTarget, ECStore, EndpointServerPools, InstanceContext};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use crate::storage::storage_api::{Endpoint, Endpoints, PoolEndpoints};
|
pub(crate) use crate::storage::storage_api::{Endpoint, Endpoints, PoolEndpoints};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -36,7 +36,9 @@ use crate::server::{
|
|||||||
};
|
};
|
||||||
use crate::storage_api::server::http as storage;
|
use crate::storage_api::server::http as storage;
|
||||||
use crate::storage_api::server::http::rpc::InternodeRpcService;
|
use crate::storage_api::server::http::rpc::InternodeRpcService;
|
||||||
|
#[cfg(test)]
|
||||||
use crate::storage_api::server::http::tonic_service::make_server;
|
use crate::storage_api::server::http::tonic_service::make_server;
|
||||||
|
use crate::storage_api::server::http::tonic_service::make_server_for_slot;
|
||||||
use crate::storage_api::server::http::{
|
use crate::storage_api::server::http::{
|
||||||
ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge,
|
ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge,
|
||||||
tonic_boot_epoch_response_headers, verify_tonic_rpc_signature_with_bootstrap,
|
tonic_boot_epoch_response_headers, verify_tonic_rpc_signature_with_bootstrap,
|
||||||
@@ -1834,7 +1836,7 @@ fn process_connection(
|
|||||||
// each service in the auth interceptor.
|
// each service in the auth interceptor.
|
||||||
let rpc_max_message_size = rustfs_protos::internode_rpc_max_message_size();
|
let rpc_max_message_size = rustfs_protos::internode_rpc_max_message_size();
|
||||||
let node_service = InterceptedService::new(
|
let node_service = InterceptedService::new(
|
||||||
NodeServiceServer::new(make_server())
|
NodeServiceServer::new(make_server_for_slot(Arc::clone(&server_ctx)))
|
||||||
.max_decoding_message_size(rpc_max_message_size)
|
.max_decoding_message_size(rpc_max_message_size)
|
||||||
.max_encoding_message_size(rpc_max_message_size),
|
.max_encoding_message_size(rpc_max_message_size),
|
||||||
check_auth,
|
check_auth,
|
||||||
|
|||||||
@@ -124,9 +124,6 @@ pub(crate) async fn run_embedded_startup(args: EmbeddedStartupArgs) -> Result<Em
|
|||||||
} else {
|
} else {
|
||||||
bootstrap_instance_ctx()
|
bootstrap_instance_ctx()
|
||||||
};
|
};
|
||||||
// This server's request-path context slot (backlog#1052 S2).
|
|
||||||
let server_ctx = ServerContextSlot::new();
|
|
||||||
|
|
||||||
let EmbeddedStartupConfig {
|
let EmbeddedStartupConfig {
|
||||||
config,
|
config,
|
||||||
identity,
|
identity,
|
||||||
@@ -151,6 +148,7 @@ pub(crate) async fn run_embedded_startup(args: EmbeddedStartupArgs) -> Result<Em
|
|||||||
.await
|
.await
|
||||||
.map_err(init_error)?;
|
.map_err(init_error)?;
|
||||||
|
|
||||||
|
let server_ctx = ServerContextSlot::with_instance_context(instance_ctx.clone());
|
||||||
let http_server = start_embedded_http_server(&config, listen_context.readiness.clone(), server_ctx.clone()).await?;
|
let http_server = start_embedded_http_server(&config, listen_context.readiness.clone(), server_ctx.clone()).await?;
|
||||||
let shutdown_handle = http_server.shutdown_handle;
|
let shutdown_handle = http_server.shutdown_handle;
|
||||||
let bound_addr = http_server.bound_addr;
|
let bound_addr = http_server.bound_addr;
|
||||||
|
|||||||
@@ -141,10 +141,6 @@ async fn run(config: Config) -> Result<()> {
|
|||||||
// the storage path explicitly (Phase 5 follow-up, backlog#1052); a future
|
// the storage path explicitly (Phase 5 follow-up, backlog#1052); a future
|
||||||
// multi-instance server constructs its own context here instead.
|
// multi-instance server constructs its own context here instead.
|
||||||
let instance_ctx = bootstrap_instance_ctx();
|
let instance_ctx = bootstrap_instance_ctx();
|
||||||
// This server's request-path context slot (backlog#1052 S2): handed to the
|
|
||||||
// HTTP service now, installed once IAM bootstrap completes.
|
|
||||||
let server_ctx = ServerContextSlot::new();
|
|
||||||
|
|
||||||
let StartupListenContext {
|
let StartupListenContext {
|
||||||
readiness,
|
readiness,
|
||||||
server_addr,
|
server_addr,
|
||||||
@@ -152,6 +148,7 @@ async fn run(config: Config) -> Result<()> {
|
|||||||
} = init_startup_listen_context(&config, &instance_ctx).await?;
|
} = init_startup_listen_context(&config, &instance_ctx).await?;
|
||||||
|
|
||||||
let endpoint_pools = init_startup_storage_foundation(&server_address, &config.volumes, &instance_ctx).await?;
|
let endpoint_pools = init_startup_storage_foundation(&server_address, &config.volumes, &instance_ctx).await?;
|
||||||
|
let server_ctx = ServerContextSlot::with_instance_context(instance_ctx.clone());
|
||||||
let StartupHttpServers {
|
let StartupHttpServers {
|
||||||
state_manager,
|
state_manager,
|
||||||
s3_shutdown_tx,
|
s3_shutdown_tx,
|
||||||
|
|||||||
@@ -28,9 +28,9 @@ use crate::storage::storage_api::rpc_consumer::node_service::{
|
|||||||
SCANNER_PUBLICATION_LEASE_TTL_MS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _,
|
SCANNER_PUBLICATION_LEASE_TTL_MS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _,
|
||||||
StorageResult, all_local_disk_path, find_local_disk_by_ref, reload_transition_tier_config,
|
StorageResult, all_local_disk_path, find_local_disk_by_ref, reload_transition_tier_config,
|
||||||
};
|
};
|
||||||
use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, runtime_sources};
|
use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, ServerContextSlot, runtime_sources};
|
||||||
use crate::storage::storage_api::{
|
use crate::storage::storage_api::{
|
||||||
sign_tonic_rpc_response_proof, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
|
BootstrapLocalTarget, sign_tonic_rpc_response_proof, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
|
||||||
verify_tonic_mutation_body_digest_reject_unsigned,
|
verify_tonic_mutation_body_digest_reject_unsigned,
|
||||||
};
|
};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
@@ -482,6 +482,13 @@ mod metrics;
|
|||||||
pub struct NodeService {
|
pub struct NodeService {
|
||||||
local_peer: LocalPeerS3Client,
|
local_peer: LocalPeerS3Client,
|
||||||
context: Option<Arc<runtime_sources::AppContext>>,
|
context: Option<Arc<runtime_sources::AppContext>>,
|
||||||
|
server_ctx: Option<Arc<ServerContextSlot>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
enum LocalMutationTarget {
|
||||||
|
Ready(Arc<ECStore>),
|
||||||
|
Bootstrap(BootstrapLocalTarget),
|
||||||
|
Unbound,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for NodeService {
|
impl std::fmt::Debug for NodeService {
|
||||||
@@ -507,7 +514,19 @@ pub fn make_server() -> NodeService {
|
|||||||
|
|
||||||
pub fn make_server_for_context(context: Option<Arc<runtime_sources::AppContext>>) -> NodeService {
|
pub fn make_server_for_context(context: Option<Arc<runtime_sources::AppContext>>) -> NodeService {
|
||||||
let local_peer = LocalPeerS3Client::new(None, None);
|
let local_peer = LocalPeerS3Client::new(None, None);
|
||||||
NodeService { local_peer, context }
|
NodeService {
|
||||||
|
local_peer,
|
||||||
|
context,
|
||||||
|
server_ctx: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn make_server_for_slot(server_ctx: Arc<ServerContextSlot>) -> NodeService {
|
||||||
|
// Unrelated RPCs retain their existing context policy. Target mutations
|
||||||
|
// resolve exclusively through this listener slot on each request.
|
||||||
|
let mut service = make_server();
|
||||||
|
service.server_ctx = Some(server_ctx);
|
||||||
|
service
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default)]
|
#[derive(Clone, Debug, Default)]
|
||||||
@@ -1074,6 +1093,24 @@ impl heal_control_service_server::HealControlService for HealControlRpcService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl NodeService {
|
impl NodeService {
|
||||||
|
fn local_mutation_target(&self) -> LocalMutationTarget {
|
||||||
|
if let Some(slot) = &self.server_ctx {
|
||||||
|
// Capture exactly once per request, not at connection acceptance.
|
||||||
|
// A captured Bootstrap request cannot upgrade across a later await.
|
||||||
|
if let Some(store) = slot.installed_object_store() {
|
||||||
|
LocalMutationTarget::Ready(store)
|
||||||
|
} else if let Some(target) = slot.bootstrap_target() {
|
||||||
|
LocalMutationTarget::Bootstrap(target)
|
||||||
|
} else {
|
||||||
|
LocalMutationTarget::Unbound
|
||||||
|
}
|
||||||
|
} else if let Some(context) = &self.context {
|
||||||
|
LocalMutationTarget::Ready(context.object_store())
|
||||||
|
} else {
|
||||||
|
LocalMutationTarget::Unbound
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn resolve_object_store(&self) -> Option<Arc<ECStore>> {
|
fn resolve_object_store(&self) -> Option<Arc<ECStore>> {
|
||||||
let context = self.context.clone().or_else(runtime_sources::current_app_context);
|
let context = self.context.clone().or_else(runtime_sources::current_app_context);
|
||||||
runtime_sources::current_object_store_handle_for_context(context.as_deref())
|
runtime_sources::current_object_store_handle_for_context(context.as_deref())
|
||||||
@@ -2680,6 +2717,7 @@ mod tests {
|
|||||||
validate_admin_heal_control_start,
|
validate_admin_heal_control_start,
|
||||||
};
|
};
|
||||||
use crate::storage::rpc::node_service::heal::heal_topology_fingerprint;
|
use crate::storage::rpc::node_service::heal::heal_topology_fingerprint;
|
||||||
|
use crate::storage::storage_api::ecstore_disk::DiskAPI as _;
|
||||||
use crate::storage::storage_api::rpc_consumer::node_service::{DiskError, HealBucketInfo};
|
use crate::storage::storage_api::rpc_consumer::node_service::{DiskError, HealBucketInfo};
|
||||||
use crate::storage::storage_api::set_tonic_canonical_body_digest;
|
use crate::storage::storage_api::set_tonic_canonical_body_digest;
|
||||||
use crate::storage::storage_api::{
|
use crate::storage::storage_api::{
|
||||||
@@ -4530,6 +4568,411 @@ mod tests {
|
|||||||
assert!(rename_response.error.is_some());
|
assert!(rename_response.error.is_some());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct TargetRpcFixture {
|
||||||
|
_root: tempfile::TempDir,
|
||||||
|
env: rustfs_test_utils::TestECStoreEnv,
|
||||||
|
instance: Arc<crate::storage::storage_api::InstanceContext>,
|
||||||
|
context: Arc<crate::runtime_sources::AppContext>,
|
||||||
|
iam: Arc<rustfs_iam::sys::IamSys<ObjectStore>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn target_rpc_fixture() -> TargetRpcFixture {
|
||||||
|
super::timeout(Duration::from_secs(90), async {
|
||||||
|
let root = tempfile::tempdir().expect("target RPC root");
|
||||||
|
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||||
|
.base_dir(root.path())
|
||||||
|
.init_bucket_metadata(false)
|
||||||
|
.build()
|
||||||
|
.await;
|
||||||
|
ObjectStore::new(env.ecstore.clone())
|
||||||
|
.save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
|
||||||
|
.await
|
||||||
|
.expect("seed real IAM format");
|
||||||
|
let iam = rustfs_iam::build_iam_sys(env.ecstore.clone())
|
||||||
|
.await
|
||||||
|
.expect("build fixture IAM");
|
||||||
|
let context = Arc::new(crate::runtime_sources::AppContext::with_default_interfaces(
|
||||||
|
env.ecstore.clone(),
|
||||||
|
iam.clone(),
|
||||||
|
Arc::new(KmsServiceManager::new()),
|
||||||
|
));
|
||||||
|
let instance = crate::storage::storage_api::bootstrap_instance_ctx();
|
||||||
|
assert!(
|
||||||
|
super::BootstrapLocalTarget::new(instance.clone()).is_for_store(&env.ecstore),
|
||||||
|
"the standard builder must use this exact instance context"
|
||||||
|
);
|
||||||
|
super::timeout(Duration::from_secs(10), async {
|
||||||
|
while env.ecstore.scanner_data_usage_publication_blocked().await {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("startup namespace commits drain before test");
|
||||||
|
TargetRpcFixture {
|
||||||
|
_root: root,
|
||||||
|
env,
|
||||||
|
instance,
|
||||||
|
context,
|
||||||
|
iam,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded real fixture initialization")
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn stage_target_rpc(fixture: &TargetRpcFixture) -> (super::DiskStore, rustfs_filemeta::FileInfo, Vec<u8>) {
|
||||||
|
use crate::storage::storage_api::ecstore_disk::{DiskAPI, ReadOptions};
|
||||||
|
let disk = fixture
|
||||||
|
.instance
|
||||||
|
.local_disk_map()
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.values()
|
||||||
|
.find_map(Clone::clone)
|
||||||
|
.expect("local target");
|
||||||
|
let mut fi = rustfs_filemeta::FileInfo::new("destination", 1, 0);
|
||||||
|
fi.erasure.index = 1;
|
||||||
|
fi.version_id = Some(Uuid::new_v4());
|
||||||
|
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||||
|
fi.size = 17;
|
||||||
|
fi.parts = vec![rustfs_filemeta::ObjectPartInfo {
|
||||||
|
number: 1,
|
||||||
|
size: 17,
|
||||||
|
actual_size: 17,
|
||||||
|
..Default::default()
|
||||||
|
}];
|
||||||
|
fi.data = Some(Bytes::from_static(b"target-rpc-inline"));
|
||||||
|
fi.set_inline_data();
|
||||||
|
disk.make_volume("target-rpc").await.expect("target volume");
|
||||||
|
disk.write_metadata("target-rpc", "target-rpc", "staged", fi.clone())
|
||||||
|
.await
|
||||||
|
.expect("stage real inline body");
|
||||||
|
let read = disk
|
||||||
|
.read_version(
|
||||||
|
"target-rpc",
|
||||||
|
"target-rpc",
|
||||||
|
"staged",
|
||||||
|
&fi.version_id.expect("version").to_string(),
|
||||||
|
&ReadOptions {
|
||||||
|
read_data: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("read staged body before mutation");
|
||||||
|
assert_eq!(read.data, fi.data);
|
||||||
|
let before = tokio::fs::read(disk.path().join("target-rpc/staged/xl.meta"))
|
||||||
|
.await
|
||||||
|
.expect("staged bytes");
|
||||||
|
(disk, fi, before)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn target_rename_request(disk: &super::DiskStore, fi: &rustfs_filemeta::FileInfo) -> Request<RenameDataRequest> {
|
||||||
|
let mut request = Request::new(RenameDataRequest {
|
||||||
|
disk: disk.endpoint().to_string(),
|
||||||
|
src_volume: "target-rpc".to_string(),
|
||||||
|
src_path: "staged".to_string(),
|
||||||
|
dst_volume: "target-rpc".to_string(),
|
||||||
|
dst_path: "destination".to_string(),
|
||||||
|
file_info: serde_json::to_string(fi).expect("real FileInfo JSON"),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let body = rustfs_protos::canonical_rename_data_request_body(request.get_ref()).expect("canonical target body");
|
||||||
|
set_tonic_canonical_body_digest(&mut request, &body).expect("body digest");
|
||||||
|
// Direct-handler precondition only; this does not stand in for wire authentication.
|
||||||
|
mark_v2_authenticated(&mut request);
|
||||||
|
request
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn target_slot_rejects_mismatched_and_repeated_install_before_global_publication() {
|
||||||
|
let fixture = target_rpc_fixture().await;
|
||||||
|
assert!(
|
||||||
|
crate::runtime_sources::current_app_context().is_none(),
|
||||||
|
"requires a separate nextest process"
|
||||||
|
);
|
||||||
|
let wrong = super::ServerContextSlot::with_instance_context(crate::storage::storage_api::new_instance_ctx());
|
||||||
|
let error = crate::runtime_sources::AppContext::ensure_startup_after_iam(
|
||||||
|
fixture.env.ecstore.clone(),
|
||||||
|
Arc::new(KmsServiceManager::new()),
|
||||||
|
&wrong,
|
||||||
|
fixture.iam.clone(),
|
||||||
|
)
|
||||||
|
.expect_err("mismatched startup must fail");
|
||||||
|
assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput);
|
||||||
|
assert!(wrong.installed_app_context().is_none());
|
||||||
|
assert!(
|
||||||
|
crate::runtime_sources::current_app_context().is_none(),
|
||||||
|
"failed install must not publish globally"
|
||||||
|
);
|
||||||
|
assert!(!wrong.install(fixture.context.clone()), "bool adapter cannot bypass identity checks");
|
||||||
|
let slot = super::ServerContextSlot::with_instance_context(fixture.instance.clone());
|
||||||
|
crate::runtime_sources::AppContext::ensure_startup_after_iam(
|
||||||
|
fixture.env.ecstore.clone(),
|
||||||
|
Arc::new(KmsServiceManager::new()),
|
||||||
|
&slot,
|
||||||
|
fixture.iam.clone(),
|
||||||
|
)
|
||||||
|
.expect("matching startup installation");
|
||||||
|
let installed = slot.installed_app_context().expect("installed A");
|
||||||
|
assert!(Arc::ptr_eq(
|
||||||
|
&crate::runtime_sources::current_app_context().expect("published A"),
|
||||||
|
&installed
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
slot.try_install(installed.clone())
|
||||||
|
.expect_err("same Arc is still a duplicate")
|
||||||
|
.kind(),
|
||||||
|
std::io::ErrorKind::AlreadyExists
|
||||||
|
);
|
||||||
|
assert!(!slot.install(installed.clone()));
|
||||||
|
assert!(Arc::ptr_eq(&slot.installed_app_context().expect("first winner retained"), &installed));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn target_slot_captures_bootstrap_once_and_next_request_observes_ready() {
|
||||||
|
let fixture = target_rpc_fixture().await;
|
||||||
|
let (disk, fi, before) = stage_target_rpc(&fixture).await;
|
||||||
|
let slot = super::ServerContextSlot::with_instance_context(fixture.instance.clone());
|
||||||
|
let service = super::make_server_for_slot(slot.clone());
|
||||||
|
let captured = service.local_mutation_target();
|
||||||
|
slot.try_install(fixture.context.clone())
|
||||||
|
.expect("install after the request captures bootstrap");
|
||||||
|
let super::LocalMutationTarget::Bootstrap(target) = captured else {
|
||||||
|
panic!("pre-install request must capture bootstrap");
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
target
|
||||||
|
.rename_local_data(
|
||||||
|
&disk.endpoint().to_string(),
|
||||||
|
("target-rpc", "staged"),
|
||||||
|
&fi,
|
||||||
|
("target-rpc", "destination"),
|
||||||
|
None
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"captured request cannot acquire Ready privileges"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
tokio::fs::read(disk.path().join("target-rpc/staged/xl.meta"))
|
||||||
|
.await
|
||||||
|
.expect("original source"),
|
||||||
|
before
|
||||||
|
);
|
||||||
|
assert!(!disk.path().join("target-rpc/destination").exists());
|
||||||
|
assert!(
|
||||||
|
matches!(service.local_mutation_target(), super::LocalMutationTarget::Ready(_)),
|
||||||
|
"the same service must read the installed slot for its next request"
|
||||||
|
);
|
||||||
|
let result = service
|
||||||
|
.rename_data(target_rename_request(&disk, &fi))
|
||||||
|
.await
|
||||||
|
.expect("ready handler")
|
||||||
|
.into_inner();
|
||||||
|
assert!(result.success, "{:?}", result.error);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn target_unbound_slot_never_mutates_a_published_global_store() {
|
||||||
|
let fixture = target_rpc_fixture().await;
|
||||||
|
let (disk, fi, before) = stage_target_rpc(&fixture).await;
|
||||||
|
let published = crate::runtime_sources::publish_test_app_context(fixture.context.clone());
|
||||||
|
assert!(Arc::ptr_eq(&published, &fixture.context));
|
||||||
|
let service = super::make_server_for_slot(super::ServerContextSlot::new());
|
||||||
|
let result = service
|
||||||
|
.rename_data(target_rename_request(&disk, &fi))
|
||||||
|
.await
|
||||||
|
.expect("handler reply")
|
||||||
|
.into_inner();
|
||||||
|
assert!(!result.success);
|
||||||
|
assert!(result.error.is_some());
|
||||||
|
assert_eq!(
|
||||||
|
tokio::fs::read(disk.path().join("target-rpc/staged/xl.meta"))
|
||||||
|
.await
|
||||||
|
.expect("source remains"),
|
||||||
|
before
|
||||||
|
);
|
||||||
|
assert!(!disk.path().join("target-rpc/destination").exists());
|
||||||
|
assert!(!fixture.env.ecstore.scanner_data_usage_publication_blocked().await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn target_undo_rejects_force_delete_marker_before_mutation() {
|
||||||
|
let fixture = target_rpc_fixture().await;
|
||||||
|
let (disk, fi, before) = stage_target_rpc(&fixture).await;
|
||||||
|
let service = make_server_for_context(Some(fixture.context.clone()));
|
||||||
|
let opts = crate::storage::storage_api::ecstore_disk::DeleteOptions {
|
||||||
|
undo_write: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let mut request = Request::new(DeleteVersionRequest {
|
||||||
|
disk: disk.endpoint().to_string(),
|
||||||
|
volume: "target-rpc".to_string(),
|
||||||
|
path: "staged".to_string(),
|
||||||
|
file_info: serde_json::to_string(&fi).expect("FileInfo"),
|
||||||
|
opts: serde_json::to_string(&opts).expect("opts"),
|
||||||
|
force_del_marker: true,
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let body = rustfs_protos::canonical_delete_version_request_body(request.get_ref()).expect("canonical undo body");
|
||||||
|
set_tonic_canonical_body_digest(&mut request, &body).expect("body digest");
|
||||||
|
mark_v2_authenticated(&mut request);
|
||||||
|
let result = service.delete_version(request).await.expect("handler reply").into_inner();
|
||||||
|
assert!(!result.success);
|
||||||
|
assert!(result.error.is_some());
|
||||||
|
assert_eq!(
|
||||||
|
tokio::fs::read(disk.path().join("target-rpc/staged/xl.meta"))
|
||||||
|
.await
|
||||||
|
.expect("source remains"),
|
||||||
|
before
|
||||||
|
);
|
||||||
|
assert!(!fixture.env.ecstore.scanner_data_usage_publication_blocked().await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn target_handler_cancellation_retains_namespace_through_physical_rename() {
|
||||||
|
use crate::storage::storage_api::{
|
||||||
|
LocalPublicationPause, LocalPublicationStage,
|
||||||
|
ecstore_disk::{DiskAPI, ReadOptions},
|
||||||
|
};
|
||||||
|
let fixture = target_rpc_fixture().await;
|
||||||
|
let (disk, fi, _) = stage_target_rpc(&fixture).await;
|
||||||
|
let slot = super::ServerContextSlot::with_instance_context(fixture.instance.clone());
|
||||||
|
slot.try_install(fixture.context.clone()).expect("ready target");
|
||||||
|
let service = super::make_server_for_slot(slot);
|
||||||
|
let mut pause =
|
||||||
|
LocalPublicationPause::install(&disk, "target-rpc", "destination/xl.meta", LocalPublicationStage::PreparedRename)
|
||||||
|
.expect("install scoped physical pause");
|
||||||
|
let mut handler = Box::pin(service.rename_data(target_rename_request(&disk, &fi)));
|
||||||
|
super::timeout(Duration::from_secs(10), async {
|
||||||
|
tokio::select! {
|
||||||
|
result = &mut handler => panic!("handler completed before physical entry: {result:?}"),
|
||||||
|
entered = pause.entered() => entered.expect("physical executor entered"),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded physical entry");
|
||||||
|
drop(handler);
|
||||||
|
assert!(
|
||||||
|
fixture.env.ecstore.scanner_data_usage_publication_blocked().await,
|
||||||
|
"dropping the actual target handler must not release its physical owner"
|
||||||
|
);
|
||||||
|
drop(pause);
|
||||||
|
super::timeout(Duration::from_secs(10), async {
|
||||||
|
while fixture.env.ecstore.scanner_data_usage_publication_blocked().await {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("physical owner must drain");
|
||||||
|
let read = disk
|
||||||
|
.read_version(
|
||||||
|
"target-rpc",
|
||||||
|
"target-rpc",
|
||||||
|
"destination",
|
||||||
|
&fi.version_id.expect("version").to_string(),
|
||||||
|
&ReadOptions {
|
||||||
|
read_data: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("read real late commit");
|
||||||
|
assert_eq!(read.data, fi.data);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(windows))]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn target_undo_handler_cancellation_retains_owner_until_backup_restoration() {
|
||||||
|
use crate::storage::storage_api::{
|
||||||
|
LocalPublicationPause, LocalPublicationStage,
|
||||||
|
ecstore_disk::{DeleteOptions, DiskAPI, ReadOptions},
|
||||||
|
};
|
||||||
|
let fixture = target_rpc_fixture().await;
|
||||||
|
let (disk, fi, _) = stage_target_rpc(&fixture).await;
|
||||||
|
let mut old = fi.clone();
|
||||||
|
old.data = Some(Bytes::from_static(b"previous-rpc-body"));
|
||||||
|
assert_eq!(old.data.as_ref().expect("old body").len(), 17);
|
||||||
|
disk.write_metadata("target-rpc", "target-rpc", "destination", old.clone())
|
||||||
|
.await
|
||||||
|
.expect("old actual version");
|
||||||
|
let old_bytes = tokio::fs::read(disk.path().join("target-rpc/destination/xl.meta"))
|
||||||
|
.await
|
||||||
|
.expect("old metadata bytes");
|
||||||
|
let committed = fixture
|
||||||
|
.env
|
||||||
|
.ecstore
|
||||||
|
.rename_local_data(
|
||||||
|
&disk.endpoint().to_string(),
|
||||||
|
("target-rpc", "staged"),
|
||||||
|
&fi,
|
||||||
|
("target-rpc", "destination"),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("real overwrite creates rollback backup");
|
||||||
|
let opts = DeleteOptions {
|
||||||
|
undo_write: true,
|
||||||
|
old_data_dir: Some(committed.rollback_data_dir.expect("real rollback backup")),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let service = make_server_for_context(Some(fixture.context.clone()));
|
||||||
|
let mut request = Request::new(DeleteVersionRequest {
|
||||||
|
disk: disk.endpoint().to_string(),
|
||||||
|
volume: "target-rpc".to_string(),
|
||||||
|
path: "destination".to_string(),
|
||||||
|
file_info: serde_json::to_string(&fi).expect("FileInfo"),
|
||||||
|
opts: serde_json::to_string(&opts).expect("undo options"),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
let body = rustfs_protos::canonical_delete_version_request_body(request.get_ref()).expect("canonical undo body");
|
||||||
|
set_tonic_canonical_body_digest(&mut request, &body).expect("body digest");
|
||||||
|
mark_v2_authenticated(&mut request);
|
||||||
|
let mut pause = LocalPublicationPause::install(&disk, "target-rpc", "destination/xl.meta", LocalPublicationStage::Rename)
|
||||||
|
.expect("pause actual backup restoration");
|
||||||
|
let mut handler = Box::pin(service.delete_version(request));
|
||||||
|
super::timeout(Duration::from_secs(10), async {
|
||||||
|
tokio::select! {
|
||||||
|
result = &mut handler => panic!("undo completed before physical entry: {result:?}"),
|
||||||
|
entered = pause.entered() => entered.expect("physical restore entered"),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("bounded physical restore entry");
|
||||||
|
drop(handler);
|
||||||
|
assert!(fixture.env.ecstore.scanner_data_usage_publication_blocked().await);
|
||||||
|
drop(pause);
|
||||||
|
super::timeout(Duration::from_secs(10), async {
|
||||||
|
while fixture.env.ecstore.scanner_data_usage_publication_blocked().await {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("restore owner drains");
|
||||||
|
assert_eq!(
|
||||||
|
tokio::fs::read(disk.path().join("target-rpc/destination/xl.meta"))
|
||||||
|
.await
|
||||||
|
.expect("restored bytes"),
|
||||||
|
old_bytes
|
||||||
|
);
|
||||||
|
let read = disk
|
||||||
|
.read_version(
|
||||||
|
"target-rpc",
|
||||||
|
"target-rpc",
|
||||||
|
"destination",
|
||||||
|
&fi.version_id.expect("version").to_string(),
|
||||||
|
&ReadOptions {
|
||||||
|
read_data: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("restored readable version");
|
||||||
|
assert_eq!(read.data, old.data);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn rename_data_same_uuid_uses_captured_instance_instead_of_global_disk() {
|
async fn rename_data_same_uuid_uses_captured_instance_instead_of_global_disk() {
|
||||||
use crate::storage::storage_api::{
|
use crate::storage::storage_api::{
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use super::NodeService;
|
use super::{LocalMutationTarget, NodeService};
|
||||||
use crate::storage::storage_api::rpc_consumer::node_service::{
|
use crate::storage::storage_api::rpc_consumer::node_service::{
|
||||||
BatchReadVersionReq, BatchReadVersionResp, DeleteOptions, DiskError, DiskInfoOptions, FileInfoVersions, ReadMultipleReq,
|
BatchReadVersionReq, BatchReadVersionResp, DeleteOptions, DiskError, DiskInfoOptions, FileInfoVersions, ReadMultipleReq,
|
||||||
ReadMultipleResp, ReadOptions, StorageDiskRpcExt as _, UpdateMetadataOpts, validate_batch_read_version_item_count,
|
ReadMultipleResp, ReadOptions, StorageDiskRpcExt as _, UpdateMetadataOpts, validate_batch_read_version_item_count,
|
||||||
@@ -39,6 +39,46 @@ use tonic::{Request, Response, Status};
|
|||||||
use tracing::debug;
|
use tracing::debug;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
impl LocalMutationTarget {
|
||||||
|
async fn rename_local_data(
|
||||||
|
&self,
|
||||||
|
disk_ref: &str,
|
||||||
|
source: (&str, &str),
|
||||||
|
fi: &FileInfo,
|
||||||
|
destination: (&str, &str),
|
||||||
|
scanner_token: Option<Uuid>,
|
||||||
|
) -> Result<RenameDataResp, DiskError> {
|
||||||
|
match self {
|
||||||
|
Self::Ready(store) => {
|
||||||
|
store
|
||||||
|
.rename_local_data(disk_ref, source, fi, destination, scanner_token)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
Self::Bootstrap(target) => {
|
||||||
|
target
|
||||||
|
.rename_local_data(disk_ref, source, fi, destination, scanner_token)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
Self::Unbound => Err(DiskError::other("target disk instance is unavailable")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn undo_local_write(
|
||||||
|
&self,
|
||||||
|
disk_ref: &str,
|
||||||
|
volume: &str,
|
||||||
|
path: &str,
|
||||||
|
fi: FileInfo,
|
||||||
|
opts: DeleteOptions,
|
||||||
|
) -> Result<(), DiskError> {
|
||||||
|
match self {
|
||||||
|
Self::Ready(store) => store.undo_local_write(disk_ref, volume, path, fi, opts).await,
|
||||||
|
Self::Bootstrap(target) => target.undo_local_write(disk_ref, volume, path, fi, opts).await,
|
||||||
|
Self::Unbound => Err(DiskError::other("target disk instance is unavailable")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Initial capacity hint (bytes) for typical small msgpack requests and responses.
|
/// Initial capacity hint (bytes) for typical small msgpack requests and responses.
|
||||||
const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512;
|
const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512;
|
||||||
const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024;
|
const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024;
|
||||||
@@ -670,55 +710,59 @@ impl NodeService {
|
|||||||
"delete_version",
|
"delete_version",
|
||||||
)?;
|
)?;
|
||||||
let request = request.into_inner();
|
let request = request.into_inner();
|
||||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
let file_info = match decode_msgpack_or_json::<FileInfo>(&request.file_info_bin, &request.file_info, "FileInfo") {
|
||||||
let file_info = match decode_msgpack_or_json::<FileInfo>(&request.file_info_bin, &request.file_info, "FileInfo") {
|
Ok(file_info) => file_info,
|
||||||
Ok(file_info) => file_info,
|
Err(err) => {
|
||||||
Err(err) => {
|
return Ok(Response::new(DeleteVersionResponse {
|
||||||
return Ok(Response::new(DeleteVersionResponse {
|
success: false,
|
||||||
success: false,
|
raw_file_info: "".to_string(),
|
||||||
raw_file_info: "".to_string(),
|
error: Some(DiskError::other(format!("decode FileInfo failed: {err}")).into()),
|
||||||
error: Some(DiskError::other(format!("decode FileInfo failed: {err}")).into()),
|
}));
|
||||||
}));
|
}
|
||||||
}
|
};
|
||||||
};
|
let opts = match decode_msgpack_or_json::<DeleteOptions>(&request.opts_bin, &request.opts, "DeleteOptions") {
|
||||||
let opts = match decode_msgpack_or_json::<DeleteOptions>(&request.opts_bin, &request.opts, "DeleteOptions") {
|
Ok(opts) => opts,
|
||||||
Ok(opts) => opts,
|
Err(err) => {
|
||||||
Err(err) => {
|
return Ok(Response::new(DeleteVersionResponse {
|
||||||
return Ok(Response::new(DeleteVersionResponse {
|
success: false,
|
||||||
success: false,
|
raw_file_info: "".to_string(),
|
||||||
raw_file_info: "".to_string(),
|
error: Some(DiskError::other(format!("decode DeleteOptions failed: {err}")).into()),
|
||||||
error: Some(DiskError::other(format!("decode DeleteOptions failed: {err}")).into()),
|
}));
|
||||||
}));
|
}
|
||||||
}
|
};
|
||||||
};
|
let result = if opts.undo_write {
|
||||||
match disk
|
if request.force_del_marker {
|
||||||
.delete_version(&request.volume, &request.path, file_info, request.force_del_marker, opts)
|
Err(DiskError::other("undo_write cannot force a delete marker"))
|
||||||
|
} else {
|
||||||
|
let target = self.local_mutation_target();
|
||||||
|
target
|
||||||
|
.undo_local_write(&request.disk, &request.volume, &request.path, file_info, opts)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
} else if let Some(disk) = self.find_disk(&request.disk).await {
|
||||||
|
disk.delete_version(&request.volume, &request.path, file_info, request.force_del_marker, opts)
|
||||||
.await
|
.await
|
||||||
{
|
} else {
|
||||||
Ok(raw_file_info) => match serde_json::to_string(&raw_file_info) {
|
Err(DiskError::other("cannot find disk"))
|
||||||
Ok(raw_file_info) => Ok(Response::new(DeleteVersionResponse {
|
};
|
||||||
success: true,
|
match result {
|
||||||
raw_file_info,
|
Ok(raw_file_info) => match serde_json::to_string(&raw_file_info) {
|
||||||
error: None,
|
Ok(raw_file_info) => Ok(Response::new(DeleteVersionResponse {
|
||||||
})),
|
success: true,
|
||||||
Err(err) => Ok(Response::new(DeleteVersionResponse {
|
raw_file_info,
|
||||||
success: false,
|
error: None,
|
||||||
raw_file_info: "".to_string(),
|
})),
|
||||||
error: Some(DiskError::other(format!("encode data failed: {err}")).into()),
|
|
||||||
})),
|
|
||||||
},
|
|
||||||
Err(err) => Ok(Response::new(DeleteVersionResponse {
|
Err(err) => Ok(Response::new(DeleteVersionResponse {
|
||||||
success: false,
|
success: false,
|
||||||
raw_file_info: "".to_string(),
|
raw_file_info: "".to_string(),
|
||||||
error: Some(err.into()),
|
error: Some(DiskError::other(format!("encode data failed: {err}")).into()),
|
||||||
})),
|
})),
|
||||||
}
|
},
|
||||||
} else {
|
Err(err) => Ok(Response::new(DeleteVersionResponse {
|
||||||
Ok(Response::new(DeleteVersionResponse {
|
|
||||||
success: false,
|
success: false,
|
||||||
raw_file_info: "".to_string(),
|
raw_file_info: "".to_string(),
|
||||||
error: Some(DiskError::other("cannot find disk".to_string()).into()),
|
error: Some(err.into()),
|
||||||
}))
|
})),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1206,98 +1250,59 @@ impl NodeService {
|
|||||||
"rename_data",
|
"rename_data",
|
||||||
)?;
|
)?;
|
||||||
let request = request.into_inner();
|
let request = request.into_inner();
|
||||||
if let Some(disk) = self.find_disk(&request.disk).await {
|
let target = self.local_mutation_target();
|
||||||
let decoded_file_info = match decode_rename_data_request_file_info(&request.file_info_bin, &request.file_info) {
|
let decoded_file_info = match decode_rename_data_request_file_info(&request.file_info_bin, &request.file_info) {
|
||||||
Ok(file_info) => file_info,
|
Ok(file_info) => file_info,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
return Ok(Response::new(RenameDataResponse {
|
return Ok(Response::new(RenameDataResponse {
|
||||||
success: false,
|
success: false,
|
||||||
rename_data_resp: String::new(),
|
rename_data_resp: String::new(),
|
||||||
rename_data_resp_bin: Vec::new().into(),
|
rename_data_resp_bin: Vec::new().into(),
|
||||||
error: Some(DiskError::other(format!("decode FileInfo failed: {err}")).into()),
|
error: Some(DiskError::other(format!("decode FileInfo failed: {err}")).into()),
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let scanner_publication_lease_token = if request.scanner_publication_lease_token.is_empty() {
|
let scanner_publication_lease_token = if request.scanner_publication_lease_token.is_empty() {
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
let token = Uuid::from_slice(&request.scanner_publication_lease_token)
|
let token = Uuid::from_slice(&request.scanner_publication_lease_token)
|
||||||
.map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?;
|
.map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?;
|
||||||
if token.is_nil() {
|
if token.is_nil() {
|
||||||
return Err(Status::invalid_argument("scanner publication lease token must not be nil"));
|
return Err(Status::invalid_argument("scanner publication lease token must not be nil"));
|
||||||
}
|
}
|
||||||
Some(token)
|
Some(token)
|
||||||
};
|
};
|
||||||
// The target owns this read guard. It must span the complete
|
let request_decoded_from_msgpack = decoded_file_info.from_msgpack;
|
||||||
// disk rename, not merely the preflight, so a movement transition
|
match target
|
||||||
// cannot restart after validation and before rename linearization.
|
.rename_local_data(
|
||||||
let scanner_publication_lease_guard: Option<Arc<dyn Send + Sync>> =
|
&request.disk,
|
||||||
if let Some(token) = scanner_publication_lease_token {
|
(&request.src_volume, &request.src_path),
|
||||||
let Some(store) = self.resolve_object_store() else {
|
&decoded_file_info.value,
|
||||||
return Ok(Response::new(RenameDataResponse {
|
(&request.dst_volume, &request.dst_path),
|
||||||
success: false,
|
scanner_publication_lease_token,
|
||||||
rename_data_resp: String::new(),
|
)
|
||||||
rename_data_resp_bin: Vec::new().into(),
|
.await
|
||||||
error: Some(DiskError::other("scanner publication lease owner is unavailable").into()),
|
{
|
||||||
}));
|
Ok(rename_data_resp) => match encode_rename_data_response_payloads(&rename_data_resp, request_decoded_from_msgpack) {
|
||||||
};
|
Ok((rename_data_resp, rename_data_resp_bin)) => Ok(Response::new(RenameDataResponse {
|
||||||
match store.acquire_scanner_publication_lease_guard(token).await {
|
success: true,
|
||||||
Ok(guard) => Some(Arc::new(guard)),
|
rename_data_resp,
|
||||||
Err(err) => {
|
rename_data_resp_bin: rename_data_resp_bin.into(),
|
||||||
return Ok(Response::new(RenameDataResponse {
|
error: None,
|
||||||
success: false,
|
})),
|
||||||
rename_data_resp: String::new(),
|
|
||||||
rename_data_resp_bin: Vec::new().into(),
|
|
||||||
error: Some(DiskError::other(err.to_string()).into()),
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
let request_decoded_from_msgpack = decoded_file_info.from_msgpack;
|
|
||||||
match disk
|
|
||||||
.rename_data_borrowed_with_fence_and_guard(
|
|
||||||
&request.src_volume,
|
|
||||||
&request.src_path,
|
|
||||||
&decoded_file_info.value,
|
|
||||||
&request.dst_volume,
|
|
||||||
&request.dst_path,
|
|
||||||
scanner_publication_lease_token,
|
|
||||||
scanner_publication_lease_guard,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(rename_data_resp) => {
|
|
||||||
match encode_rename_data_response_payloads(&rename_data_resp, request_decoded_from_msgpack) {
|
|
||||||
Ok((rename_data_resp, rename_data_resp_bin)) => Ok(Response::new(RenameDataResponse {
|
|
||||||
success: true,
|
|
||||||
rename_data_resp,
|
|
||||||
rename_data_resp_bin: rename_data_resp_bin.into(),
|
|
||||||
error: None,
|
|
||||||
})),
|
|
||||||
Err(err) => Ok(Response::new(RenameDataResponse {
|
|
||||||
success: false,
|
|
||||||
rename_data_resp: String::new(),
|
|
||||||
rename_data_resp_bin: Vec::new().into(),
|
|
||||||
error: Some(err.into()),
|
|
||||||
})),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(err) => Ok(Response::new(RenameDataResponse {
|
Err(err) => Ok(Response::new(RenameDataResponse {
|
||||||
success: false,
|
success: false,
|
||||||
rename_data_resp: String::new(),
|
rename_data_resp: String::new(),
|
||||||
rename_data_resp_bin: Vec::new().into(),
|
rename_data_resp_bin: Vec::new().into(),
|
||||||
error: Some(err.into()),
|
error: Some(err.into()),
|
||||||
})),
|
})),
|
||||||
}
|
},
|
||||||
} else {
|
Err(err) => Ok(Response::new(RenameDataResponse {
|
||||||
Ok(Response::new(RenameDataResponse {
|
|
||||||
success: false,
|
success: false,
|
||||||
rename_data_resp: String::new(),
|
rename_data_resp: String::new(),
|
||||||
rename_data_resp_bin: Vec::new().into(),
|
rename_data_resp_bin: Vec::new().into(),
|
||||||
error: Some(DiskError::other("cannot find disk".to_string()).into()),
|
error: Some(err.into()),
|
||||||
}))
|
})),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -377,10 +377,12 @@ pub(crate) mod timeout_wrapper_consumer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) mod tonic_service_consumer {
|
pub(crate) mod tonic_service_consumer {
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) use super::super::tonic_service::make_server;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use super::super::tonic_service::{heal_topology_fingerprint, make_heal_control_server_for_source};
|
pub(crate) use super::super::tonic_service::{heal_topology_fingerprint, make_heal_control_server_for_source};
|
||||||
pub(crate) use super::super::tonic_service::{
|
pub(crate) use super::super::tonic_service::{
|
||||||
make_heal_control_server_with_cache, make_scanner_control_server, make_server, make_tier_mutation_control_server,
|
make_heal_control_server_with_cache, make_scanner_control_server, make_server_for_slot, make_tier_mutation_control_server,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -600,8 +602,8 @@ pub(crate) mod ecstore_storage {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
|
pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
|
||||||
pub(crate) use rustfs_ecstore::api::storage::{
|
pub(crate) use rustfs_ecstore::api::storage::{
|
||||||
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path,
|
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||||
find_local_disk_by_ref, init_local_disks_with_instance_ctx, init_lock_clients,
|
all_local_disk_path, find_local_disk_by_ref, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||||
prewarm_local_disk_id_map_with_instance_ctx,
|
prewarm_local_disk_id_map_with_instance_ctx,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -677,6 +679,9 @@ type EcstoreReplicationStats = ecstore_bucket::replication::ReplicationStats;
|
|||||||
pub(crate) type DynReplicationPool = StorageReplicationPoolHandle;
|
pub(crate) type DynReplicationPool = StorageReplicationPoolHandle;
|
||||||
pub(crate) type DynReader = ecstore_rio::DynReader;
|
pub(crate) type DynReader = ecstore_rio::DynReader;
|
||||||
pub(crate) type ECStore = ecstore_storage::ECStore;
|
pub(crate) type ECStore = ecstore_storage::ECStore;
|
||||||
|
pub(crate) type BootstrapLocalTarget = ecstore_storage::BootstrapLocalTarget;
|
||||||
|
#[cfg(all(test, not(windows)))]
|
||||||
|
pub(crate) use rustfs_ecstore::api::disk::{LocalPublicationPause, LocalPublicationStage};
|
||||||
pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint;
|
pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) type Endpoints = ecstore_layout::Endpoints;
|
pub(crate) type Endpoints = ecstore_layout::Endpoints;
|
||||||
|
|||||||
@@ -13,8 +13,8 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) use crate::storage::rpc::node_service::make_heal_control_server_with_cache;
|
pub(crate) use crate::storage::rpc::node_service::make_heal_control_server_with_cache;
|
||||||
pub(crate) use crate::storage::rpc::node_service::make_scanner_control_server;
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use crate::storage::rpc::node_service::{heal::heal_topology_fingerprint, make_heal_control_server_for_source};
|
pub(crate) use crate::storage::rpc::node_service::{heal::heal_topology_fingerprint, make_heal_control_server_for_source};
|
||||||
|
pub(crate) use crate::storage::rpc::node_service::{make_scanner_control_server, make_server_for_slot};
|
||||||
pub use crate::storage::rpc::{make_heal_control_server, make_server, make_tier_mutation_control_server};
|
pub use crate::storage::rpc::{make_heal_control_server, make_server, make_tier_mutation_control_server};
|
||||||
pub type NodeService = crate::storage::rpc::NodeService;
|
pub type NodeService = crate::storage::rpc::NodeService;
|
||||||
|
|||||||
@@ -171,12 +171,15 @@ pub(crate) mod server {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) mod tonic_service {
|
pub(crate) mod tonic_service {
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) use crate::storage::storage_api::tonic_service_consumer::make_server;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use crate::storage::storage_api::tonic_service_consumer::{
|
pub(crate) use crate::storage::storage_api::tonic_service_consumer::{
|
||||||
heal_topology_fingerprint, make_heal_control_server_for_source,
|
heal_topology_fingerprint, make_heal_control_server_for_source,
|
||||||
};
|
};
|
||||||
pub(crate) use crate::storage::storage_api::tonic_service_consumer::{
|
pub(crate) use crate::storage::storage_api::tonic_service_consumer::{
|
||||||
make_heal_control_server_with_cache, make_scanner_control_server, make_server, make_tier_mutation_control_server,
|
make_heal_control_server_with_cache, make_scanner_control_server, make_server_for_slot,
|
||||||
|
make_tier_mutation_control_server,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user