fix(iceberg): recover renames and unblock namespace deletion (#9440)

* fix(iceberg): recover renames and unblock namespace deletion

* fix(iceberg): fence rename recovery with restore lease

---------

Co-authored-by: Hauser Zhi <housemecn@gmail.com>
This commit is contained in:
GatewayJ
2026-11-16 23:54:03 +08:00
committed by GitHub
parent 94a138b994
commit 0278e1cd0e
14 changed files with 1105 additions and 67 deletions
@@ -1,6 +1,6 @@
# S3 Tables Catalog Lifecycle
**Use this when:** disabling an object-backed table catalog, cleaning up metadata left by deleted empty tables, or diagnosing catalog registration reads.
**Use this when:** disabling an object-backed table catalog, cleaning up metadata left by deleted empty tables, diagnosing catalog registration reads, or recovering an interrupted table rename.
**Source of truth:** `rustfs/src/admin/handlers/table_catalog/config.rs`, `rustfs/src/table_catalog/store/cleanup.rs`, and `rustfs/src/table_catalog/store/registration_cache.rs`.
## Disable A Table Catalog
@@ -30,3 +30,11 @@ Each server keeps a bounded memory cache of validated ownership records. Reuse r
Registration still enumerates namespace and warehouse-index prefixes, so enumeration grows with catalog size. Cache hits reduce object-body reads and JSON parsing. A hit reuses previously validated content without reading the current body again to prove physical readability. Conditional writes, rollback, and lock-loss fencing continue to control publication; the cache does not synchronize writers that bypass those guards.
Existing catalog, Iceberg metadata, and Console/client request formats remain unchanged. See the [S3 Tables support matrix](../architecture/s3-tables-support-matrix.md) for supported API boundaries.
## Interrupted Table Renames
Every server starts rename recovery once its own application context is ready, including embedded servers. This runs independently of `RUSTFS_TABLE_MAINTENANCE_SCHEDULER_ENABLED`. Recovery scans up to 32 bucket directories per pass, skips their child objects, and admits at most four recoveries concurrently. Passes are separated by five seconds plus jitter; failed buckets retry with exponential delays from 10 seconds up to five minutes. Larger catalogs may need multiple passes. These intervals do not guarantee a recovery deadline while storage or locks remain unavailable.
Recovery resumes the rename referenced by the bucket's `active_rename_id` through the same restore fence, publication locks, conditional writes, and state validation used by `RenameTable`. Reads remain fenced until finalization completes. Missing or invalid intent records retain the fence and produce `table_catalog_rename_recovery` warnings; preserve those records and collect the warnings for investigation. Shutdown stops new admissions and drains started recoveries.
A completed rename preserves the table UUID, warehouse location, and metadata location. Dropping the empty source namespace removes its namespace record and retains valid `DELETED` table/view records. Active objects, pending states, child namespaces, unreadable records, and active rename fences still prevent namespace deletion. Namespace deletion does not remove metadata or data files referenced by the destination table.
+4
View File
@@ -330,6 +330,7 @@ impl RustFSServer {
cancel_token: started.cancel_token,
temp_dir: started.temp_dir,
runtime_owner: Some(runtime_owner),
table_rename_recovery: Some(started.table_rename_recovery),
}
}
}
@@ -350,6 +351,7 @@ pub struct RustFSServer {
cancel_token: CancellationToken,
temp_dir: Option<PathBuf>,
runtime_owner: Option<EmbeddedRuntimeOwner>,
table_rename_recovery: Option<crate::table_catalog::CatalogBackgroundTask>,
}
impl RustFSServer {
@@ -400,6 +402,7 @@ impl RustFSServer {
&mut self.shutdown_handle,
&mut self.temp_dir,
Some(runtime_owner),
self.table_rename_recovery.take(),
)
.await;
}
@@ -417,6 +420,7 @@ impl Drop for RustFSServer {
&mut self.shutdown_handle,
&mut self.temp_dir,
Some(runtime_owner),
self.table_rename_recovery.take(),
);
}
}
+2
View File
@@ -76,6 +76,7 @@ impl EmbeddedStartupArgs {
}
pub(crate) struct EmbeddedStartedServer {
pub(crate) table_rename_recovery: crate::table_catalog::CatalogBackgroundTask,
pub(crate) bound_addr: SocketAddr,
pub(crate) access_key: String,
pub(crate) secret_key: String,
@@ -208,6 +209,7 @@ pub(crate) async fn run_embedded_startup(args: EmbeddedStartupArgs) -> Result<Em
release_embedded_startup_guard();
Ok(EmbeddedStartedServer {
table_rename_recovery: service_runtime.table_rename_recovery,
bound_addr,
access_key: identity.access_key,
secret_key: identity.secret_key,
+2
View File
@@ -210,6 +210,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(
} = lifecycle;
let StartupServiceRuntime {
table_maintenance,
table_rename_recovery,
oidc_recovery,
optional_runtimes,
heartbeat,
@@ -246,6 +247,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(
StartupShutdownServices {
capacity: capacity_tasks,
table_maintenance,
table_rename_recovery,
#[cfg(feature = "internode-transport")]
peer_control: peer_control.take(),
},
+10 -4
View File
@@ -43,6 +43,7 @@ const LOG_COMPONENT_CONNECT: &str = "connect";
const LOG_SUBSYSTEM_INVENTORY: &str = "inventory";
pub(crate) struct StartupServiceRuntime {
pub(crate) table_rename_recovery: Option<crate::table_catalog::CatalogBackgroundTask>,
pub(crate) table_maintenance: Option<crate::table_catalog::PeriodicMaintenanceRuntime>,
pub(crate) oidc_recovery: Option<AbortOnDropHandle<()>>,
pub(crate) optional_runtimes: OptionalRuntimeServices,
@@ -54,6 +55,7 @@ pub(crate) struct StartupServiceRuntime {
}
pub(crate) struct EmbeddedStartupServiceRuntime {
pub(crate) table_rename_recovery: crate::table_catalog::CatalogBackgroundTask,
pub(crate) iam_bootstrap: IamBootstrapDisposition,
}
@@ -67,12 +69,15 @@ pub(crate) async fn init_embedded_startup_runtime_services(
) -> Result<EmbeddedStartupServiceRuntime> {
init_embedded_optional_service_runtime(config, store.clone()).await;
let buckets = init_embedded_bucket_metadata_runtime(store.clone(), &ctx).await?;
let iam_bootstrap = init_embedded_iam_runtime(store, ctx, readiness, server_ctx)
let iam_bootstrap = init_embedded_iam_runtime(store, ctx.clone(), readiness, server_ctx.clone())
.await
.map_err(|err| std::io::Error::other(format!("IAM bootstrap setup: {err}")))?;
init_embedded_notification_runtime(endpoint_pools, buckets).await;
Ok(EmbeddedStartupServiceRuntime { iam_bootstrap })
Ok(EmbeddedStartupServiceRuntime {
iam_bootstrap,
table_rename_recovery: crate::table_catalog::spawn_table_rename_recovery(server_ctx, &ctx),
})
}
pub(crate) async fn init_startup_runtime_services(
@@ -121,11 +126,12 @@ pub(crate) async fn init_startup_runtime_services(
let heartbeat = start_heartbeat_runtime(heartbeat_config.clone(), heartbeat_nodes, &ctx)?;
let inventory = start_inventory_runtime(heartbeat_config, heartbeat_nodes, inventory_drives, store, &ctx)?;
let table_maintenance =
maintenance_config.map(|config| crate::table_catalog::PeriodicMaintenanceRuntime::spawn(config, server_ctx, &ctx));
let table_maintenance = maintenance_config
.map(|config| crate::table_catalog::PeriodicMaintenanceRuntime::spawn(config, server_ctx.clone(), &ctx));
Ok(StartupServiceRuntime {
table_maintenance,
table_rename_recovery: Some(crate::table_catalog::spawn_table_rename_recovery(server_ctx, &ctx)),
oidc_recovery,
optional_runtimes,
heartbeat,
+46 -16
View File
@@ -225,7 +225,7 @@ where
Ok(()) => true,
Err(error) => error
.get_ref()
.and_then(|error| error.downcast_ref::<MaintenanceShutdownFailure>())
.and_then(|error| error.downcast_ref::<CatalogShutdownFailure>())
.is_some_and(|failure| failure.joined),
};
if !maintenance_retired || !front_result.producers_retired() {
@@ -380,13 +380,13 @@ impl CapacityShutdownReport {
}
#[derive(Debug)]
struct MaintenanceShutdownFailure {
struct CatalogShutdownFailure {
timed_out: bool,
joined: bool,
join_error: Option<tokio::task::JoinError>,
}
impl std::fmt::Display for MaintenanceShutdownFailure {
impl std::fmt::Display for CatalogShutdownFailure {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let reason = match &self.join_error {
Some(error) => join_failure_reason(error),
@@ -394,21 +394,21 @@ impl std::fmt::Display for MaintenanceShutdownFailure {
None => "actual join unconfirmed",
};
if self.timed_out {
write!(f, "maintenance shutdown exceeded its original deadline: {reason}")
write!(f, "catalog background shutdown exceeded its original deadline: {reason}")
} else {
write!(f, "maintenance shutdown failed: {reason}")
write!(f, "catalog background shutdown failed: {reason}")
}
}
}
impl std::error::Error for MaintenanceShutdownFailure {
impl std::error::Error for CatalogShutdownFailure {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
self.join_error.as_ref().map(|error| error as _)
}
}
pub(crate) async fn stop_table_maintenance(
runtime: Option<crate::table_catalog::PeriodicMaintenanceRuntime>,
pub(crate) async fn stop_catalog_background_task(
runtime: Option<crate::table_catalog::CatalogBackgroundTask>,
) -> std::io::Result<()> {
let Some(mut runtime) = runtime else {
return Ok(());
@@ -434,7 +434,7 @@ pub(crate) async fn stop_table_maintenance(
} else {
std::io::ErrorKind::Other
},
MaintenanceShutdownFailure {
CatalogShutdownFailure {
timed_out,
joined: observed_join.is_some(),
join_error: observed_join.and_then(std::result::Result::err),
@@ -629,6 +629,7 @@ pub(crate) struct StartupShutdownServices {
#[cfg(feature = "internode-transport")]
pub(crate) peer_control: Option<std::sync::Arc<crate::storage_api::startup::storage::InstalledControlService>>,
pub(crate) capacity: Option<CapacityBackgroundTasks>,
pub(crate) table_rename_recovery: Option<crate::table_catalog::CatalogBackgroundTask>,
pub(crate) table_maintenance: Option<crate::table_catalog::PeriodicMaintenanceRuntime>,
}
@@ -646,6 +647,7 @@ pub(crate) async fn run_startup_shutdown_sequence(
peer_control,
capacity: capacity_tasks,
table_maintenance,
table_rename_recovery,
} = background_tasks;
ctx.cancel();
@@ -847,7 +849,11 @@ pub(crate) async fn run_startup_shutdown_sequence(
)
.await;
let front_result = front_failures;
let maintenance_result = stop_table_maintenance(table_maintenance).await;
let (maintenance_result, recovery_result) = tokio::join!(
stop_catalog_background_task(table_maintenance),
stop_catalog_background_task(table_rename_recovery),
);
let maintenance_result = maintenance_result.and(recovery_result);
// Unknown front leaves cannot authorize the downstream fence. A retired HTTP
// scope may finish cleanup after a late/failed receipt, while its marker stays dirty.
finish_detached_mutations_and_clear_unclean_shutdown_markers(
@@ -933,7 +939,15 @@ pub(crate) fn signal_embedded_startup_shutdown(shutdown_handle: &ShutdownHandle,
ctx.cancel();
}
async fn release_embedded_runtime_after_drain(mut runtime_owner: Option<EmbeddedRuntimeOwner>) {
async fn release_embedded_runtime_after_drain(
mut runtime_owner: Option<EmbeddedRuntimeOwner>,
table_rename_recovery: Option<crate::table_catalog::CatalogBackgroundTask>,
) {
if let Err(error) = stop_catalog_background_task(table_rename_recovery).await {
warn!(component = LOG_COMPONENT_EMBEDDED, subsystem = LOG_SUBSYSTEM_EMBEDDED,
event = EVENT_EMBEDDED_SHUTDOWN_CLEANUP_FAILED, service = "table_rename_recovery", error = %error,
"Embedded shutdown cleanup failed");
}
if let Some(runtime_owner) = runtime_owner.as_mut()
&& let Some(cleanup) = runtime_owner.release()
{
@@ -947,6 +961,7 @@ pub(crate) fn run_embedded_server_drop_cleanup(
shutdown_handle: &mut Option<ShutdownHandle>,
temp_dir: &mut Option<PathBuf>,
runtime_owner: Option<EmbeddedRuntimeOwner>,
table_rename_recovery: Option<crate::table_catalog::CatalogBackgroundTask>,
) {
ctx.cancel();
if let Some(shutdown_handle) = shutdown_handle.as_ref() {
@@ -958,7 +973,7 @@ pub(crate) fn run_embedded_server_drop_cleanup(
runtime.spawn(finish_embedded_server_cleanup(
shutdown_handle,
temp_dir,
release_embedded_runtime_after_drain(runtime_owner),
release_embedded_runtime_after_drain(runtime_owner, table_rename_recovery),
));
}
@@ -993,6 +1008,7 @@ pub(crate) async fn run_embedded_server_shutdown(
shutdown_handle: &mut Option<ShutdownHandle>,
temp_dir: &mut Option<PathBuf>,
runtime_owner: Option<EmbeddedRuntimeOwner>,
table_rename_recovery: Option<crate::table_catalog::CatalogBackgroundTask>,
) {
info!(
target: "rustfs::embedded",
@@ -1012,7 +1028,7 @@ pub(crate) async fn run_embedded_server_shutdown(
let cleanup_task = runtime.spawn(finish_embedded_server_cleanup(
shutdown_handle.take(),
temp_dir.take(),
release_embedded_runtime_after_drain(runtime_owner),
release_embedded_runtime_after_drain(runtime_owner, table_rename_recovery),
));
if let Err(err) = cleanup_task.await {
warn!(
@@ -1322,17 +1338,17 @@ mod tests {
let task = tokio::spawn(async { panic!("expected maintenance marker fixture failure") });
let join_error = task.await.expect_err("actual task should fail");
for failure in [
super::MaintenanceShutdownFailure {
super::CatalogShutdownFailure {
timed_out: true,
joined: true,
join_error: None,
},
super::MaintenanceShutdownFailure {
super::CatalogShutdownFailure {
timed_out: false,
joined: false,
join_error: None,
},
super::MaintenanceShutdownFailure {
super::CatalogShutdownFailure {
timed_out: false,
joined: true,
join_error: Some(join_error),
@@ -1480,18 +1496,32 @@ mod tests {
let temp_path = temp_dir.path().to_path_buf();
let mut owned_temp_path = Some(temp_path.clone());
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
let (draining_tx, draining_rx) = tokio::sync::oneshot::channel();
let recovery = crate::table_catalog::CatalogBackgroundTask::start(&cancel_token, |stop| async move {
stop.cancelled().await;
let _ = draining_tx.send(());
let _ = release_rx.await;
});
run_embedded_server_drop_cleanup(
&tokio::runtime::Handle::current(),
&cancel_token,
&mut shutdown_handle,
&mut owned_temp_path,
None,
Some(recovery),
);
tokio::time::timeout(Duration::from_secs(1), observed_rx)
.await
.expect("drop cleanup should signal shutdown")
.expect("shutdown signal should be delivered");
tokio::time::timeout(Duration::from_secs(1), draining_rx)
.await
.unwrap()
.unwrap();
assert!(temp_path.exists(), "catalog publications retain the storage directory until drained");
release_tx.send(()).unwrap();
tokio::time::timeout(Duration::from_secs(1), async {
while temp_path.exists() {
tokio::task::yield_now().await;
+56
View File
@@ -0,0 +1,56 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::future::Future;
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
/// Retains a catalog worker until its in-flight publications have drained.
pub(crate) struct CatalogBackgroundTask {
pub(in crate::table_catalog) cancellation: CancellationToken,
pub(in crate::table_catalog) task: Option<JoinHandle<()>>,
}
impl CatalogBackgroundTask {
pub(crate) fn start<F, W>(shutdown: &CancellationToken, worker: W) -> Self
where
F: Future<Output = ()> + Send + 'static,
W: FnOnce(CancellationToken) -> F,
{
let cancellation = shutdown.child_token();
let task = tokio::spawn(worker(cancellation.clone()));
Self {
cancellation,
task: Some(task),
}
}
pub(crate) fn request_stop(&self) {
self.cancellation.cancel();
}
pub(crate) async fn join(&mut self) -> Option<Result<(), tokio::task::JoinError>> {
// A cancelled waiter leaves the actual task in this owner.
let result = self.task.as_mut()?.await;
self.task = None;
Some(result)
}
}
impl Drop for CatalogBackgroundTask {
fn drop(&mut self) {
// Detach, never abort: in-flight writes retain their publication owner.
self.cancellation.cancel();
}
}
@@ -20,7 +20,7 @@ use crate::table_catalog::{
TableCatalogStoreError, TableCatalogStoreResult,
};
use std::{collections::BTreeSet, future::Future, sync::Arc, time::Duration};
use tokio::{task::JoinHandle, time::Instant};
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use tracing::{info, warn};
@@ -91,38 +91,11 @@ impl PeriodicMaintenanceConfig {
}
}
/// Owns queueing until completion; never runs a worker or deletes table objects.
pub(crate) struct PeriodicMaintenanceRuntime {
cancellation: CancellationToken,
task: Option<JoinHandle<()>>,
}
impl Drop for PeriodicMaintenanceRuntime {
fn drop(&mut self) {
// Detach, never abort: in-flight sidecar writes retain their publication owner.
self.cancellation.cancel();
}
}
pub(crate) type PeriodicMaintenanceRuntime = crate::table_catalog::CatalogBackgroundTask;
impl PeriodicMaintenanceRuntime {
pub(crate) fn spawn(config: PeriodicMaintenanceConfig, server: Arc<ServerContextSlot>, shutdown: &CancellationToken) -> Self {
let cancellation = shutdown.child_token();
let task = tokio::spawn(run(config, server, cancellation.clone()));
Self {
cancellation,
task: Some(task),
}
}
pub(crate) fn request_stop(&self) {
self.cancellation.cancel();
}
pub(crate) async fn join(&mut self) -> Option<Result<(), tokio::task::JoinError>> {
// A cancelled waiter leaves the actual task in this owner.
let result = self.task.as_mut()?.await;
self.task = None;
Some(result)
Self::start(shutdown, move |cancellation| run(config, server, cancellation))
}
}
@@ -359,7 +332,7 @@ mod tests {
task: Some(task),
};
started.await.expect("real owned pass must be held");
let mut stopping = Box::pin(crate::startup_shutdown::stop_table_maintenance(Some(runtime)));
let mut stopping = Box::pin(crate::startup_shutdown::stop_catalog_background_task(Some(runtime)));
tokio::select! {
biased;
result = stopping.as_mut() => panic!("held pass returned before shutdown was observed: {result:?}"),
@@ -397,7 +370,7 @@ mod tests {
task: Some(task),
};
started.await.expect("actual task is held");
let mut stopping = Box::pin(crate::startup_shutdown::stop_table_maintenance(Some(runtime)));
let mut stopping = Box::pin(crate::startup_shutdown::stop_catalog_background_task(Some(runtime)));
tokio::select! {
biased;
result = stopping.as_mut() => panic!("held task returned before stop: {result:?}"),
@@ -451,7 +424,7 @@ mod tests {
task: Some(task),
};
entered_rx.await.expect("the original task must start before observation");
let mut stopping = Box::pin(crate::startup_shutdown::stop_table_maintenance(Some(runtime)));
let mut stopping = Box::pin(crate::startup_shutdown::stop_catalog_background_task(Some(runtime)));
let first = std::future::poll_fn(|cx| std::task::Poll::Ready(std::future::Future::poll(stopping.as_mut(), cx))).await;
let initially_pending = first.is_pending();
// Do not poll the owner again until the deadline and actual task exit.
@@ -485,7 +458,7 @@ mod tests {
#[tokio::test]
async fn periodic_shutdown_accepts_ready_and_disabled_but_rejects_missing_join() {
crate::startup_shutdown::stop_table_maintenance(None)
crate::startup_shutdown::stop_catalog_background_task(None)
.await
.expect("disabled maintenance has no owner to drain");
let task = tokio::spawn(async {});
@@ -496,14 +469,14 @@ mod tests {
cancellation: CancellationToken::new(),
task: Some(task),
};
crate::startup_shutdown::stop_table_maintenance(Some(runtime))
crate::startup_shutdown::stop_catalog_background_task(Some(runtime))
.await
.expect("actual promptly completed task must join");
let runtime = PeriodicMaintenanceRuntime {
cancellation: CancellationToken::new(),
task: None,
};
let error = crate::startup_shutdown::stop_table_maintenance(Some(runtime))
let error = crate::startup_shutdown::stop_catalog_background_task(Some(runtime))
.await
.expect_err("configured runtime with no join observation must fail closed");
assert_eq!(error.kind(), std::io::ErrorKind::Other);
+4
View File
@@ -59,14 +59,17 @@ use crate::storage_api::table::{
StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader,
};
mod background;
mod cleanup_metadata;
mod error;
mod iceberg;
mod identifier;
mod maintenance;
mod model;
mod rename_recovery;
mod store;
pub(crate) use background::CatalogBackgroundTask;
pub use error::{CatalogIdentifierError, TableObjectMutationError};
pub(crate) use error::{TableCatalogStoreError, TableCatalogStoreResult};
pub(crate) use iceberg::*;
@@ -82,6 +85,7 @@ pub(crate) use identifier::{
};
pub(crate) use maintenance::*;
pub(crate) use model::*;
pub(crate) use rename_recovery::spawn_table_rename_recovery;
pub(crate) use store::*;
pub(crate) const TABLE_BUCKET_MARKER_CONFIG: &str = BUCKET_TABLE_CONFIG;
+202
View File
@@ -0,0 +1,202 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::{
CatalogBackgroundTask, EcStoreTableCatalogObjectBackend, ObjectTableCatalogStore, TableCatalogObjectBackend,
TableCatalogStoreError,
};
use crate::storage_api::startup::services::ServerContextSlot;
use futures::{StreamExt, stream};
use std::{
collections::{BTreeMap, BTreeSet},
sync::Arc,
time::Duration,
};
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use tracing::{info, warn};
const SCAN_BATCH_SIZE: usize = 32;
const RECOVERY_CONCURRENCY: usize = 4;
const RETRY_CACHE_SIZE: usize = 1024;
const SCAN_INTERVAL: Duration = Duration::from_secs(5);
const INITIAL_RETRY_DELAY: Duration = Duration::from_secs(10);
const MAX_RETRY_DELAY: Duration = Duration::from_secs(300);
const LOG_COMPONENT: &str = "table_catalog";
const LOG_SUBSYSTEM: &str = "rename";
const EVENT_RENAME_RECOVERY: &str = "table_catalog_rename_recovery";
/// Recovery is always enabled, independently of the optional maintenance scheduler.
pub(crate) fn spawn_table_rename_recovery(server: Arc<ServerContextSlot>, shutdown: &CancellationToken) -> CatalogBackgroundTask {
CatalogBackgroundTask::start(shutdown, move |cancellation| async move {
loop {
if cancellation.is_cancelled() {
return;
}
if let Some(context) = server.installed_app_context() {
let store = ObjectTableCatalogStore::new(EcStoreTableCatalogObjectBackend::new(context.object_store()));
run(store, cancellation).await;
return;
}
if !wait(&cancellation, Duration::from_secs(1)).await {
return;
}
}
})
}
async fn wait(cancellation: &CancellationToken, delay: Duration) -> bool {
tokio::select! {
biased;
_ = cancellation.cancelled() => false,
_ = tokio::time::sleep(delay) => true,
}
}
// Decode failures can contain persisted values; log only their error category.
fn recovery_error_kind(error: &TableCatalogStoreError) -> &'static str {
match error {
TableCatalogStoreError::NotFound(_)
| TableCatalogStoreError::NamespaceNotFound(_)
| TableCatalogStoreError::TableNotFound(_) => "not_found",
TableCatalogStoreError::AlreadyExists(_) => "already_exists",
TableCatalogStoreError::Conflict(_) => "conflict",
TableCatalogStoreError::Invalid(_) => "invalid",
TableCatalogStoreError::Unsupported(_) => "unsupported",
TableCatalogStoreError::Unavailable(_) => "unavailable",
TableCatalogStoreError::LockUnavailable { .. } => "lock_unavailable",
TableCatalogStoreError::Internal(_) => "internal",
}
}
struct Retry {
after: Instant,
delay: Duration,
}
#[derive(Default)]
struct RecoveryScan {
cursor: Option<String>,
retries: BTreeMap<String, Retry>,
discovery_degraded: bool,
}
impl RecoveryScan {
async fn pass<B: TableCatalogObjectBackend>(&mut self, store: &ObjectTableCatalogStore<B>, cancellation: &CancellationToken) {
let mut buckets = BTreeSet::new();
let mut discovery_failed = false;
for _ in 0..SCAN_BATCH_SIZE {
if cancellation.is_cancelled() {
break;
}
let previous = self.cursor.clone();
match store.next_table_bucket_for_rename_recovery(&mut self.cursor).await {
Ok(Some(entry)) => {
if entry.active_rename_id.is_none() {
self.retries.remove(&entry.table_bucket);
} else if self
.retries
.get(&entry.table_bucket)
.is_none_or(|retry| Instant::now() >= retry.after)
{
buckets.insert(entry.table_bucket);
}
}
Ok(None) => break,
Err(error) => {
discovery_failed = true;
if !self.discovery_degraded {
warn!(
event = EVENT_RENAME_RECOVERY,
component = LOG_COMPONENT,
subsystem = LOG_SUBSYSTEM,
state = "deferred",
reason = "discovery_failed",
error_type = recovery_error_kind(&error),
"Table rename recovery discovery failed"
);
self.discovery_degraded = true;
}
// A listing failure cannot advance; retry on the next tick.
// Invalid entries already advanced past their bucket directory.
if previous == self.cursor {
break;
}
}
}
}
if !discovery_failed {
self.discovery_degraded = false;
}
let recoveries = stream::iter(buckets)
.map(|bucket| async move {
if cancellation.is_cancelled() {
return (bucket, None);
}
let result = store.recover_pending_table_rename(&bucket).await;
(bucket, Some(result))
})
.buffer_unordered(RECOVERY_CONCURRENCY);
tokio::pin!(recoveries);
// Never cancel a started recovery: its publication lease must cover every
// in-flight storage write, including shutdown and ambiguous write results.
while let Some((bucket, result)) = recoveries.next().await {
match result {
Some(Ok(())) => {
self.retries.remove(&bucket);
info!(event = EVENT_RENAME_RECOVERY, component = LOG_COMPONENT, subsystem = LOG_SUBSYSTEM,
bucket = %bucket, state = "completed", "Table rename recovery completed");
}
Some(Err(error)) => {
let delay = self
.retries
.get(&bucket)
.map_or(INITIAL_RETRY_DELAY, |retry| (retry.delay * 2).min(MAX_RETRY_DELAY));
if !self.retries.contains_key(&bucket)
&& self.retries.len() >= RETRY_CACHE_SIZE
&& let Some(oldest) = self
.retries
.iter()
.min_by_key(|(_, retry)| retry.after)
.map(|(key, _)| key.clone())
{
self.retries.remove(&oldest);
}
self.retries.insert(
bucket.clone(),
Retry {
after: Instant::now() + delay,
delay,
},
);
warn!(event = EVENT_RENAME_RECOVERY, component = LOG_COMPONENT, subsystem = LOG_SUBSYSTEM,
bucket = %bucket, state = "deferred", reason = "recovery_failed", retry_delay_secs = delay.as_secs(), error_type = recovery_error_kind(&error),
"Table rename recovery will retry");
}
None => {}
}
}
}
}
pub(super) async fn run<B: TableCatalogObjectBackend>(store: ObjectTableCatalogStore<B>, cancellation: CancellationToken) {
let mut scan = RecoveryScan::default();
loop {
scan.pass(&store, &cancellation).await;
let delay = SCAN_INTERVAL + Duration::from_millis(rand::random_range(0..=1000));
if !wait(&cancellation, delay).await {
return;
}
}
}
+5 -1
View File
@@ -1297,6 +1297,10 @@ impl Default for TableCatalogObjectPaths {
}
impl TableCatalogObjectPaths {
pub fn table_bucket_entries_prefix(&self) -> String {
format!("{}/{}/", self.catalog_root, TABLE_BUCKET_ROOT)
}
pub fn table_bucket_entry_path(&self, table_bucket: &str) -> String {
format!("{}{}", self.table_bucket_root_prefix(table_bucket), TABLE_BUCKET_ENTRY_FILE)
}
@@ -1623,7 +1627,7 @@ impl TableCatalogObjectPaths {
}
fn table_bucket_root_prefix(&self, table_bucket: &str) -> String {
format!("{}/{}/{}/", self.catalog_root, TABLE_BUCKET_ROOT, table_catalog_path_hash(table_bucket))
format!("{}{}/", self.table_bucket_entries_prefix(), table_catalog_path_hash(table_bucket))
}
}
+71 -2
View File
@@ -717,8 +717,25 @@ where
.list_objects_page(self.catalog_bucket(), &prefix, cursor.as_deref(), scan_limit)
.await?;
let last_scanned = page.objects.last().cloned();
if page.objects.iter().any(|object| object.ends_with(entry_file)) {
return Ok(true);
for object in page.objects.iter().filter(|object| object.ends_with(entry_file)) {
let state = if entry_file == TABLE_ENTRY_FILE {
let Some((entry, _)) = self.read_entry::<TableEntry>(self.catalog_bucket(), object).await? else {
continue;
};
validate_table_entry_object(&self.paths, object, &entry)?;
entry.state
} else {
let Some((entry, _)) = self.read_entry::<ViewEntry>(self.catalog_bucket(), object).await? else {
continue;
};
validate_view_entry_object(&self.paths, object, &entry)?;
entry.state
};
// Retain tombstones for conditional identifier reuse. Pending
// publications and malformed records must still block deletion.
if state != TableCatalogEntryState::Deleted {
return Ok(true);
}
}
if !page.is_truncated {
break;
@@ -1613,6 +1630,58 @@ where
.await
}
/// Advances by bucket directories, regardless of the number of table objects.
/// The cursor advances before decoding so one invalid bucket cannot starve later buckets.
pub(crate) async fn next_table_bucket_for_rename_recovery(
&self,
cursor: &mut Option<String>,
) -> TableCatalogStoreResult<Option<TableBucketEntry>> {
let prefix = self.paths.table_bucket_entries_prefix();
let page = self
.backend
.list_objects_page(self.catalog_bucket(), &prefix, cursor.as_deref(), NonZeroUsize::MIN)
.await?;
let Some(object) = page.objects.first() else {
*cursor = None;
return Ok(None);
};
if cursor.as_deref().is_some_and(|previous| object.as_str() <= previous) {
return Err(TableCatalogStoreError::Internal("catalog bucket scan did not advance".to_string()));
}
*cursor = Some(object.clone());
let directory = object
.strip_prefix(&prefix)
.and_then(|suffix| suffix.split_once('/'))
.map(|(directory, _)| directory)
.ok_or_else(|| TableCatalogStoreError::Invalid("invalid catalog bucket object path".to_string()))?;
*cursor = Some(format!("{prefix}{directory}/\u{10ffff}"));
let entry_object = format!("{prefix}{directory}/{TABLE_BUCKET_ENTRY_FILE}");
let (entry, _) = self
.read_entry::<TableBucketEntry>(self.catalog_bucket(), &entry_object)
.await?
.ok_or_else(|| TableCatalogStoreError::Invalid("catalog bucket entry missing during discovery".to_string()))?;
validate_table_bucket_entry_object(&self.paths, &entry_object, &entry)?;
Ok(Some(entry))
}
/// Uses the same restore fence, publication lease and state machine as RenameTable.
pub(crate) fn recover_pending_table_rename<'a>(
&'a self,
table_bucket: &'a str,
) -> futures::future::BoxFuture<'a, TableCatalogStoreResult<()>> {
// Keep the publication future opaque to the background concurrency adapters.
Box::pin(async move {
let publication = TableCommitLockPublication::new(&self.backend);
publication
.attach_catalog_migration_read_guard(self.acquire_object_backed_catalog_write_permit(table_bucket).await?)?;
let _completion = TableCommitPublicationCompletion::new(&publication);
let _migration_guard = self
.begin_table_bucket_with_migration_permit(table_bucket, &publication)
.await?;
self.recover_active_table_rename(table_bucket, &publication).await
})
}
async fn recover_active_table_rename(
&self,
table_bucket: &str,
+33 -7
View File
@@ -33,6 +33,9 @@ mod object_maintenance_equivalence;
#[path = "tests/object_backup_transfer.rs"]
mod object_backup_transfer;
#[path = "tests/rename_recovery.rs"]
mod rename_recovery;
#[tokio::test]
async fn catalog_backings_fence_direct_commits_with_publication_lock() {
let object_backend = TestCatalogObjectBackend::default();
@@ -1249,11 +1252,6 @@ async fn catalog_backings_hide_and_reject_mutation_of_inactive_resources() {
.expect("views should list")
.is_empty()
);
assert_matches!(
store.drop_namespace(&bucket, &namespace.public_name()).await,
Err(TableCatalogStoreError::Conflict(_))
);
backend.seed_object(&bucket, &new_metadata, b"{}".to_vec()).await;
let commit_error = store
.commit_table(TableCommitRequest {
@@ -1285,6 +1283,18 @@ async fn catalog_backings_hide_and_reject_mutation_of_inactive_resources() {
.await
.expect_err("inactive view must reject replacement");
assert_matches!(view_error, TableCatalogStoreError::NotFound(_));
store
.drop_namespace(&bucket, &namespace.public_name())
.await
.expect("deleted resources should allow namespace deletion");
assert!(
store
.get_namespace(&bucket, &namespace.public_name())
.await
.expect("namespace lookup should succeed")
.is_none()
);
}
}
@@ -7876,7 +7886,7 @@ async fn object_catalog_pagination_bounds_sparse_namespace_scans() {
}
#[tokio::test]
async fn object_catalog_drop_namespace_scans_all_pages_for_inactive_resources() {
async fn object_catalog_drop_namespace_scans_all_pages_for_pending_resources() {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let bucket = "analytics";
@@ -7903,7 +7913,7 @@ async fn object_catalog_drop_namespace_scans_all_pages_for_inactive_resources()
&table,
default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
);
inactive.state = TableCatalogEntryState::Deleted;
inactive.state = TableCatalogEntryState::Deleting;
store
.write_entry(
store.catalog_bucket(),
@@ -16040,6 +16050,22 @@ async fn object_catalog_source_recreation_after_rename_preserves_the_original_id
.seed_object(bucket, &replacement_metadata, serde_json::to_vec(&replacement_json).unwrap())
.await;
if destination_namespace != "sales" {
restarted.drop_namespace(bucket, "sales").await.unwrap();
assert!(restarted.get_namespace(bucket, "sales").await.unwrap().is_none());
assert_eq!(backend.read_object(bucket, &next).await.unwrap(), Some(before_next.clone()));
assert_eq!(
restarted
.load_table(bucket, destination_namespace, "orders_v2")
.await
.unwrap(),
Some(renamed.clone())
);
restarted
.create_namespace(test_namespace_entry(bucket, &source_namespace))
.await
.unwrap();
}
restarted
.create_table(replacement.clone())
.await
@@ -0,0 +1,652 @@
use super::*;
use tokio_util::sync::CancellationToken;
async fn interrupted_rename(
backend: &TestCatalogObjectBackend,
bucket: &'static str,
stage: TableRenameIntentState,
) -> (TableEntry, String) {
let store = ObjectTableCatalogStore::new(backend.clone());
let namespace = Namespace::parse("sales").unwrap();
let table = IdentifierSegment::parse("orders").unwrap();
let metadata = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json");
seed_table_for_metadata_maintenance(&store, bucket, &namespace, &table, metadata).await;
store
.create_namespace(test_namespace_entry(bucket, &Namespace::parse("curated").unwrap()))
.await
.unwrap();
let source = store.load_table(bucket, "sales", "orders").await.unwrap().unwrap();
let bucket_path = store.paths.table_bucket_entry_path(bucket);
let marker_attempt = backend.put_attempt_count(RUSTFS_META_BUCKET, &bucket_path).await + 1;
let pause = backend
.pause_put_attempt(RUSTFS_META_BUCKET, &bucket_path, marker_attempt)
.await;
let rename = tokio::spawn(async move { store.rename_table(bucket, "sales", "orders", "curated", "orders_v2").await });
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, pause.wait_started())
.await
.unwrap();
let intent_path = backend
.state
.lock()
.await
.objects
.iter()
.find_map(|((_, key), record)| {
serde_json::from_slice::<TableRenameIntent>(&record.data)
.ok()
.filter(|intent| intent.table_bucket == bucket)
.map(|_| key.clone())
})
.expect("prepared intent precedes the bucket marker");
if stage == TableRenameIntentState::Completed {
backend
.fail_put_attempt(RUSTFS_META_BUCKET, &bucket_path, marker_attempt + 1)
.await;
} else {
let attempt = match stage {
TableRenameIntentState::Prepared => 2,
TableRenameIntentState::MaintenancePrepared => 3,
TableRenameIntentState::SourceFenced => 4,
TableRenameIntentState::DestinationWritten => 5,
TableRenameIntentState::SourceTombstoned => 6,
TableRenameIntentState::IndexPublished => 7,
TableRenameIntentState::DestinationPublished => 8,
TableRenameIntentState::Completed => unreachable!(),
};
backend.fail_put_attempt(RUSTFS_META_BUCKET, &intent_path, attempt).await;
}
pause.release();
assert!(
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, rename)
.await
.unwrap()
.unwrap()
.is_err()
);
let store = ObjectTableCatalogStore::new(backend.clone());
let (intent, _) = store
.read_entry::<TableRenameIntent>(RUSTFS_META_BUCKET, &intent_path)
.await
.unwrap()
.unwrap();
assert_eq!(intent.state, stage);
assert_matches!(store.list_tables(bucket, "sales").await, Err(TableCatalogStoreError::Unavailable(_)));
(source, intent_path)
}
async fn wait_for_recovery(store: &ObjectTableCatalogStore<TestCatalogObjectBackend>, bucket: &str) {
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async {
loop {
if store
.get_table_bucket(bucket)
.await
.unwrap()
.unwrap()
.active_rename_id
.is_none()
{
return;
}
tokio::task::yield_now().await;
}
})
.await
.expect("background recovery should restore read availability without another mutation");
}
#[tokio::test]
async fn background_rename_recovery_resumes_every_durable_phase_after_restart() {
for stage in [
TableRenameIntentState::Prepared,
TableRenameIntentState::MaintenancePrepared,
TableRenameIntentState::SourceFenced,
TableRenameIntentState::DestinationWritten,
TableRenameIntentState::SourceTombstoned,
TableRenameIntentState::IndexPublished,
TableRenameIntentState::DestinationPublished,
TableRenameIntentState::Completed,
] {
let backend = TestCatalogObjectBackend::default();
let (source, intent_path) = interrupted_rename(&backend, "analytics", stage).await;
let store = ObjectTableCatalogStore::new(backend.clone());
let cancellation = CancellationToken::new();
let recovered_store = store.clone();
let mut runtime =
CatalogBackgroundTask::start(&cancellation, |stop| super::super::rename_recovery::run(recovered_store, stop));
wait_for_recovery(&store, "analytics").await;
runtime.request_stop();
runtime.join().await.unwrap().unwrap();
assert!(store.list_tables("analytics", "sales").await.unwrap().is_empty());
let destination = store.load_table("analytics", "curated", "orders_v2").await.unwrap().unwrap();
assert_eq!(destination.table_id, source.table_id);
assert_eq!(destination.table_uuid, source.table_uuid);
assert_eq!(destination.metadata_location, source.metadata_location);
assert_eq!(destination.warehouse_location, source.warehouse_location);
assert_eq!(destination.generation, source.generation);
assert_eq!(destination.version_token, source.version_token);
assert_eq!(
store
.read_entry::<TableRenameIntent>(RUSTFS_META_BUCKET, &intent_path)
.await
.unwrap()
.unwrap()
.0
.state,
TableRenameIntentState::Completed
);
store.drop_namespace("analytics", "sales").await.unwrap();
assert!(store.get_namespace("analytics", "sales").await.unwrap().is_none());
assert_eq!(store.load_table("analytics", "curated", "orders_v2").await.unwrap(), Some(destination));
}
}
#[tokio::test]
async fn background_rename_recovery_retains_invalid_intents_and_recovers_other_buckets() {
for corrupt in [false, true] {
let backend = TestCatalogObjectBackend::default();
let (_, invalid_intent) = interrupted_rename(&backend, "broken", TableRenameIntentState::IndexPublished).await;
let (_, valid_intent) = interrupted_rename(&backend, "healthy", TableRenameIntentState::IndexPublished).await;
if corrupt {
backend
.seed_object(RUSTFS_META_BUCKET, &invalid_intent, b"{broken".to_vec())
.await;
} else {
backend
.state
.lock()
.await
.objects
.remove(&(RUSTFS_META_BUCKET.to_string(), invalid_intent.clone()));
}
let store = ObjectTableCatalogStore::new(backend.clone());
let marker_before = store.get_table_bucket("broken").await.unwrap().unwrap();
let broken_key = (RUSTFS_META_BUCKET.to_string(), invalid_intent.clone());
let reads_before = backend.state.lock().await.read_attempts[&broken_key];
let cancellation = CancellationToken::new();
let worker_store = store.clone();
let mut runtime =
CatalogBackgroundTask::start(&cancellation, |stop| super::super::rename_recovery::run(worker_store, stop));
wait_for_recovery(&store, "healthy").await;
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async {
while backend.state.lock().await.read_attempts[&broken_key] == reads_before {
tokio::task::yield_now().await;
}
})
.await
.expect("the invalid intent must also be attempted");
runtime.request_stop();
runtime.join().await.unwrap().unwrap();
assert_eq!(store.get_table_bucket("broken").await.unwrap(), Some(marker_before));
assert_matches!(store.list_tables("broken", "curated").await, Err(TableCatalogStoreError::Unavailable(_)));
assert_matches!(store.drop_namespace("broken", "sales").await, Err(TableCatalogStoreError::Unavailable(_)));
assert_eq!(
store
.read_entry::<TableRenameIntent>(RUSTFS_META_BUCKET, &valid_intent)
.await
.unwrap()
.unwrap()
.0
.state,
TableRenameIntentState::Completed
);
}
}
#[tokio::test(start_paused = true)]
async fn background_rename_recovery_drains_on_shutdown_and_serializes_competing_workers() {
let backend = TestCatalogObjectBackend::default();
let (_, intent_path) = interrupted_rename(&backend, "analytics", TableRenameIntentState::IndexPublished).await;
let store = ObjectTableCatalogStore::new(backend.clone());
let bucket_path = store.paths.table_bucket_entry_path("analytics");
let pause = backend
.pause_put_attempt(
RUSTFS_META_BUCKET,
&bucket_path,
backend.put_attempt_count(RUSTFS_META_BUCKET, &bucket_path).await + 1,
)
.await;
let cancellation = CancellationToken::new();
let worker_store = store.clone();
let mut runtime = CatalogBackgroundTask::start(&cancellation, |stop| super::super::rename_recovery::run(worker_store, stop));
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, pause.wait_started())
.await
.unwrap();
let publication_lock = default_table_bucket_publication_lock_path();
let before = backend.write_lock_acquisition_count("analytics", &publication_lock).await;
let other_store = store.clone();
let competing = tokio::spawn(async move { other_store.recover_pending_table_rename("analytics").await });
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async {
while backend.write_lock_acquisition_count("analytics", &publication_lock).await == before {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
cancellation.cancel();
assert!(
tokio::time::timeout(StdDuration::from_millis(10), runtime.join())
.await
.is_err()
);
assert!(!competing.is_finished());
pause.release();
runtime.join().await.unwrap().unwrap();
competing.await.unwrap().unwrap();
assert!(
store
.get_table_bucket("analytics")
.await
.unwrap()
.unwrap()
.active_rename_id
.is_none()
);
assert_eq!(
store
.read_entry::<TableRenameIntent>(RUSTFS_META_BUCKET, &intent_path)
.await
.unwrap()
.unwrap()
.0
.state,
TableRenameIntentState::Completed
);
}
#[tokio::test]
async fn rename_recovery_discovery_skips_bucket_children_and_advances_past_invalid_entries() {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
for bucket in ["first", "second"] {
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
let root = store
.paths
.table_bucket_entry_path(bucket)
.trim_end_matches(TABLE_BUCKET_ENTRY_FILE)
.to_string();
for index in 0..1100 {
backend
.seed_object(RUSTFS_META_BUCKET, &format!("{root}renames/{index}.json"), Vec::new())
.await;
}
}
let broken = store.paths.table_bucket_entry_path("broken");
backend.seed_object(RUSTFS_META_BUCKET, &broken, b"{}".to_vec()).await;
backend.reset_call_counts().await;
let mut cursor = None;
let mut found = BTreeSet::new();
let mut failures = 0;
for _ in 0..4 {
match store.next_table_bucket_for_rename_recovery(&mut cursor).await {
Ok(Some(entry)) => {
found.insert(entry.table_bucket);
}
Err(_) => {
failures += 1;
}
Ok(None) => break,
}
}
assert_eq!(found, BTreeSet::from(["first".to_string(), "second".to_string()]));
assert_eq!(failures, 1);
assert!(cursor.is_none());
assert_eq!(backend.list_call_count().await, 4);
}
#[tokio::test]
async fn drop_namespace_validates_table_and_view_states_across_pages() {
for view in [false, true] {
for state in [
TableCatalogEntryState::Active,
TableCatalogEntryState::Renaming,
TableCatalogEntryState::Deleting,
TableCatalogEntryState::Deleted,
] {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let namespace = Namespace::parse("sales").unwrap();
let table = IdentifierSegment::parse("orders").unwrap();
store.put_table_bucket(test_bucket_entry("analytics")).await.unwrap();
store
.create_namespace(test_namespace_entry("analytics", &namespace))
.await
.unwrap();
let path = if view {
store.paths.view_entry_path("analytics", &namespace, &table)
} else {
store.paths.table_entry_path("analytics", &namespace, &table)
};
let prefix = if view {
store.paths.view_entries_prefix("analytics", &namespace)
} else {
store.paths.table_entries_prefix("analytics", &namespace)
};
for index in 0..TABLE_CATALOG_LIST_MAX_KEYS {
backend
.seed_object(RUSTFS_META_BUCKET, &format!("{prefix}0000-spacer/{index:04}.json"), Vec::new())
.await;
}
let data = if view {
let mut entry = test_view_entry("analytics", &namespace, &table, "metadata.json".to_string());
entry.state = state.clone();
serde_json::to_vec(&entry).unwrap()
} else {
let mut entry = test_table_entry("analytics", &namespace, &table, "metadata.json".to_string());
entry.state = state.clone();
serde_json::to_vec(&entry).unwrap()
};
backend.seed_object(RUSTFS_META_BUCKET, &path, data).await;
let before = backend.read_object(RUSTFS_META_BUCKET, &path).await.unwrap();
let result = store.drop_namespace("analytics", "sales").await;
if state == TableCatalogEntryState::Deleted {
result.unwrap();
assert!(store.get_namespace("analytics", "sales").await.unwrap().is_none());
} else {
assert_matches!(result, Err(TableCatalogStoreError::Conflict(_)));
assert!(store.get_namespace("analytics", "sales").await.unwrap().is_some());
}
assert_eq!(backend.read_object(RUSTFS_META_BUCKET, &path).await.unwrap(), before);
}
}
}
#[tokio::test]
async fn drop_namespace_rejects_invalid_tombstones_and_read_failures() {
for view in [false, true] {
for failure in ["decode", "identity", "version", "read"] {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let namespace = Namespace::parse("sales").unwrap();
let table = IdentifierSegment::parse("orders").unwrap();
store.put_table_bucket(test_bucket_entry("analytics")).await.unwrap();
store
.create_namespace(test_namespace_entry("analytics", &namespace))
.await
.unwrap();
let (path, mut entry) = if view {
(
store.paths.view_entry_path("analytics", &namespace, &table),
serde_json::to_value(test_view_entry("analytics", &namespace, &table, "metadata.json".to_string())).unwrap(),
)
} else {
(
store.paths.table_entry_path("analytics", &namespace, &table),
serde_json::to_value(test_table_entry("analytics", &namespace, &table, "metadata.json".to_string())).unwrap(),
)
};
entry["state"] = serde_json::json!("DELETED");
if failure == "identity" {
entry["namespace"] = serde_json::json!("curated");
}
if failure == "version" {
entry["version"] = serde_json::json!(999);
}
let data = if failure == "decode" {
b"{broken".to_vec()
} else {
serde_json::to_vec(&entry).unwrap()
};
backend.seed_object(RUSTFS_META_BUCKET, &path, data).await;
if failure == "read" {
backend.fail_next_read(RUSTFS_META_BUCKET, &path).await;
}
assert!(store.drop_namespace("analytics", "sales").await.is_err());
assert!(store.get_namespace("analytics", "sales").await.unwrap().is_some());
}
}
}
#[tokio::test]
async fn rename_recovery_respects_restore_barriers_and_publication_lock_failure() {
let backend = TestCatalogObjectBackend::default();
interrupted_rename(&backend, "analytics", TableRenameIntentState::IndexPublished).await;
let store = ObjectTableCatalogStore::new(backend.clone());
let restore_path = store.paths.catalog_restore_state_path("analytics");
let restore = serde_json::json!({
"version": 1, "table_bucket": "analytics", "restore_epoch": Uuid::new_v4().to_string(),
"restore_id": "active-restore", "backup_id": "backup", "request_fingerprint": "a".repeat(64), "state": "PREPARED"
});
backend
.seed_object(RUSTFS_META_BUCKET, &restore_path, serde_json::to_vec(&restore).unwrap())
.await;
let before = maintenance_test_object_versions(&backend).await;
assert_matches!(
store.recover_pending_table_rename("analytics").await,
Err(TableCatalogStoreError::Unavailable(_))
);
assert_eq!(maintenance_test_object_versions(&backend).await, before);
backend
.state
.lock()
.await
.objects
.remove(&(RUSTFS_META_BUCKET.to_string(), restore_path));
*backend.fail_write_lock_path.lock().await = Some((
default_table_bucket_publication_lock_path(),
TableCatalogStoreError::Unavailable("injected publication lock failure".to_string()),
));
let before = maintenance_test_object_versions(&backend).await;
assert_matches!(
store.recover_pending_table_rename("analytics").await,
Err(TableCatalogStoreError::Unavailable(_))
);
assert_eq!(maintenance_test_object_versions(&backend).await, before);
store.recover_pending_table_rename("analytics").await.unwrap();
assert!(
store
.get_table_bucket("analytics")
.await
.unwrap()
.unwrap()
.active_rename_id
.is_none()
);
}
#[tokio::test(start_paused = true)]
async fn background_rename_recovery_backs_off_and_retries_without_client_mutations() {
let backend = TestCatalogObjectBackend::default();
let (_, intent_path) = interrupted_rename(&backend, "analytics", TableRenameIntentState::IndexPublished).await;
let store = ObjectTableCatalogStore::new(backend.clone());
backend.fail_next_read(RUSTFS_META_BUCKET, &intent_path).await;
let key = (RUSTFS_META_BUCKET.to_string(), intent_path);
let reads_before = backend.state.lock().await.read_attempts[&key];
let cancellation = CancellationToken::new();
let worker_store = store.clone();
let mut runtime = CatalogBackgroundTask::start(&cancellation, |stop| super::super::rename_recovery::run(worker_store, stop));
while backend.state.lock().await.read_attempts[&key] == reads_before {
tokio::task::yield_now().await;
}
tokio::task::yield_now().await;
let list_calls = backend.list_call_count().await;
tokio::time::advance(StdDuration::from_secs(6)).await;
while backend.list_call_count().await == list_calls {
tokio::task::yield_now().await;
}
assert_eq!(
backend.state.lock().await.read_attempts[&key],
reads_before + 1,
"the next scan must respect the retry delay"
);
assert!(
store
.get_table_bucket("analytics")
.await
.unwrap()
.unwrap()
.active_rename_id
.is_some()
);
tokio::task::yield_now().await;
tokio::time::advance(StdDuration::from_secs(6)).await;
wait_for_recovery(&store, "analytics").await;
runtime.request_stop();
runtime.join().await.unwrap().unwrap();
}
#[tokio::test]
async fn renamed_table_can_commit_and_back_up_after_dropping_its_source_namespace() {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let bucket = "analytics";
let namespace = Namespace::parse("sales").unwrap();
let table = IdentifierSegment::parse("orders").unwrap();
let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json");
let next = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json");
let mut source = test_table_entry(bucket, &namespace, &table, current.clone());
source.table_uuid = Uuid::new_v4().to_string();
let mut metadata = crate::table_catalog::test_support::table_metadata_json(&source.table_uuid, &source.warehouse_location);
metadata["format-version"] = serde_json::json!(source.format_version);
let metadata = serde_json::to_vec(&metadata).unwrap();
backend.seed_object(bucket, &current, metadata.clone()).await;
backend.seed_object(bucket, &next, metadata.clone()).await;
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
store
.create_namespace(test_namespace_entry(bucket, &namespace))
.await
.unwrap();
store
.create_namespace(test_namespace_entry(bucket, &Namespace::parse("curated").unwrap()))
.await
.unwrap();
store.create_table(source.clone()).await.unwrap();
store
.rename_table(bucket, "sales", "orders", "curated", "orders_v2")
.await
.unwrap();
store.drop_namespace(bucket, "sales").await.unwrap();
let resource = store
.resolve_table_metadata_data_plane_resource(bucket, &current)
.await
.unwrap()
.unwrap();
assert_eq!(resource.namespace, "curated");
assert_eq!(resource.table, "orders_v2");
assert_eq!(resource.table_id, source.table_id);
store
.commit_table(TableCommitRequest {
table_bucket: bucket.to_string(),
namespace: "curated".to_string(),
table: "orders_v2".to_string(),
commit_id: "post-rename-commit".to_string(),
idempotency_key: None,
operation: "update-metadata".to_string(),
expected_version_token: source.version_token,
expected_metadata_location: current.clone(),
new_metadata_location: next.clone(),
requirements: Vec::new(),
writer: None,
})
.await
.unwrap();
let renamed = store.load_table(bucket, "curated", "orders_v2").await.unwrap().unwrap();
assert_eq!(renamed.metadata_location, next);
assert_eq!(renamed.table_uuid, source.table_uuid);
assert_eq!(renamed.warehouse_location, source.warehouse_location);
let backup = store.create_object_catalog_backup(bucket, None).await.unwrap();
assert!(!backup.backup_id.is_empty());
assert_eq!(backend.read_object(bucket, &current).await.unwrap().unwrap().data, metadata);
assert_eq!(backend.read_object(bucket, &next).await.unwrap().unwrap().data, metadata);
}
#[tokio::test(start_paused = true)]
async fn background_rename_recovery_continues_beyond_one_scan_batch() {
let backend = TestCatalogObjectBackend::default();
interrupted_rename(&backend, "analytics", TableRenameIntentState::IndexPublished).await;
let store = ObjectTableCatalogStore::new(backend.clone());
let pending_path = store.paths.table_bucket_entry_path("analytics");
let mut preceding = 0;
for index in 0..1000 {
let bucket = format!("bucket-{index:04}");
if store.paths.table_bucket_entry_path(&bucket) < pending_path {
store.put_table_bucket(test_bucket_entry(&bucket)).await.unwrap();
preceding += 1;
if preceding == 33 {
break;
}
}
}
assert_eq!(preceding, 33);
backend.reset_call_counts().await;
let cancellation = CancellationToken::new();
let worker_store = store.clone();
let mut runtime = CatalogBackgroundTask::start(&cancellation, |stop| super::super::rename_recovery::run(worker_store, stop));
while backend.list_call_count().await < 32 {
tokio::task::yield_now().await;
}
tokio::task::yield_now().await;
assert!(
store
.get_table_bucket("analytics")
.await
.unwrap()
.unwrap()
.active_rename_id
.is_some()
);
tokio::time::advance(StdDuration::from_secs(6)).await;
wait_for_recovery(&store, "analytics").await;
runtime.request_stop();
runtime.join().await.unwrap().unwrap();
}
#[tokio::test]
async fn rename_recovery_rechecks_restore_after_waiting_for_publication() {
let backend = TestCatalogObjectBackend::default();
interrupted_rename(&backend, "analytics", TableRenameIntentState::IndexPublished).await;
let store = ObjectTableCatalogStore::new(backend.clone());
let lock_path = default_table_bucket_publication_lock_path();
let guard = backend.acquire_write_lock("analytics", &lock_path).await.unwrap();
backend.lock_attempts.lock().await.clear();
let recovery_store = store.clone();
let recovery = tokio::spawn(async move { recovery_store.recover_pending_table_rename("analytics").await });
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async {
loop {
if backend
.lock_attempts
.lock()
.await
.contains(&("analytics".to_string(), lock_path.clone()))
{
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("recovery reached the publication lock");
let restore_path = store.paths.catalog_restore_state_path("analytics");
let restore = serde_json::json!({
"version": 1, "table_bucket": "analytics", "restore_epoch": Uuid::new_v4().to_string(),
"restore_id": "active-restore", "backup_id": "backup", "request_fingerprint": "a".repeat(64), "state": "PREPARED"
});
backend
.seed_object(RUSTFS_META_BUCKET, &restore_path, serde_json::to_vec(&restore).unwrap())
.await;
let before = maintenance_test_object_versions(&backend).await;
drop(guard);
assert_matches!(
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, recovery)
.await
.unwrap()
.unwrap(),
Err(TableCatalogStoreError::Unavailable(_))
);
assert_eq!(maintenance_test_object_versions(&backend).await, before);
backend
.state
.lock()
.await
.objects
.remove(&(RUSTFS_META_BUCKET.to_string(), restore_path));
store.recover_pending_table_rename("analytics").await.unwrap();
assert!(
store
.get_table_bucket("analytics")
.await
.unwrap()
.unwrap()
.active_rename_id
.is_none()
);
}