Compare commits

..

3 Commits

Author SHA1 Message Date
唐小鸭 a7e5181e2e test: make app fixtures portable to native Windows
- Token- and cache-shaped keys use Windows-legal spellings at the same
  sort position, so pagination still runs without relaxing key admission.
- The TLS root test falls back to a directory junction only when a
  symlink needs a privilege the runner lacks, and asserts the root is a link.
- Snapshot timestamps step by 1 us, which Windows SystemTime can represent.
- The receiver disk URL uses a slash-led path instead of a raw drive path.
2026-11-07 22:26:09 +08:00
唐小鸭 d45468ecab fix(ecstore): name the target record when a migration write fails
The existence probe reports a blocked record with its path, but the
commit path returns bare disk errors. On Windows a regular file in the
record's place reads as missing, so migration only fails at the write,
and startup reported "File access denied" without naming the record.
2026-11-07 22:26:09 +08:00
唐小鸭 4a69fd2e2e test: bypass the disk independence check in path-only topology tests
These tests parse endpoints whose local paths are never created. Unix
leaves missing paths out of the physical-disk independence check, but
Windows resolves them to their volume, so every disk counted as one
shared device and the parsing assertions never ran. The check keeps its
own coverage in the dedicated shared-volume tests.
2026-11-07 22:26:09 +08:00
104 changed files with 24020 additions and 17880 deletions
+2 -2
View File
@@ -110,7 +110,7 @@ threads-required = "num-test-threads"
# Reserve capacity so sibling disk-heavy tests cannot consume its 2s
# registration budget; retain the queued backup and deadlock assertions.
[[profile.default.overrides]]
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_object_registration_completes_while_other_bucket_backup_waits)'
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_strong_registration_completes_while_other_bucket_backup_waits)'
threads-required = "num-test-threads"
[[profile.default.overrides]]
@@ -410,7 +410,7 @@ threads-required = "num-test-threads"
# Reserve capacity so sibling disk-heavy tests cannot consume its 2s
# registration budget; retain the queued backup and deadlock assertions.
[[profile.ci.overrides]]
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_object_registration_completes_while_other_bucket_backup_waits)'
filter = 'package(rustfs) & test(=admin::handlers::table_catalog::tests::cold_strong_registration_completes_while_other_bucket_backup_waits)'
threads-required = "num-test-threads"
[[profile.ci.overrides]]
+2 -3
View File
@@ -32,9 +32,8 @@ pub const ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD: &str = "RUSTFS_OBJECT_MEDIUM_
/// Environment variable name for maximum concurrent disk reads for object operations.
/// - Purpose: Limit the number of concurrent disk read operations for object reads to prevent I/O saturation.
/// - Unit: request count (usize) for GET; estimated drive fanout for native LIST.
/// - Semantics: GET bodies and native LIST walks use separate per-instance budgets of this size.
/// A LIST consumes up to one budget, including its disk-producer cleanup. `0` disables both caps.
/// - Unit: request count (usize).
/// - Semantics: Throttling disk reads helps maintain overall system responsiveness under load.
/// - Example: `export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=16`
/// - Note: This setting may interact with OS-level I/O scheduling and should be tuned based on hardware capabilities.
pub const ENV_OBJECT_MAX_CONCURRENT_DISK_READS: &str = "RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS";
@@ -114,10 +114,10 @@ fn staged_commit(metadata: &Value, location: &str) -> Value {
})
}
async fn location_claim_preserves_other_users_objects_after_drop_and_restart() -> TestResult {
async fn location_claim_preserves_other_users_objects_after_drop_and_restart(backing: &str) -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
let server_env = [("RUSTFS_TABLE_CATALOG_BACKING", "object")];
let server_env = [("RUSTFS_TABLE_CATALOG_BACKING", backing)];
env.start_rustfs_server_without_cleanup_with_env(&[("RUSTFS_TABLE_CATALOG_BACKING", "object")])
.await?;
env.wait_for_server_ready().await?;
@@ -147,6 +147,39 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart() -
StatusCode::OK,
)
.await?;
if backing == "durable-strong" {
catalog_request(
&env,
&root,
Method::POST,
&format!("/iceberg/v1/{bucket}/catalog/warehouse-index/backfill"),
None,
StatusCode::NO_CONTENT,
)
.await?;
let preflight = catalog_request(
&env,
&root,
Method::GET,
&format!("/iceberg/v1/{bucket}/catalog/migration"),
None,
StatusCode::OK,
)
.await?;
assert_eq!(preflight["blockers"], json!([]), "{preflight}");
let migration = catalog_request(
&env,
&root,
Method::POST,
&format!("/iceberg/v1/{bucket}/catalog/migration"),
None,
StatusCode::OK,
)
.await?;
assert_eq!(migration["ready_to_enable_durable_strong"], json!(true), "{migration}");
env.restart_server_preserving_data(vec![], &server_env).await?;
env.wait_for_server_ready().await?;
}
admin_create_user(&env, &creator.0, &creator.1).await?;
admin_create_user(&env, &reader.0, &reader.1).await?;
let create_statement = json!({
@@ -308,7 +341,7 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart() -
StatusCode::FORBIDDEN,
)
.await?;
{
if backing == "object" {
catalog_request(
&env,
&creator,
@@ -375,5 +408,10 @@ async fn location_claim_preserves_other_users_objects_after_drop_and_restart() -
#[tokio::test]
async fn object_catalog_location_claim_preserves_objects_after_drop_and_restart() -> TestResult {
location_claim_preserves_other_users_objects_after_drop_and_restart().await
location_claim_preserves_other_users_objects_after_drop_and_restart("object").await
}
#[tokio::test]
async fn durable_strong_catalog_location_claim_preserves_objects_after_drop_and_restart() -> TestResult {
location_claim_preserves_other_users_objects_after_drop_and_restart("durable-strong").await
}
+38 -3
View File
@@ -17,7 +17,7 @@
use crate::bucket::metadata::BUCKET_METADATA_FILE;
use crate::bucket::replication::ReplicationMigrationBridge;
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::error::{Error, Result, is_err_strict_not_found, is_err_strict_volume_not_found};
use crate::error::{Error, Result, is_err_strict_not_found, is_err_strict_volume_not_found, to_object_err};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::storage_api_contracts::{
bucket::{BucketInfo, BucketOperations, BucketOptions},
@@ -334,6 +334,14 @@ async fn migration_target_exists<S: EcstoreObjectOperations>(store: &S, path: &s
}
}
/// Names the target record in a failed migration write, as the existence
/// probe's error already does. The commit path returns bare disk errors, and on
/// Windows a blocked record is first seen here: reading through a regular-file
/// ancestor reports the record missing instead of refusing access to it.
fn migration_write_error(err: Error, path: &str) -> Error {
to_object_err(err, vec![RUSTFS_META_BUCKET, path])
}
async fn migrate_one_if_missing<S>(
store: Arc<S>,
opts: &ObjectOptions,
@@ -369,7 +377,10 @@ where
.unwrap_or(data);
let mut put_data = PutObjReader::from_vec(data);
store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await?;
store
.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts)
.await
.map_err(|err| migration_write_error(err, path))?;
info!("Migrated {label}");
Ok(())
}
@@ -481,7 +492,10 @@ where
Err(_) => return Err(MigrationMetadataError::Incompatible(path.to_owned()).into()),
};
let mut put_data = PutObjReader::from_vec(data);
store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await?;
store
.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts)
.await
.map_err(|err| migration_write_error(err, path))?;
info!("Migrated IAM config: {path}");
total_migrated += 1;
}
@@ -531,6 +545,27 @@ mod tests {
));
}
#[test]
fn migration_write_errors_name_the_target_record_without_fragmenting_groups() {
use super::{Error, migration_write_error};
let first = migration_write_error(Error::FileAccessDenied, "buckets/first/.metadata.bin");
let second = migration_write_error(Error::FileAccessDenied, "config/iam/format.json");
assert!(matches!(first, Error::PrefixAccessDenied(..)), "unexpected error: {first:?}");
assert!(
first.to_string().contains(".rustfs.sys/buckets/first/.metadata.bin"),
"the startup error must name the blocked record: {first}"
);
assert_eq!(first, second, "record paths must not fragment error grouping");
let unrelated = migration_write_error(Error::other("disk offline"), "buckets/first/.metadata.bin");
assert_eq!(
unrelated,
Error::other("disk offline"),
"errors without a record form pass through unchanged"
);
}
#[test]
fn migration_errors_group_by_cause_and_retain_typed_record_context() {
use super::{Error, MigrationMetadataError};
@@ -217,8 +217,6 @@ impl Drop for NotifyOnDrop {
#[derive(Default)]
pub struct ListPathRawOptions {
/// Retain native listing admission through producer cancellation/draining.
pub read_budget: Option<Arc<tokio::sync::OwnedSemaphorePermit>>,
pub disks: Vec<Option<DiskStore>>,
pub fallback_disks: Vec<Option<DiskStore>>,
pub bucket: String,
@@ -257,7 +255,6 @@ pub struct ListPathRawOptions {
impl Clone for ListPathRawOptions {
fn clone(&self) -> Self {
Self {
read_budget: self.read_budget.clone(),
disks: self.disks.clone(),
fallback_disks: self.fallback_disks.clone(),
bucket: self.bucket.clone(),
@@ -392,9 +389,7 @@ async fn list_path_raw_inner(
let disk_io_context_clone = disk_io_context.clone();
let (rd, wr) = tokio::io::duplex(64 * 1024);
readers.push(MetacacheReader::new(rd));
let read_budget = opts.read_budget.clone();
let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
let _read_budget = read_budget;
let mut wr = PublishedBytesWriter::new(wr);
#[cfg(test)]
if let Some(observer) = &test_disk_io_context_observed {
@@ -1902,42 +1897,6 @@ mod tests {
assert_eq!(seen.lock().expect("seen mutex poisoned").as_slice(), &["bucket/object".to_string()]);
}
#[tokio::test]
async fn list_read_budget_is_retained_by_raw_producer_until_cancellation() {
let semaphore = Arc::new(tokio::sync::Semaphore::new(4));
let grant = Arc::new(semaphore.clone().acquire_many_owned(4).await.expect("listing grant"));
let weak_grant = Arc::downgrade(&grant);
let entered = Arc::new(tokio::sync::Notify::new());
let stopped = Arc::new(tokio::sync::Notify::new());
let worker = tokio::spawn(list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
read_budget: Some(grant),
disks: vec![None],
min_disks: 1,
test_reader_behaviors: vec![TestReaderBehavior::StallWithSignals {
entered: entered.clone(),
stopped: stopped.clone(),
}],
..Default::default()
},
));
timeout(Duration::from_secs(5), entered.notified())
.await
.expect("producer entered");
assert!(weak_grant.strong_count() >= 2, "producer must retain a grant independently of the parent");
assert_eq!(semaphore.available_permits(), 0);
worker.abort();
assert!(worker.await.expect_err("parent cancelled").is_cancelled());
timeout(Duration::from_secs(5), stopped.notified())
.await
.expect("producer stopped");
let _next = timeout(Duration::from_secs(5), semaphore.acquire_many(4))
.await
.expect("cancelled listing releases all permits")
.expect("budget stays open");
}
#[tokio::test]
async fn list_path_raw_aborts_unresponsive_producer_after_quorum_eof() {
let result = timeout(
@@ -423,15 +423,11 @@ impl RemoteClient {
/// peer toward the offline threshold. See issue #4567.
fn is_transport_failure(status: &tonic::Status) -> bool {
use tonic::Code;
// A valid stale-epoch request is rejected before execution. Its signed
// response refreshes the peer epoch on the existing healthy channel.
let epoch_rejection = status.code() == Code::Unavailable && status.message() == "RPC boot epoch changed";
std::error::Error::source(status).is_some()
|| (!epoch_rejection
&& matches!(
status.code(),
Code::Unavailable | Code::DeadlineExceeded | Code::Unknown | Code::Cancelled
))
|| matches!(
status.code(),
Code::Unavailable | Code::DeadlineExceeded | Code::Unknown | Code::Cancelled
)
}
async fn evict_connection(&self, op: &'static str, reason: &str, resource_summary: &str) {
@@ -2831,37 +2827,6 @@ mod tests {
}
}
#[tokio::test]
async fn stale_boot_epoch_rejections_do_not_open_lock_request_breaker() {
let addr = "http://stale-boot-epoch-lock-breaker:9000";
let client = RemoteClient::new(addr.to_string());
reset_lock_peer_health_for_test(addr);
for _ in 0..LOCK_RPC_REQUEST_BACKOFF_THRESHOLD {
let error = client
.execute_rpc::<(), _>(
"lock",
"bucket/new-object",
Duration::from_secs(1),
async { Err(tonic::Status::unavailable("RPC boot epoch changed")) },
RpcCleanup::new(None, None, None),
)
.await
.expect_err("a stale epoch still rejects the operation before execution");
assert!(error.to_string().contains("RPC boot epoch changed"));
}
let health = lock_peer_health_for_test(addr);
assert_eq!(health.request_consecutive_timeouts, 0);
assert!(health.request_backoff_until.is_none());
assert!(health.last_eviction.is_none());
let admission = client
.admit_lock_request("lock")
.expect("authenticated epoch refresh may proceed immediately");
client.release_lock_request("lock", Some(admission));
reset_lock_peer_health_for_test(addr);
}
#[test]
#[serial_test::serial]
fn test_remote_client_rpc_timeout_honors_configured_deadline() {
+6 -26
View File
@@ -2208,14 +2208,9 @@ fn ensure_decommission_capacity_reservations_available(
continue;
}
metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1);
let detail = format!(
return Err(decommission_capacity_blocked_error(format!(
"target pool {target_pool_index} reservation requires {required} physical bytes, but only {available} bytes remain during {phase}"
);
return Err(if growth_pool.is_some() {
decommission_target_no_longer_fits_error(target_pool_index, detail)
} else {
decommission_capacity_blocked_error(detail)
});
)));
}
Ok(())
}
@@ -2346,13 +2341,6 @@ fn grow_decommission_target_reservation(
let existing_remaining = existing_target
.map(|target| target.remaining_reserved_physical_bytes(reservation.temporary_copies))
.unwrap_or_default();
let existing_reserved = existing_target
.map(|target| target.reserved_physical_bytes)
.unwrap_or_default();
let allocation_limit = existing_target
.map(|target| target.physical_free_at_reservation)
.unwrap_or_default()
.max(target_capacity.physical_free);
if required_peak <= existing_remaining {
return Ok(false);
}
@@ -2388,15 +2376,6 @@ fn grow_decommission_target_reservation(
.predicted_physical_bytes
.saturating_add(reservation.temporary_physical_bytes);
let allocation_growth = reservation.remaining_peak_physical_bytes().saturating_sub(current_peak);
let required_allocation = existing_reserved.saturating_add(allocation_growth);
if required_allocation > allocation_limit {
return Err(decommission_target_no_longer_fits_error(
target_pool_index,
format!(
"target pool {target_pool_index} reservation requires {required_allocation} physical bytes, but its observed allocation limit is {allocation_limit} bytes"
),
));
}
if let Some(target) = reservation
.targets
.iter_mut()
@@ -37911,9 +37890,10 @@ mod pools_tests {
);
let before = serde_json::to_value(&meta).expect("snapshot reservation before rejected growth");
let shortage = vec![initial[0], DecommissionPoolCapacityInfo::for_test(1, layout, 43, 64, 21)];
let error = grow_decommission_target_reservation(&mut meta, &shortage, 0, 1, 32, uuid::Uuid::new_v4(), now)
.expect_err("real free-space shortage still rejects growth");
assert_eq!(super::decommission_capacity_stale_target_index(&error), Some(1));
assert!(
grow_decommission_target_reservation(&mut meta, &shortage, 0, 1, 32, uuid::Uuid::new_v4(), now).is_err(),
"real free-space shortage still rejects growth"
);
assert_eq!(serde_json::to_value(&meta).expect("snapshot after rejected growth"), before);
}
+4 -35
View File
@@ -1616,7 +1616,7 @@ mod decommission_lock_order_tests {
run_routine_to_a_deferral_pause(
&store,
1,
"target pool 1 no longer fits the selected mutation: target pool 1 reservation requires",
"target pool 1 no longer fits the selected mutation: source pool 0 reservation estimate is exhausted",
)
.await;
});
@@ -1647,7 +1647,7 @@ mod decommission_lock_order_tests {
run_routine_to_a_deferral_pause(
&store,
1,
"target pool 1 no longer fits the selected mutation: target pool 1 reservation requires",
"target pool 1 no longer fits the selected mutation: source pool 0 reservation estimate is exhausted",
)
.await;
});
@@ -6062,12 +6062,6 @@ mod decommission_lock_order_tests {
)
.await
.expect("read the next source object for the multipart capacity oracle");
// Reconciled bytes remain charged. With no physical room for a new
// reservation, a distinct mutation must fail instead of reusing them.
let mut exhausted_capacity = capacity_snapshot();
exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
let next_result = tokio::time::timeout(
Duration::from_secs(30),
data_movement::migrate_decommission_object(
@@ -7833,12 +7827,6 @@ mod decommission_lock_order_tests {
.expect("same-mutation retry should publish a readable target");
assert_eq!(retry_info.size, i64::try_from(body.len()).expect("test body size fits i64"));
// Reconciled bytes remain charged. With no physical room for a new
// reservation, a distinct mutation must fail instead of reusing them.
let mut exhausted_capacity = capacity_snapshot();
exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
let next_object = "data-movement-put-after-retry.bin";
let next_version_id = uuid::Uuid::new_v4().to_string();
let mut next_opts = ObjectOptions {
@@ -7853,11 +7841,7 @@ mod decommission_lock_order_tests {
let next_result = other_store
.put_object_for_data_movement(RUSTFS_META_BUCKET, next_object, &mut next_data, &next_opts, None)
.await;
let next_error = match next_result {
Err(error) | Ok((_, Err(error))) => error,
Ok((_, Ok(_))) => panic!("a different mutation must not reuse A's recovered capacity"),
};
assert!(next_error.to_string().contains("decommission_capacity_blocked"), "{next_error}");
assert!(next_result.is_err(), "a different mutation must not reuse A's recovered capacity");
let next_target_err = other_store.pools[2]
.get_object_info(
RUSTFS_META_BUCKET,
@@ -9945,12 +9929,6 @@ mod decommission_lock_order_tests {
)
.await
.expect("read the next source object for the capacity oracle");
// Reconciled bytes remain charged. With no physical room for a new
// reservation, a distinct mutation must fail instead of reusing them.
let mut exhausted_capacity = capacity_snapshot();
exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 0, target_total, target_total);
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
let next_result = tokio::time::timeout(
Duration::from_secs(30),
data_movement::migrate_decommission_object(
@@ -10212,20 +10190,11 @@ mod decommission_lock_order_tests {
assert_eq!(reconciled_reservation.pending_target_physical_bytes, 0);
assert_eq!(reconciled_reservation.consumed_target_physical_bytes, 1);
// Reconciled bytes remain charged. With no physical room for a new
// reservation, a distinct mutation must fail instead of reusing them.
let mut exhausted_capacity = capacity_snapshot();
exhausted_capacity[2] = DecommissionPoolCapacityInfo::for_test(2, layout, 1, 2, 1);
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![exhausted_capacity]);
let next_after_reconcile = other_store
.delete_object(&bucket, next_object, next_opts)
.await
.expect_err("the consumed exact-fit capacity must reject the next marker");
assert!(
next_after_reconcile.to_string().contains("decommission_capacity_blocked"),
"{next_after_reconcile}"
);
assert!(next_after_reconcile.to_string().contains("decommission_capacity_blocked"));
let next_target_err = other_store.pools[2]
.get_object_info(
&bucket,
+24
View File
@@ -2149,6 +2149,15 @@ mod test {
}
}
/// Runs endpoint parsing cases whose local paths do not exist. Unix leaves
/// missing paths out of the physical-disk independence check, but Windows
/// resolves them to their volume, so paths such as `/d1` and `/d2` count as
/// one shared device there. These cases cover parsing, not disk
/// independence, which has its own tests below.
async fn with_disk_independence_check_bypassed<F: std::future::Future<Output = ()>>(cases: F) {
async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true"))], cases).await;
}
#[test]
fn endpoint_parse_errors_do_not_echo_url_credentials() {
let err = Endpoints::try_from(["http://:topsecret@server/path"].as_slice()).unwrap_err();
@@ -2157,8 +2166,13 @@ mod test {
assert!(!err.to_string().contains("topsecret"));
}
#[serial]
#[tokio::test]
async fn explicit_local_endpoint_host_selects_only_exact_host_and_server_port() {
with_disk_independence_check_bypassed(explicit_local_endpoint_host_selects_only_exact_host_and_server_port_cases()).await;
}
async fn explicit_local_endpoint_host_selects_only_exact_host_and_server_port_cases() {
let args = vec![
"http://rustfs-0.rustfs-headless.ns.svc.cluster.local./data0",
"http://rustfs-0.rustfs-headless.ns.svc.cluster.local.:9443/data1",
@@ -2921,8 +2935,13 @@ mod test {
}
}
#[serial]
#[tokio::test]
async fn test_create_pool_endpoints() {
with_disk_independence_check_bypassed(test_create_pool_endpoints_cases()).await;
}
async fn test_create_pool_endpoints_cases() {
#[derive(Default)]
struct TestCase<'a> {
num: usize,
@@ -3541,8 +3560,13 @@ mod test {
(urls, local_flags)
}
#[serial]
#[tokio::test]
async fn test_create_server_endpoints() {
with_disk_independence_check_bypassed(test_create_server_endpoints_cases()).await;
}
async fn test_create_server_endpoints_cases() {
let test_cases = [
// Invalid input.
("", vec![], false),
+1 -132
View File
@@ -58,7 +58,7 @@ use std::sync::{
Arc, OnceLock,
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
};
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, OwnedSemaphorePermit, RwLock, Semaphore};
use tokio::sync::{Mutex, Notify, OnceCell, OwnedRwLockReadGuard, RwLock};
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker;
@@ -187,38 +187,6 @@ impl DecommissionCountLedger {
}
}
#[derive(Debug)]
struct ListReadBudget {
permits: Arc<Semaphore>,
capacity: usize,
}
impl ListReadBudget {
fn new(capacity: usize) -> Self {
let capacity = capacity
.min(Semaphore::MAX_PERMITS)
.min(usize::try_from(u32::MAX).unwrap_or(usize::MAX));
Self {
permits: Arc::new(Semaphore::new(capacity)),
capacity,
}
}
async fn acquire(&self, fanout: usize) -> Result<Option<Arc<OwnedSemaphorePermit>>, tokio::sync::AcquireError> {
if self.capacity == 0 {
return Ok(None);
}
// A single walk must remain admissible even when the topology exceeds
// the configured budget. Tokio accepts at most u32::MAX in one grant.
let weight = u32::try_from(fanout.max(1).min(self.capacity)).unwrap_or(u32::MAX);
self.permits
.clone()
.acquire_many_owned(weight)
.await
.map(|permit| Some(Arc::new(permit)))
}
}
/// Runtime state owned by a single `ECStore` instance.
///
/// This is intentionally minimal in the first migration slice; subsequent
@@ -233,7 +201,6 @@ pub struct InstanceContext {
/// drift out of sync). Stored as one value so the three predicates can
/// never observe a torn intermediate state.
erasure_kind: RwLock<SetupType>,
list_read_budget: OnceLock<ListReadBudget>,
/// This instance's namespace lock manager (Phase 5 Slice 3, backlog#939).
///
/// Owned per-instance so two instances no longer share a lock namespace
@@ -387,24 +354,6 @@ impl InstanceContext {
Self::with_lock_manager(Arc::new(GlobalLockManager::new()))
}
/// Bound native listing fanout independently of GET body permits. A walk
/// keeps its shared grant until all of its disk producers have terminated.
/// Acquire before creating the merge channels or taking object locks.
pub(crate) async fn acquire_list_read_budget(
&self,
fanout: usize,
) -> Result<Option<Arc<OwnedSemaphorePermit>>, tokio::sync::AcquireError> {
self.list_read_budget
.get_or_init(|| {
ListReadBudget::new(rustfs_utils::get_env_usize(
rustfs_config::ENV_OBJECT_MAX_CONCURRENT_DISK_READS,
rustfs_config::DEFAULT_OBJECT_MAX_CONCURRENT_DISK_READS,
))
})
.acquire(fanout)
.await
}
/// Build a context bound to a specific lock manager.
///
/// [`bootstrap_ctx`] uses this to alias the process-global lock manager so
@@ -413,7 +362,6 @@ impl InstanceContext {
fn with_lock_manager(lock_manager: Arc<GlobalLockManager>) -> Self {
Self {
erasure_kind: RwLock::new(SetupType::Unknown),
list_read_budget: OnceLock::new(),
lock_manager,
region: OnceLock::new(),
deployment_id: OnceLock::new(),
@@ -1163,85 +1111,6 @@ pub fn bootstrap_ctx() -> Arc<InstanceContext> {
mod tests {
use super::*;
#[tokio::test]
async fn list_read_budget_weights_and_clamps_fanout() {
let budget = ListReadBudget::new(8);
let first = budget.acquire(3).await.expect("first walk").expect("bounded grant");
assert_eq!(budget.permits.available_permits(), 5);
let mut blocked = std::pin::pin!(budget.acquire(6));
assert!(futures::poll!(&mut blocked).is_pending());
drop(first);
let second = blocked.await.expect("queued walk").expect("bounded grant");
assert_eq!(budget.permits.available_permits(), 2);
drop(second);
let oversized = budget
.acquire(usize::MAX)
.await
.expect("oversized walk")
.expect("bounded grant");
assert_eq!(budget.permits.available_permits(), 0);
drop(oversized);
assert_eq!(budget.permits.available_permits(), 8);
}
#[tokio::test]
async fn list_read_budget_zero_preserves_unbounded_configuration() {
let budget = ListReadBudget::new(0);
assert!(budget.acquire(usize::MAX).await.expect("disabled budget").is_none());
}
#[tokio::test]
async fn list_read_budget_cancelled_waiter_returns_partial_grant() {
let budget = ListReadBudget::new(4);
let held = budget.acquire(3).await.expect("first walk");
{
let mut waiter = std::pin::pin!(budget.acquire(4));
assert!(futures::poll!(&mut waiter).is_pending());
}
assert_eq!(budget.permits.available_permits(), 1);
drop(held);
assert!(budget.acquire(4).await.expect("next walk").is_some());
}
#[tokio::test]
async fn list_read_budget_shared_grant_survives_parent_drop() {
let budget = ListReadBudget::new(4);
let parent = budget.acquire(4).await.expect("first walk").expect("bounded grant");
let producer = parent.clone();
drop(parent);
assert_eq!(budget.permits.available_permits(), 0);
drop(producer);
assert_eq!(budget.permits.available_permits(), 4);
}
#[tokio::test]
async fn list_read_budget_is_shared_by_context_and_isolated_between_instances() {
let first = Arc::new(InstanceContext::new());
first.list_read_budget.set(ListReadBudget::new(4)).expect("fresh context");
let same = first.clone();
let other = InstanceContext::new();
other.list_read_budget.set(ListReadBudget::new(4)).expect("fresh context");
let held = first.acquire_list_read_budget(4).await.expect("first context");
let mut waiter = std::pin::pin!(same.acquire_list_read_budget(1));
assert!(futures::poll!(&mut waiter).is_pending());
assert!(
other
.acquire_list_read_budget(4)
.await
.expect("independent context")
.is_some()
);
drop(held);
assert!(waiter.await.expect("shared context resumed").is_some());
}
#[test]
fn list_read_budget_extreme_configuration_is_bounded_without_panicking() {
let budget = ListReadBudget::new(usize::MAX);
assert!(budget.capacity <= Semaphore::MAX_PERMITS);
assert!(u32::try_from(budget.capacity).is_ok());
}
#[test]
fn decommission_count_ledger_bounds_retained_entries() {
let mut ledger = super::DecommissionCountLedger::default();
-68
View File
@@ -368,9 +368,6 @@ fn log_list_path_finished(
#[derive(Debug, Default, Clone)]
pub struct ListPathOptions {
pub caller: ListObjectsGatherCaller,
/// Internal native-walk admission, shared with every disk producer so
/// aborting an HTTP request cannot release its budget ahead of cleanup.
pub read_budget: Option<Arc<tokio::sync::OwnedSemaphorePermit>>,
pub id: Option<String>,
// Bucket of the listing.
@@ -1481,11 +1478,6 @@ impl ECStore {
if o.transient {
o.create = false;
}
let fanout = self.pools.iter().fold(0usize, |total, pool| {
total.saturating_add(pool.disk_set.len().saturating_mul(pool.set_drive_count))
});
o.read_budget = self.ctx.acquire_list_read_budget(fanout).await.map_err(Error::other)?;
let log_context = ListPathLogContext::from_options(&o);
// cancel channel
@@ -2886,12 +2878,6 @@ impl SetDisks {
if o.transient {
o.create = false;
}
o.read_budget = self
.instance_ctx()
.acquire_list_read_budget(self.set_drive_count)
.await
.map_err(Error::other)?;
let log_context = ListPathLogContext::from_options(&o);
let cancel = CancellationToken::new();
@@ -2963,7 +2949,6 @@ impl SetDisks {
let result = list_path_raw_with_claim_tracker(
rx,
ListPathRawOptions {
read_budget: opts.read_budget.clone(),
disks: disks.into_iter().map(Some).collect(),
bucket: opts.bucket,
path: opts.base_dir,
@@ -3122,7 +3107,6 @@ impl SetDisks {
let result = list_path_raw_with_outcome(
rx,
ListPathRawOptions {
read_budget: opts.read_budget.clone(),
disks: disks.into_iter().map(Some).collect(),
fallback_disks: fallback_disks.into_iter().map(Some).collect(),
bucket: opts.bucket,
@@ -6804,58 +6788,6 @@ mod test {
(dirs, store)
}
#[tokio::test]
async fn list_read_budget_bounds_store_and_set_entrypoints_and_recovers_after_errors() {
let (_dirs, store) = single_set_store(4).await;
let held = store
.ctx
.acquire_list_read_budget(usize::MAX)
.await
.expect("occupy native budget");
let semaphore = held.as_ref().expect("bounded instance budget").semaphore();
let owners = Arc::strong_count(semaphore);
let opts = ListPathOptions {
bucket: "list-budget-absent-bucket".into(),
limit: 1,
authoritative: true,
..Default::default()
};
let set = store.pools[0].disk_set[0].clone();
set.disks.write().await.fill(None);
{
let mut store_walk = std::pin::pin!(store.clone().list_path(&opts));
let mut set_walk = std::pin::pin!(set.clone().list_path_result(&opts));
assert!(futures::poll!(&mut store_walk).is_pending(), "store walk must await its budget");
assert_eq!(Arc::strong_count(semaphore), owners + 1, "store must queue on the shared budget");
assert!(futures::poll!(&mut set_walk).is_pending(), "set walk shares the same instance budget");
assert_eq!(Arc::strong_count(semaphore), owners + 2, "set must queue on the same budget");
let empty = store
.clone()
.list_path(&ListPathOptions {
limit: 0,
..opts.clone()
})
.await
.expect("empty page needs no disk admission");
assert!(empty.exhausted);
}
assert_eq!(Arc::strong_count(semaphore), owners, "cancelled entry points release the queue ownership");
drop(held);
let result = timeout(Duration::from_secs(5), store.clone().list_path(&opts))
.await
.expect("failed walk terminates");
assert_eq!(
result.err(),
Some(DiskError::ErasureReadQuorum.into()),
"offline drives retain their quorum error"
);
let next = timeout(Duration::from_secs(5), store.ctx.acquire_list_read_budget(usize::MAX))
.await
.expect("failed and cancelled walks release their grants")
.expect("budget remains open");
assert!(next.is_some());
}
fn versioned_object_metadata(bucket: &str, name: &str, versions: &[(u128, i64)], slot: usize) -> Bytes {
let mut meta = FileMeta::new();
for (id, modified) in versions {
+1 -1
View File
@@ -58,7 +58,7 @@ pub enum Error {
TaskExecutionFailed { message: String },
/// Only the qualified MRF Deep check may produce this terminal result.
#[error("Independent verification unavailable for legacy object {target}; durable MRF responsibility retained")]
#[error("Heal task execution failed: Missing exact storage proof for durable MRF repair {target}")]
LegacyMrfUnverified { target: String },
#[error("Replacement failure could not be persisted: {failure}; persistence error: {persistence}")]
-17
View File
@@ -292,20 +292,6 @@ impl HealManager {
"Heal scheduler task completed"
);
}
Err(e @ Error::LegacyMrfUnverified { .. }) => {
warn!(
target: "rustfs::heal::manager",
event = EVENT_HEAL_SCHEDULER_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "verification_unavailable",
task_id,
heal_type = %task_type_label_for_spawn,
set = %task_set_label_for_spawn,
detail = %e,
"Heal attempt completed without independent verification"
);
}
Err(e) => {
let will_retry = retry_request.is_some();
if will_retry {
@@ -497,9 +483,6 @@ impl HealManager {
// update statistics
let mut stats = statistics_clone.write().await;
match completed_status {
HealTaskStatus::Completed if matches!(&result, Err(Error::LegacyMrfUnverified { .. })) => {
stats.update_unverified_task_completion();
}
HealTaskStatus::Completed => {
stats.update_task_completion(true);
stats.add_healed_objects(completed_progress.objects_healed, completed_progress.bytes_processed);
-63
View File
@@ -1106,22 +1106,6 @@ impl HealStorageAPI for MockStorage {
if bucket.starts_with("legacy-ambiguity-scheduler-") {
return Err(Error::Storage(EcstoreError::DeleteRecoveryLegacyAmbiguous));
}
if bucket == "legacy-held-pending-scheduler" {
let drives = vec![rustfs_madmin::heal_commands::HealDriveInfo {
state: "ok".to_owned(),
..Default::default()
}];
return Ok((
HealResultItem {
legacy_identity_unverified: true,
detail: rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL.to_owned(),
before: rustfs_madmin::heal_commands::Infos { drives: drives.clone() },
after: rustfs_madmin::heal_commands::Infos { drives },
..Default::default()
},
None,
));
}
if bucket == "completed-retention-failed" {
return Err(Error::TaskExecutionFailed {
message: "retention fixture failure".to_string(),
@@ -2169,53 +2153,6 @@ fn only_typed_mrf_result_can_pause_unverified_legacy() {
}
}
#[tokio::test(flavor = "current_thread")]
async fn mrf_legacy_completed_attempt_retains_responsibility_without_failed_statistics() {
use crate::heal::failure_logging::test_support::FailureLogCapture;
let capture = FailureLogCapture::default();
let _subscriber = tracing::subscriber::set_default(capture.clone());
let (target, _) = unverified_legacy_fixture("legacy-held-pending-scheduler");
let bucket = target.bucket.clone();
let anchor = target.durable_anchor.clone().expect("durable MRF anchor");
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
let mut request = HealRequest::object(bucket.to_string(), target.object.to_string(), None);
request.source = HealRequestSource::Mrf;
request.expected_mrf_bucket_incarnation_id = Some(anchor.bucket_incarnation_id);
request.options.pool_index = Some(1);
request.options.set_index = Some(2);
let receipt = manager
.submit_mrf_heal_request_with_receipt_and_identity(request, target)
.await
.expect("admit durable MRF check");
assert_eq!(receipt.result, HealAdmissionResult::Accepted);
process_manager_queue_once(&manager).await;
tokio::time::timeout(Duration::from_secs(2), async {
while manager.get_statistics().await.unverified_tasks == 0 {
tokio::task::yield_now().await;
}
})
.await
.expect("publish completed unverified attempt");
let stats = manager.get_statistics().await;
assert_eq!(stats.unverified_tasks, 1);
assert_eq!(stats.failed_tasks, 0);
assert_eq!(stats.successful_tasks, 0);
assert_eq!(stats.total_objects_healed, 0);
let report = manager.get_task_report(&receipt.task_id).await.expect("retained task report");
assert_eq!(report.status, HealTaskStatus::Completed);
let outcome = report.outcome.expect("retained unknown outcome");
assert_eq!(outcome.counters.unknown, 1);
assert_eq!(outcome.counters.failed, 0);
assert!(rustfs_common::mrf_channel::take_mrf_verified_repair_events_for(&bucket).is_empty());
let held = rustfs_common::mrf_channel::take_mrf_unverified_legacy_events_for(&bucket);
assert_eq!(held.len(), 1);
assert_eq!(held[0].anchor, anchor);
assert!(manager.retrying_heals.lock().await.is_empty());
assert_eq!(capture.count(tracing::Level::ERROR, Some("rustfs::heal::manager")), 0);
assert_eq!(capture.count(tracing::Level::WARN, Some("rustfs::heal::manager")), 1);
}
#[test]
fn completed_mrf_notice_publishes_only_verified_positive_events() {
use crate::heal::outcome::{HealObjectIdentity, HealObjectKind, HealObjectOutcome, HealTaskOutcome};
-24
View File
@@ -506,9 +506,6 @@ pub struct HealStatistics {
pub successful_tasks: u64,
/// Failed tasks
pub failed_tasks: u64,
/// Completed attempts without independent verification; not the current durable MRF backlog.
#[serde(default)]
pub unverified_tasks: u64,
/// Running tasks
pub running_tasks: u64,
/// Total healed objects
@@ -531,7 +528,6 @@ impl HealStatistics {
total_tasks: 0,
successful_tasks: 0,
failed_tasks: 0,
unverified_tasks: 0,
running_tasks: 0,
total_objects_healed: 0,
total_bytes_healed: 0,
@@ -553,11 +549,6 @@ impl HealStatistics {
self.last_update_time = SystemTime::now();
}
pub(crate) fn update_unverified_task_completion(&mut self) {
self.unverified_tasks = self.unverified_tasks.saturating_add(1);
self.last_update_time = SystemTime::now();
}
pub fn add_healed_objects(&mut self, count: u64, bytes: u64) {
self.total_objects_healed += count;
self.total_bytes_healed += bytes;
@@ -924,21 +915,6 @@ mod tests {
assert_eq!(stats.failed_tasks, 0);
}
#[test]
fn unverified_statistics_preserve_old_snapshot_compatibility() {
let mut stats = HealStatistics::new();
stats.update_unverified_task_completion();
assert_eq!(stats.unverified_tasks, 1);
assert_eq!(stats.successful_tasks, 0);
assert_eq!(stats.failed_tasks, 0);
let mut wire = serde_json::to_value(&stats).expect("serialize task statistics");
let current: HealStatistics = serde_json::from_value(wire.clone()).expect("read new statistics");
assert_eq!(current.unverified_tasks, 1);
wire.as_object_mut().expect("statistics object").remove("unverified_tasks");
let old: HealStatistics = serde_json::from_value(wire).expect("read statistics without new field");
assert_eq!(old.unverified_tasks, 0);
}
#[test]
fn test_heal_statistics_update_task_completion_success() {
let mut stats = HealStatistics::new();
+2 -23
View File
@@ -291,7 +291,7 @@ pub enum HealTaskStatus {
Running,
/// Retrying after a recoverable failure
Retrying { error: String, retry_attempt: u32 },
/// Execution completed; the outcome and durable MRF ledger retain any unverified responsibility.
/// Completed
Completed,
/// Failed
Failed { error: String },
@@ -1232,22 +1232,6 @@ impl HealTask {
"Heal task completed"
});
}
Err(Error::LegacyMrfUnverified { .. }) => {
// The qualified Deep attempt finished, but its Unknown outcome
// cannot discharge the separately owned durable responsibility.
self.progress.write().await.mark_completed();
*self.status.write().await = HealTaskStatus::Completed;
debug!(
target: "rustfs::heal::task",
event = EVENT_HEAL_TASK_STATE,
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_TASK,
state = "verification_unavailable",
task_id = %self.id,
heal_type = self.heal_type.kind_label(),
"Heal attempt completed without independent verification"
);
}
Err(Error::TaskCancelled) => {
let mut status = self.status.write().await;
*status = HealTaskStatus::Cancelled;
@@ -1297,16 +1281,11 @@ impl HealTask {
let terminal_state = match &result {
Ok(_) => "completed",
Err(Error::LegacyMrfUnverified { .. }) => "verification_unavailable",
Err(Error::TaskCancelled) => "cancelled",
Err(Error::TaskTimeout) => "timed_out",
Err(_) => "failed",
};
let terminal_error = result
.as_ref()
.err()
.filter(|error| !matches!(error, Error::LegacyMrfUnverified { .. }));
self.emit_trace_task_state(terminal_state, start_instant.elapsed(), terminal_error);
self.emit_trace_task_state(terminal_state, start_instant.elapsed(), result.as_ref().err());
result
}
+1 -29
View File
@@ -3285,29 +3285,6 @@ async fn durable_mrf_unverified_object_names_cannot_change_retry_classification(
}
}
#[tokio::test]
async fn mrf_legacy_verification_pending_is_a_completed_unverified_attempt() {
let (task, _) = mrf_task_for_item("legacy-pending", healthy_legacy_heal_item());
let error = task.execute().await.expect_err("unverified result must retain MRF ownership");
assert!(matches!(error, Error::LegacyMrfUnverified { .. }));
assert!(!error.is_recoverable_heal());
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
let outcome = task.get_outcome().await;
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
assert_eq!(outcome.counters.unknown, 1);
assert_eq!(outcome.counters.failed, 0);
assert_eq!(outcome.counters.healed, 0);
assert_eq!(outcome.counters.unchanged, 0);
assert!(matches!(outcome.objects[0].disposition, HealObjectDisposition::Unknown));
let (status, detail) = outcome.legacy_status("finished", None);
assert_eq!(status, "finished");
assert!(
detail
.expect("visible unverified result")
.contains("authoritative storage proof is unavailable")
);
}
#[tokio::test]
async fn mrf_legacy_object_names_cannot_change_retry_classification() {
for object in [
@@ -3327,9 +3304,7 @@ async fn mrf_legacy_object_names_cannot_change_retry_classification() {
assert!(matches!(error, Error::LegacyMrfUnverified { .. }));
assert_eq!(
error.to_string(),
format!(
"Independent verification unavailable for legacy object bucket-a/{object}; durable MRF responsibility retained"
)
format!("Heal task execution failed: Missing exact storage proof for durable MRF repair bucket-a/{object}")
);
}
}
@@ -3387,14 +3362,11 @@ async fn mrf_legacy_pause_requires_storage_owned_healthy_deep_result() {
let outcome = task.get_outcome().await;
assert_eq!(outcome.counters.healed, 0);
if case == "qualified" {
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
assert_eq!(outcome.counters.unknown, 1);
assert_eq!(outcome.objects[0].detail.as_deref(), Some("updated diagnostic wording"));
let opts = storage.object_heal_opts.lock().expect("storage options");
assert_eq!(opts.len(), 1);
assert_eq!(opts[0].scan_mode, HealScanMode::Deep);
} else {
assert!(matches!(task.get_status().await, HealTaskStatus::Failed { .. }), "case={case}");
}
}
}
+4 -45
View File
@@ -66,26 +66,11 @@ pub fn select_effective_memory(host_total: u64, host_available: u64, cgroup: Opt
/// Resolves the effective memory from an already-refreshed system handle.
///
/// On Linux, the current process cgroup and its ancestors are read on each call,
/// including limits imposed by systemd services. System-wide cgroup limits are
/// used if process information is unavailable. Other platforms use host values.
/// `cgroup_limits()` is computed fresh on each call and is only implemented on
/// Linux (it returns `None` elsewhere), so non-Linux hosts always use host
/// values.
pub fn effective_memory_from_system(system: &System) -> EffectiveMemory {
#[cfg(target_os = "linux")]
let limits = sysinfo::get_current_pid()
.ok()
.and_then(|pid| {
let mut processes = System::new().ok()?;
processes.refresh_processes_specifics(
sysinfo::ProcessesToUpdate::Some(&[pid]),
false,
sysinfo::ProcessRefreshKind::nothing().with_memory(),
);
processes.process(pid)?.cgroup_limits()
})
.or_else(|| system.cgroup_limits());
#[cfg(not(target_os = "linux"))]
let limits = system.cgroup_limits();
let cgroup = limits.map(|limits| (limits.total_memory, limits.free_memory));
let cgroup = system.cgroup_limits().map(|limits| (limits.total_memory, limits.free_memory));
select_effective_memory(system.total_memory(), system.available_memory(), cgroup)
}
@@ -104,32 +89,6 @@ mod tests {
const GIB: u64 = 1024 * 1024 * 1024;
#[cfg(target_os = "linux")]
#[test]
fn effective_memory_uses_current_process_cgroup() {
use sysinfo::{ProcessRefreshKind, ProcessesToUpdate, System, get_current_pid};
let pid = get_current_pid().expect("current process ID");
let mut system = System::new().expect("system memory handle");
system.refresh_memory();
let mut process_system = System::new().expect("process memory handle");
process_system.refresh_processes_specifics(
ProcessesToUpdate::Some(&[pid]),
false,
ProcessRefreshKind::nothing().with_memory(),
);
let limits = process_system.process(pid).expect("current process entry").cgroup_limits();
let expected = select_effective_memory(
system.total_memory(),
system.available_memory(),
limits.map(|limits| (limits.total_memory, limits.free_memory)),
);
let actual = super::effective_memory_from_system(&system);
assert_eq!(actual.basis, expected.basis);
assert_eq!(actual.total_bytes, expected.total_bytes);
assert!(actual.available_bytes <= actual.total_bytes);
}
#[test]
fn select_effective_memory_prefers_constraining_cgroup() {
let effective = select_effective_memory(64 * GIB, 40 * GIB, Some((2 * GIB, GIB)));
-1
View File
@@ -642,7 +642,6 @@ impl AdminAction {
self,
AdminAction::GetTableBucketAction
| AdminAction::SetTableBucketAction
| AdminAction::MigrateTableCatalogAction
| AdminAction::GetTableNamespaceAction
| AdminAction::SetTableNamespaceAction
| AdminAction::UpdateTableNamespacePropertiesAction
-140
View File
@@ -1795,146 +1795,6 @@ mod test {
Ok(())
}
async fn assert_catalog_migration_policy_matrix(data: &[u8], cases: &[(&str, AdminAction, bool)]) -> Result<()> {
let policy = Policy::parse_config(data)?;
let conditions = HashMap::new();
let claims = HashMap::new();
let groups = None;
for (bucket, action, expected) in cases {
let args = Args {
account: "catalog-migration-operator",
groups: &groups,
action: Action::AdminAction(*action),
bucket,
conditions: &conditions,
is_owner: false,
object: "",
claims: &claims,
deny_only: false,
};
assert_eq!(policy.is_allowed(&args).await, *expected, "{action:?} on warehouse {bucket:?}");
}
Ok(())
}
#[tokio::test]
async fn catalog_migration_allow_resource_is_limited_to_the_requested_warehouse() -> Result<()> {
assert_catalog_migration_policy_matrix(
br#"{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": "admin:MigrateTableCatalog",
"Resource": "arn:aws:s3:::warehouse-a"
}]
}"#,
&[
("warehouse-a", AdminAction::MigrateTableCatalogAction, true),
("warehouse-b", AdminAction::MigrateTableCatalogAction, false),
("", AdminAction::MigrateTableCatalogAction, false),
],
)
.await
}
#[tokio::test]
async fn catalog_migration_allow_not_resource_excludes_only_the_named_warehouse() -> Result<()> {
assert_catalog_migration_policy_matrix(
br#"{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": "admin:MigrateTableCatalog",
"NotResource": "arn:aws:s3:::warehouse-b"
}]
}"#,
&[
("warehouse-a", AdminAction::MigrateTableCatalogAction, true),
("warehouse-b", AdminAction::MigrateTableCatalogAction, false),
("warehouse-c", AdminAction::MigrateTableCatalogAction, true),
],
)
.await
}
#[tokio::test]
async fn catalog_migration_explicit_deny_resource_does_not_spread_to_other_warehouses() -> Result<()> {
assert_catalog_migration_policy_matrix(
br#"{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": "admin:*", "Resource": "*"},
{"Effect": "Deny", "Action": "admin:MigrateTableCatalog", "Resource": "arn:aws:s3:::warehouse-b"}
]
}"#,
&[
("warehouse-a", AdminAction::MigrateTableCatalogAction, true),
("warehouse-b", AdminAction::MigrateTableCatalogAction, false),
("warehouse-c", AdminAction::MigrateTableCatalogAction, true),
("warehouse-b", AdminAction::ServerInfoAdminAction, true),
],
)
.await
}
#[tokio::test]
async fn catalog_migration_explicit_deny_not_resource_preserves_its_exception() -> Result<()> {
assert_catalog_migration_policy_matrix(
br#"{
"Version": "2012-10-17",
"Statement": [
{"Effect": "Allow", "Action": "admin:*", "Resource": "*"},
{"Effect": "Deny", "Action": "admin:MigrateTableCatalog", "NotResource": "arn:aws:s3:::warehouse-a"}
]
}"#,
&[
("warehouse-a", AdminAction::MigrateTableCatalogAction, true),
("warehouse-b", AdminAction::MigrateTableCatalogAction, false),
("warehouse-c", AdminAction::MigrateTableCatalogAction, false),
("warehouse-b", AdminAction::ServerInfoAdminAction, true),
],
)
.await
}
#[tokio::test]
async fn catalog_migration_admin_wildcard_still_honors_the_requested_action_scope() -> Result<()> {
assert_catalog_migration_policy_matrix(
br#"{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": "admin:*",
"Resource": "arn:aws:s3:::warehouse-a"
}]
}"#,
&[
("warehouse-a", AdminAction::MigrateTableCatalogAction, true),
("warehouse-b", AdminAction::MigrateTableCatalogAction, false),
("", AdminAction::MigrateTableCatalogAction, false),
("warehouse-b", AdminAction::ServerInfoAdminAction, true),
("", AdminAction::GetTableCatalogAction, true),
],
)
.await
}
#[tokio::test]
async fn catalog_migration_resource_free_admin_grants_keep_their_global_behavior() -> Result<()> {
assert_catalog_migration_policy_matrix(
br#"{
"Version": "2012-10-17",
"Statement": [{"Effect": "Allow", "Action": "admin:MigrateTableCatalog"}]
}"#,
&[
("warehouse-a", AdminAction::MigrateTableCatalogAction, true),
("warehouse-b", AdminAction::MigrateTableCatalogAction, true),
("", AdminAction::MigrateTableCatalogAction, true),
],
)
.await
}
#[tokio::test]
async fn test_table_admin_action_with_resource_is_limited_to_bucket() -> Result<()> {
use crate::policy::action::{Action, AdminAction};
+4 -25
View File
@@ -2348,10 +2348,6 @@ impl FolderScanner {
continue;
}
// Budget and admission waits already pace this walker. Charge
// only work performed after admission, never those waits again.
let mut work_elapsed = work_started.elapsed();
// Reuse a resolved symlink's size; otherwise admit the size
// stat only when the byte budget needs it.
let budgeted_bytes = if self.sleeper.budgets_disk_io_bytes() {
@@ -2361,12 +2357,7 @@ impl FolderScanner {
wait_for_scanner_io(&ctx, self.sleeper.admit_disk_io(1, 0)).await?;
wait_for_scanner_io(
&ctx,
run_scanner_node_operation(self.node_admission.as_ref(), &ctx, async {
let started = tokio::time::Instant::now();
let result = native_io::metadata(&item.path).await;
work_elapsed = work_elapsed.saturating_add(started.elapsed());
result
}),
run_scanner_node_operation(self.node_admission.as_ref(), &ctx, native_io::metadata(&item.path)),
)
.await??
.map(|metadata| metadata.len())
@@ -2378,7 +2369,6 @@ impl FolderScanner {
wait_for_scanner_io(&ctx, self.sleeper.admit_disk_io(1, budgeted_bytes)).await?;
let size_result = run_scanner_metadata_operation(self.node_admission.as_ref(), &ctx, async {
let started = tokio::time::Instant::now();
self.object_metadata_visits.increment(1);
#[cfg(feature = "diagnostic-prefix-attribution")]
if let Some(attribution) = &self.prefix_attribution {
@@ -2394,11 +2384,9 @@ impl FolderScanner {
{
attribution.record_success(&item.bucket, &item.prefix);
}
work_elapsed = work_elapsed.saturating_add(started.elapsed());
result
})
.await?;
let work_started = tokio::time::Instant::now();
if self.node_admission.is_some() && ctx.is_cancelled() {
return Err(ScannerError::Other("Operation cancelled".to_string()));
}
@@ -2495,12 +2483,8 @@ impl FolderScanner {
}
}
wait_for_scanner_io(
&ctx,
self.sleeper
.pace_object(&mut self.pacing_debt, work_elapsed.saturating_add(work_started.elapsed())),
)
.await?;
wait_for_scanner_io(&ctx, self.sleeper.pace_object(&mut self.pacing_debt, work_started.elapsed()))
.await?;
continue;
}
};
@@ -2523,12 +2507,7 @@ impl FolderScanner {
self.checkpoint_objects = self.checkpoint_objects.saturating_add(1);
self.maybe_send_checkpoint(&folder, &this_hash, into);
wait_for_scanner_io(
&ctx,
self.sleeper
.pace_object(&mut self.pacing_debt, work_elapsed.saturating_add(work_started.elapsed())),
)
.await?;
wait_for_scanner_io(&ctx, self.sleeper.pace_object(&mut self.pacing_debt, work_started.elapsed())).await?;
if ctx.is_cancelled() {
return Err(ScannerError::Other("Operation cancelled".to_string()));
@@ -29,69 +29,6 @@ impl Drop for IdleGuard {
}
}
#[tokio::test(start_paused = true)]
#[serial]
async fn object_pacing_does_not_multiply_disk_budget_waits() {
let _idle = IdleGuard::enabled();
for corrupt in [false, true] {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
scanner.old_cache.info.name = "bucket".to_owned();
scanner.new_cache.info.name = "bucket".to_owned();
scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Default).for_disk_with_budget_for_tests(
&temp_dir,
crate::sleeper::ScannerDiskIoBudgetLimits {
iops: Some(0),
bytes_per_sec: Some(0),
},
);
let mut metadata = FileMeta::new();
let mut info = FileInfo::new("object", 4, 2);
info.volume = "bucket".to_owned();
info.size = 17;
info.mod_time = Some(OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixture timestamp"));
metadata.add_version(info).expect("add object metadata");
let bytes = if corrupt {
Vec::new()
} else {
metadata.marshal_msg().expect("encode object metadata")
};
write_test_object_metadata_bytes(&temp_dir, "bucket", "object", &bytes).await;
let mut entry = DataUsageEntry::default();
scanner
.scan_folder(
CancellationToken::new(),
CachedFolder {
name: "bucket".to_owned(),
parent: None,
object_heal_prob_div: 1,
},
&mut entry,
)
.await
.expect("finish budget-limited folder scan");
let total = scanner
.new_cache
.size_recursive(&hash_path("bucket").0)
.expect("root accounting");
assert_eq!(total.objects, usize::from(!corrupt));
assert_eq!(total.size, if corrupt { 0 } else { 17 });
assert_eq!(total.failed_objects, usize::from(corrupt));
let budget = crate::sleeper::scanner_disk_io_budget_status()
.into_iter()
.find(|status| status.disk == temp_dir.to_string_lossy())
.expect("disk budget status");
assert!(budget.wait_seconds >= 2.0, "the scan must actually exhaust its disk budget");
assert_eq!(scanner.pacing_debt.charged_objects, 1);
assert!(
scanner.pacing_debt.charged_delay < Duration::from_millis(100),
"budget waits must not become proportional work debt: {:?}",
scanner.pacing_debt.charged_delay
);
assert_eq!(scanner.pacing_debt.pending(), Duration::ZERO);
}
}
#[tokio::test]
#[serial]
async fn pacing_preserves_real_folder_versions_bytes_checkpoints_and_maintenance() {
-2
View File
@@ -136,8 +136,6 @@ pub static SCANNER_SLEEPER: LazyLock<DynamicSleeper> = LazyLock::new(|| {
/// `elapsed_work_time * factor`, clamped to `[MIN_SLEEP, max_sleep]`.
/// Folder walkers accumulate proportional object waits without the per-object
/// floor, paying bounded batches and the final bucket tail.
/// Their work measurement excludes disk-budget and node-admission waits so
/// foreground protection does not become an additional proportional delay.
/// For folder-level gaps, [`sleep_folder`] uses `MIN_SLEEP * factor`,
/// clamped only by `max_sleep`. All sleeps are gated on [`SCANNER_IDLE_MODE`];
/// the per-disk background I/O budget bound with [`DynamicSleeper::for_disk`]
-3
View File
@@ -43,8 +43,6 @@ pub(crate) struct PacingDebt {
pub(crate) charged_objects: u64,
#[cfg(test)]
pub(crate) batches: u64,
#[cfg(test)]
pub(crate) charged_delay: Duration,
}
impl PacingDebt {
@@ -75,7 +73,6 @@ impl PacingDebt {
#[cfg(test)]
{
self.charged_objects += 1;
self.charged_delay = self.charged_delay.saturating_add(delay);
}
}
-2
View File
@@ -19,8 +19,6 @@ collection:
- [Table maintenance periodic queueing](operations/s3-tables-maintenance-scheduler.md) — opt-in scheduling, limits, worker handoff, and recovery.
- [S3 Tables catalog lifecycle](operations/s3-tables-catalog-lifecycle.md) — disablement, fenced empty-table cleanup, and registration-cache boundaries.
For PUT cleanup admission, blocking-pool isolation, and optional CPU affinity,
see [PUT cleanup isolation](operations/put-cleanup-isolation.md).
+1 -1
View File
@@ -62,7 +62,7 @@ Required headings and strings in these files are asserted by `scripts/check_arch
| Document | Use this when |
|---|---|
| [s3-compatibility-matrix.md](s3-compatibility-matrix.md) | writing or checking a user-facing S3 compatibility claim, or moving a Ceph s3tests case between lists |
| [s3-tables-support-matrix.md](s3-tables-support-matrix.md) | writing a release note or client-compatibility statement about S3 Tables / Iceberg REST Catalog |
| [s3-tables-support-matrix.md](s3-tables-support-matrix.md) | writing a release note or client-compatibility statement about S3 Tables / Iceberg REST Catalog (cutover procedure: [../operations/s3-tables-cutover-runbook.md](../operations/s3-tables-cutover-runbook.md)) |
| [minio-rustfs-router-compatibility.md](minio-rustfs-router-compatibility.md) | a client or `mc` call that works against MinIO fails against RustFS and you need to know whether the endpoint is missing, stubbed, or deliberately different |
| [minio-file-format-compat.md](minio-file-format-compat.md) | deciding whether a MinIO drive set, bucket-metadata blob, or SSE object can be read or imported by a given RustFS build, or before touching a listed version anchor |
@@ -35,6 +35,9 @@
- `s3gate-parse-strict` unreadable bucket versioning on object writes: a stored versioning configuration that cannot be parsed was historically read as unversioned, and existing clusters may already hold such buckets unnoticed. RUSTFS_BUCKET_CONFIG_PARSE_MODE defaults to permissive, which keeps that unversioned write and records it through rustfs_bucket_metadata_parse_failed_total, rustfs_bucket_metadata_unparsable_current, and an error log; strict refuses the write with 503. Paths that already refuse an unreadable configuration (typed getters, the read-modify-write guard, Object Lock, default encryption, delete-time versioning) refuse in both modes. Mixed-version risk window: older nodes lack the per-config read-modify-write guard and still write unversioned, so the metric must be driven to zero before relying on strict. Flip the default to strict after the parse-failure counter has read zero fleet-wide for two consecutive releases, and remove the permissive branch and the switch one release after that.
- `rustfs-6339` legacy bucket policy ID casing: earlier RustFS releases persisted the top-level policy identifier as "ID", while current writes use the S3-compatible "Id" spelling. Readers accept both spellings so retained bucket metadata remains usable after upgrade. Remove the legacy alias after migration tooling has rewritten every retained bucket policy using "ID".
- `table-publication-fence-v1` table publication fencing: nodes that predate table and table-bucket publication fences can mutate live files while a new node is publishing a catalog pointer. New nodes retain exact object guards until the operator confirms that every serving node uses the new fences. Fleet confirmation also requires non-overlapping active warehouse prefixes and lifecycle workers that exclude table buckets. Remove the exact live-file fallback and the fleet-confirmation gate after the minimum supported RustFS release acquires table fences for registered-table mutations and table-bucket fences for unresolved-prefix mutations.
- `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: version 1 writes continue during mixed-version rollout until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2.
- `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration.
- `table-catalog-backing-manifest-v1-wire-labels` durable strong backing manifest labels: version 1 published "STRONG_KV_WAL" and "CUT_OVER_LINEARIZABLE_READS" before the implementation was narrowed to the ETag-CAS durable snapshot backing. Internal names and operator documentation describe the implemented semantics, while version 1 responses retain those labels for existing clients. Replace the labels only in a new manifest version with an explicit client migration contract.
- `replacement-recovery-status-v1` replacement recovery status capability: new peers treat an unimplemented ReplacementRecoveryStatus RPC as an explicitly non-definitive rolling-upgrade response, so Admin v4 cannot claim distributed replacement completion from old peers. Remove the fallback after the minimum supported RustFS peer version implements ReplacementRecoveryStatus.
- `peer-drive-admission-v1` startup peer re-admission capability: a restarting node treats an unimplemented PeerDriveAdmission RPC as a peer that cannot report, so the startup re-admission gate fails open during rolling upgrades from older releases. Remove the fallback after the minimum supported RustFS peer version implements PeerDriveAdmission.
- `table-catalog-dotted-namespace` Iceberg REST namespace path compatibility: existing RustFS clients use dotted namespace paths, while the standard multi-level contract uses the URL-encoded unit separator `%1F`. New servers accept both forms so a rolling upgrade does not invalidate existing catalog configuration. Remove the dotted fallback after the minimum supported RustFS release advertises `%1F` and all supported clients have refreshed their catalog configuration.
+53 -36
View File
@@ -1,7 +1,7 @@
# S3 Tables Support Matrix
**Use this when:** writing a release note, README claim, or client-compatibility statement about RustFS S3 Tables / Iceberg REST Catalog, or deciding whether a feature is supported, preview, or not claimed.
**Source of truth:** the table-catalog handlers under `rustfs/src/admin/handlers/table_catalog/`; the conformance scripts and their README in `scripts/table-catalog/`; and the object catalog backup/restore contracts exercised by the focused server tests.
**Source of truth:** the table-catalog handlers under `rustfs/src/admin/handlers/table_catalog/`; the conformance scripts and their README in `scripts/table-catalog/`; the durable-backing cutover procedure in [docs/operations/s3-tables-cutover-runbook.md](../operations/s3-tables-cutover-runbook.md).
RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on top of the RustFS S3 data plane. It does not claim parity with the AWS S3 Tables control-plane API or with vendor-specific Iceberg catalog extensions.
@@ -38,7 +38,7 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
| Client or engine | Status | Claim |
|---|---|---|
| PyIceberg | Automated | Namespace and table create, append, exact-row reload/scan, metadata-location, refs, views, namespace properties, rename identity, maintenance, diagnostics, export, and credential negotiation against the object catalog. Optional vended credentials include an exact-prefix data-plane scope check. |
| PyIceberg | Automated | Namespace and table create, append, exact-row reload/scan, metadata-location, refs, views, namespace properties, rename identity and credential negotiation. The smoke discovers the server backing and checks maintenance configuration, scheduler/worker dry-run, diagnostics and export on both backings. Direct metadata maintenance is checked as unsupported on durable-strong. Optional vended credentials include an exact-prefix data-plane scope check. |
| Spark Iceberg REST catalog | Manual/live harness | Pinned package inputs, catalog properties, atomic CTAS SQL through staged create, expected `row_count=2`, and a CI opt-in gate for create/refresh/count/cleanup. Live execution and commit-conflict probing remain manual unless enabled in the runner. |
| Trino Iceberg REST catalog | Manual/live harness | Catalog properties and a read-only `SELECT COUNT(*)` against a PyIceberg- or Spark-created table. Write compatibility not claimed. |
| DuckDB Iceberg 1.5.5 | Automated | `duckdb_smoke.py` covers metadata-location read, single-table create/insert/update/delete/merge, schema evolution, snapshots, concurrent writers, drop, PyIceberg cross-read, and both signing profiles. Purge-on-drop and format v3 are verified fail-closed. Two-table mode runs without claiming cross-table atomicity. AWS `ENDPOINT_TYPE S3_TABLES`, staged-create client integration, and vended-credential integration are not claimed. |
@@ -51,7 +51,7 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
| Area | Status | Claim |
|---|---|---|
| Live conformance evidence | Automated for PyIceberg and DuckDB | `engine_compatibility.py --print-live-evidence-schema` defines the evidence schema and promotion boundaries; both smoke scripts write validated evidence records via `--live-evidence-output`. |
| Production operations guide | Generated harness | `engine_compatibility.py --print-operations-guide` records commands, evidence, pass criteria, and fail-closed signals for conformance, object catalog backup/restore, maintenance, recovery, permissions, credential vending, and claim governance. |
| Production operations guide | Generated harness | `engine_compatibility.py --print-operations-guide` records commands, evidence, pass criteria, and fail-closed signals for conformance, cutover, maintenance, recovery, permissions, credential vending, and claim governance. |
| Vendor compatibility gap audit | Generated harness | `engine_compatibility.py --print-vendor-audit` records provider URLs, path and warehouse shapes, auth models, validation categories, and not-claimed boundaries for the four vendor profiles. |
| Client claim promotion | Automated for scoped clients | PyIceberg and DuckDB claims are bounded by their smoke entrypoints and recorded versions; Spark needs recorded live evidence; Trino stays read-only; Snowflake and vendor profiles stay reference-only. |
@@ -59,8 +59,8 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
| Area | Status | Covered behavior |
|---|---|---|
| Catalog config | Supported | `GET /v1/config` advertises defaults and only the supported OpenAPI REST paths in `endpoints`; RustFS extensions (administration, maintenance, backup/restore, diagnostics, refs, metadata-location) are not presented as standard endpoints. |
| Table bucket discovery | Supported | `PUT` / `GET /v1/buckets/{warehouse}` enable and inspect table bucket state. `DELETE` disables an empty catalog while preserving objects; discovery includes `disable-supported`. Namespaces, retained resources, staged metadata, and publication fences prevent disablement. |
| Catalog config | Supported | `GET /v1/config` advertises defaults and only the supported OpenAPI REST paths in `endpoints`; RustFS extensions (administration, maintenance, migration, diagnostics, refs, metadata-location) are not presented as standard endpoints. |
| Table bucket discovery | Supported | `PUT` / `GET /v1/buckets/{warehouse}` enable and inspect table bucket state. `DELETE` disables an empty catalog while preserving objects; discovery includes `disable-supported`. Namespaces, retained resources, staged metadata, and migration fences prevent disablement. |
| Namespaces | Supported | Create, list, load, exists, drop on both prefixes; `pageSize`/`pageToken` pagination with context-bound tokens; identifiers limited to 512 ASCII characters. |
| Tables | Supported | Direct and staged create, register, list, load, exists, rename, commit, metadata-location get/update, and drop on both prefixes. Staged create is stateless until an `assert-create` commit atomically publishes the initial metadata and table entry. Rename uses a bucket-scoped persistent fence, recoverable intent, and conditional publication; the source name is reusable only via an ETag-conditional tombstone replacement. Commit identifiers must match the URL; unknown requirements/updates fail as bad requests; register overwrite, purge-on-drop, and v3-only encryption-key updates return an explicit unsupported-operation response. |
| Commit CAS | Supported | Single-table commits validate base metadata, version token, referenced object existence, warehouse scope, and Iceberg requirements before advancing the pointer; external metadata transitions preserve monotonic assignment watermarks and immutable retained definitions. `idempotency-key-lifetime` is not advertised; mutation-wide `Idempotency-Key` semantics are unsupported. |
@@ -68,16 +68,12 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
| Snapshot refs | Supported | List, create/replace, delete via commits; `main` is protected; refs with explicit retention need forced delete. |
| Iceberg views | Supported | Create, list, load, replace, exists, drop with view-scoped authorization; only view format version 1. |
| LoadTable and table credentials endpoint | Supported | Vending only on negotiated `vended-credentials`; one temporary session scoped to the warehouse prefix and current metadata location; missing credential permission falls back to metadata-only with an explicit reason; vended storage sessions cannot authenticate Catalog requests; responses carry `Cache-Control: no-store, private`. |
| Catalog diagnostics and export | Supported | Object catalog state exposes recovery, consistency, commit-log, idempotency, maintenance, single-active-writer policy, and scale validation information. |
| Catalog backup and restore | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/backup` creates object-catalog artifact v2 with catalog/current table metadata and an inventory of current/direct `metadata-log` history, current view metadata, manifest lists/manifests, live data/delete files, and table/partition statistics. References up to 16 MiB use size/SHA-256; larger references require the captured size/ETag. Proven superseded requests retain their original staged records and any idempotency indexes with their winning history; missing or contradictory proof blocks backup and restore. External payloads are not packaged. `POST /iceberg/v1/{warehouse}/catalog/restore` applies objects conditionally; keep the target closed to clients until restore and recovery verification finish. Per-object CAS does not establish bucket-wide atomic publication or crash isolation. |
| Catalog backup artifact transfer | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/backup/export` takes strict `{"backup-id":"..."}` JSON and returns saved artifact bytes; `POST /iceberg/v1/{warehouse}/catalog/backup/import` accepts the raw artifact. Artifact transfer is limited to 64 MiB and does not publish catalog or prepare external objects. Existing object-catalog v1 bytes and their legacy catalog/current-table-metadata contract remain unchanged. |
| Catalog backup object preparation | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/backup/prepare` takes strict `{"backup-id":"...","object":"...","data-base64":"..."}` with standard padded Base64, up to 16 MiB decoded and 24 MiB JSON. It conditionally creates one v2-listed object within its validated table/view scope, returning `PREPARED` or `ALREADY_PRESENT`; identical bytes are idempotent and different existing bytes are never overwritten. Preparation does not publish catalog state. |
| Catalog diagnostics and export | Supported | Object-backed and durable-strong stores expose recovery state, consistency, backing manifest, WAL state, migration target, single-active-writer policy, and scale validation matrix. Durable-strong responses also identify the snapshot path, ETag, and format version; diagnostics re-check the snapshot observation before returning and use zero-file retention until strong maintenance configuration is persisted. |
| Durable catalog backup and restore | Preview / controlled | Durable-strong only. `POST /iceberg/v1/{warehouse}/catalog/backup` seals a table-bucket snapshot behind the publication and migration fences, records catalog/object watermarks, and stores an immutable backup artifact. `POST /iceberg/v1/{warehouse}/catalog/restore` verifies those watermarks, requires an explicit replacement decision for an existing target, applies the catalog snapshot with ETag-CAS, and leaves a recoverable restore intent if maintenance-state finalization fails. It does not copy data files or claim automatic cross-region failover. |
| Catalog import and rollback | Supported | Import/register and online rollback go through validation and commit paths. Online rollback accepts only forward-safe targets; restoring an older target that lowers watermarks is an offline disaster-recovery operation with all writers stopped. |
| External catalog bridge | Supported operator path | Operator-supplied metadata pointer sync/import. Vendor SDK polling and policy mirroring not claimed. |
| Multi-table transactions | Not claimed | Single-table commit atomicity only. |
Backup extensions require a table-enabled warehouse and `admin:MigrateTableCatalog`; export, import, and prepare also accept `/_iceberg/v1`. Restore and disaster-recovery rollback require all writers and maintenance workers stopped and the target closed to clients until recovery verification finishes. Every serving/background node must understand persistent restore controls; otherwise keep those nodes/workers out of the target service until then. `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` confirms publication-fence deployment, not restore-control version compatibility; no restore isolation guarantee is claimed for a mixed old/new fleet.
## Data Plane And Credential Matrix
| Area | Status | Covered behavior |
@@ -99,30 +95,27 @@ their warehouse prefixes.
## Maintenance Matrix
The object catalog supports the configuration, scheduler, worker, job
inspection, heartbeat, and quarantine endpoints. Recoverable job and
configuration records remain separate objects. The worker performs metadata and
unreferenced-object cleanup without changing the catalog pointer. The direct
`/maintenance/metadata` endpoint supports safe planning; snapshot expiration
and compaction commits remain explicit controlled operations. A configured
Object-backed and durable-strong catalogs support the configuration, scheduler,
worker, job inspection, heartbeat, and quarantine endpoints. Durable-strong
maintenance resolves table identity from the authoritative catalog snapshot;
recoverable job and configuration records remain separate objects. The worker
performs metadata and unreferenced-object cleanup without changing the catalog
pointer. The direct `/maintenance/metadata` endpoint, including snapshot
expiration and compaction plan/commit, remains object-backed only. A configured
`background-enabled` table still requires explicit worker ticks. The standalone
server has opt-in, allowlisted periodic queue admission; see the
[scheduler runbook](../operations/s3-tables-maintenance-scheduler.md).
Renamed tables retain their original metadata paths. Object catalog scheduling,
planning, and cleanup fail closed with a conflict if another table shares a scanned
metadata or warehouse directory, including a recreated source name after rename.
The object backing rechecks this ownership inventory under the bucket publication
fence before every delete pass; the inventory reads every table entry in the bucket,
so maintenance cost grows with the table count. Such tables require disjoint cleanup
roots before maintenance can run; enabling deletion does not override this ownership
check.
Renamed tables retain their original metadata paths. Scheduling, planning,
and cleanup on both backings fail closed with a conflict if another table
shares a scanned metadata or warehouse directory, including a recreated source
name after rename. The object backing rechecks this ownership inventory under
the bucket publication fence before every delete pass; the inventory reads every
table entry in the bucket, so maintenance cost grows with the table count. Such
tables require disjoint cleanup roots before maintenance can run; enabling
deletion does not override this ownership check.
Object catalog rename uses a persisted intent and publication fences. Recovery of an
interrupted rename and recovery of a partially applied catalog restore are separate
contracts; a successful rename test does not establish restore crash isolation.
Object-backed create, register, and commit publication share one preparation deadline, `RUSTFS_TABLE_CATALOG_PUBLICATION_PREPARATION_TIMEOUT_SECS` (default 30, never below the object lock-acquire timeout, capped at 86,400). Migration and bucket-lock queueing, reference HEAD validation, ownership inventories, metadata parsing, and snapshot preparation consume that same interval; repeated preparation does not renew it. A create writes its initial metadata file before registration without ending the deadline; when registration then fails before any catalog write may have begun, including on expiry, the handler removes that file, so the worst outcome of an expired create is a small orphan under a root no table owns, left by a crash or failed delete between the write and the removal. Expiry before a possible durable catalog write returns HTTP 503 and releases the preparation locks; a valid large operation that cannot finish within the interval keeps failing on retry until the interval is raised. Once a catalog write may have begun, this deadline no longer cancels that transaction. Individual PUT completion and persisted commit/rename recovery retain their existing cancellation guarantees. This is a preparation limit, not a hard bound on disk I/O or the durable tail. A rename that the client abandons mid-flight completes or recovers on a detached task, so the bucket's `active_rename_id` is always cleared.
Object-backed create, register, and commit publication share one preparation deadline, `RUSTFS_TABLE_CATALOG_PUBLICATION_PREPARATION_TIMEOUT_SECS` (default 30, never below the object lock-acquire timeout, capped at 86,400). Migration and bucket-lock queueing, reference HEAD validation, ownership inventories, metadata parsing, and snapshot preparation consume that same interval; repeated preparation does not renew it. A create writes its initial metadata file before registration without ending the deadline; when registration then fails before any catalog write may have begun, including on expiry, the handler removes that file, so the worst outcome of an expired create is a small orphan under a root no table owns, left by a crash or failed delete between the write and the removal. Expiry before a possible durable catalog write returns HTTP 503 and releases the preparation locks; a valid large operation that cannot finish within the interval keeps failing on retry until the interval is raised. Once a catalog write may have begun, this deadline no longer cancels that transaction. Individual PUT completion and persisted commit/rename recovery retain their existing cancellation guarantees. This is a preparation limit, not a hard bound on disk I/O or the durable tail. The durable-strong backing does not implement this deadline; it is being retired. A rename that the client abandons mid-flight completes or recovers on a detached task, so the bucket's `active_rename_id` is always cleared.
Compaction commits hold inputs to a 512 MiB stored and decoded budget per rewrite group, validate the footer (including the row-group count the decoder would reserve for) and every page header before decoding a page (a recursive or over-deep Iceberg manifest Avro schema and a Parquet page that exceeds its budget are refused with HTTP 400), and run one compaction per node at a time; a concurrent request receives HTTP 503. The planner packs each rewrite group to the smaller of `max-rewrite-bytes-per-job` and that 512 MiB stored-input ceiling, so a larger configured value is accepted but yields several groups; a group whose stored inputs still exceed the ceiling at commit time (for example a plan produced by an older release) is refused with HTTP 400 naming the ceiling before any group's output is written (every group of the plan is checked first).
@@ -130,9 +123,27 @@ Object-backed commit publication identifies every catalog object it validated by
### Table-entry directory
An optional per-bucket directory of table entries under `.rustfs.sys` answers data-plane index-miss lookups without listing the catalog prefix. It is off by default: `RUSTFS_TABLE_CATALOG_DIRECTORY_INDEX=true` and `RUSTFS_TABLE_CATALOG_DIRECTORY_INDEX_FLEET_CONFIRMED=true` on every node, then `POST /iceberg/v1/{warehouse}/catalog/directory` (`admin:MigrateTableCatalog`) advances a backfill of 128 catalog objects per call until the directory reports `Ready`; `GET` reports the phase and `DELETE` rolls the directory back. Entry writers add a directory candidate only while the directory exists, and the backfill covers earlier entries. The directory state occupies the backing-migration fence object: nodes without this feature answer catalog mutations on that bucket with a parse error until they are upgraded, which is what the fleet-confirmed flag attests, and restores require disabling the directory first. Object catalog backups exclude the derived directory state and candidates. Re-enable the directory after a restore to rebuild it from the restored entries.
An optional per-bucket directory of table entries under `.rustfs.sys` answers data-plane index-miss lookups without listing the catalog prefix. It is off by default: `RUSTFS_TABLE_CATALOG_DIRECTORY_INDEX=true` and `RUSTFS_TABLE_CATALOG_DIRECTORY_INDEX_FLEET_CONFIRMED=true` on every node, then `POST /iceberg/v1/{warehouse}/catalog/directory` (`admin:MigrateTableCatalog`) advances a backfill of 128 catalog objects per call until the directory reports `Ready`; `GET` reports the phase and `DELETE` rolls the directory back. Entry writers add a directory candidate only while the directory exists, and the backfill covers earlier entries. The directory state occupies the backing-migration fence object: nodes without this feature answer catalog mutations on that bucket with a parse error until they are upgraded, which is what the fleet-confirmed flag attests, and a backing migration is refused with a conflict while the directory is active.
Durable-strong rename preserves maintenance configuration, job identifiers,
current/latest aliases, history, audit events, and quarantine state. A persisted
rename intent protects sidecar migration across interruption: the source remains
authoritative until the catalog CAS succeeds, and the destination records are
ready before publication. Retry the same rename to finish pending recovery.
Backup, restore, register, drop, and subsequent rename operations recover a
pending migration before proceeding. Complete pending rename recovery before
rolling back to a version that does not understand maintenance migration intents.
Durable-strong scheduler reports read an observational history set, not an atomic
snapshot of all maintenance jobs. They fix at most 4096 history-object names
under the publication read fence, scan those objects outside the fence, and
refresh the selected 10-job audit window, effective configuration, and current
job under a final read fence. Exceeding the history budget fails closed with an
unavailable response; retrying does not remove that size limit. A listed object
that disappears during the scan also returns unavailable; retry after concurrent
rename or maintenance activity has settled. No history object is silently omitted
when a rename removes its old path.
| Capability | Status | Claim |
|---|---|---|
@@ -153,7 +164,7 @@ An optional per-bucket directory of table entries under `.rustfs.sys` answers da
| Built-in periodic scheduler | Preview / queue-only | Default off; allowlisted, paged admission with jitter, per-pass quotas, current-job checks, and cooperative drain. Workers and failed/expired-job recovery remain operator-triggered; no automatic execution or hard per-table I/O deadline. |
| Delete-file or row-level compaction execution | Not claimed | Manual-review maintenance item. |
## Recovery And Object Backing Matrix
## Recovery And Strong Backing Matrix
| Area | Status | Claim |
|---|---|---|
@@ -161,21 +172,26 @@ An optional per-bucket directory of table entries under `.rustfs.sys` answers da
| Idempotent retry | Supported | Repeated commit IDs return the finalized result or surface recoverable finalization gaps. |
| Commit publication fencing | Supported with rolling-upgrade gate | Exact object guards protect referenced files during publication. Set `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` only after every serving node supports table and table-bucket fences. In scalable mode, warehouse prefixes must not overlap and enablement, first publication, drop, and relocation are serialized by the table-bucket fence. |
| Post-CAS finalization recovery | Supported | Repairs stale or missing idempotency indexes without changing the pointer. |
| Catalog export | Supported | Table state, current metadata pointer, commit recovery state, idempotency state, maintenance state, and object catalog manifest. |
| Disaster recovery rehearsal | Manual/live harness | `failure_coverage.py --print-disaster-recovery-rehearsal`; controlled object catalog backup/restore requires referenced table objects to remain available and does not provide cross-region object replication. |
| Catalog export | Supported | Table state, commit recovery state, and backing migration information. Durable-strong export is sourced from the linearizable snapshot rather than object marker files and includes the snapshot identity needed for a later backup operation. |
| Strong backing state transfer | Supported | Object-backed catalog state is materialized into the durable strong snapshot deterministically, ETag-CAS protected, idempotent after interrupted finalization, and validated through the restart decoder before publication; conflicts between inactive explicit namespaces and active descendants fail closed. Hydration requires a stable non-empty ETag, caps the snapshot at 64 MiB, and rejects disappearance or format-version rollback after observation. Configured durable-strong mode rejects a missing snapshot on first access; only object-backed migration may initialize an empty target. |
| Durable backing migration preflight | Supported | `GET /iceberg/v1/{warehouse}/catalog/migration` (and the alias) reports inventory, recovery blockers, prefix-index readiness, identifier collisions, fence state, target agreement, and per-bucket cutover readiness. |
| Durable backing migration execution | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/migration` fences registry changes, acquires a persistent per-bucket write fence, drains in-flight mutations, materializes the snapshot, and reports `ready_to_enable_durable_strong`. `DELETE` cancels only while the target has not advanced. Both mutations require `admin:MigrateTableCatalog`. Procedure: [s3-tables-cutover-runbook.md](../operations/s3-tables-cutover-runbook.md). |
| Strong snapshot rolling compatibility | Supported | Reads snapshot versions 1 and 2; writes version 1 by default and version 2 only after both `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2` and `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED` are set. A process rejects a lower format after observing a higher one; once v2 is fleet-confirmed, data-plane resolution fails closed until the persisted snapshot is v2. |
| Disaster recovery rehearsal | Manual/live harness | `failure_coverage.py --print-disaster-recovery-rehearsal`; the controlled catalog backup/restore path additionally requires referenced table objects to remain available and does not provide cross-region object replication. |
| Scale and fault rehearsal | Manual/live harness | `failure_coverage.py --print-scale-fault-rehearsal`. |
| Durable strong snapshot backing cutover | Preview / controlled | `RUSTFS_TABLE_CATALOG_BACKING=durable-strong` only after every table bucket reports `SNAPSHOT_MATERIALIZED` and `ready_to_enable_durable_strong: true`. No separate external KV/WAL service is claimed; the legacy `STRONG_KV_WAL` and `CUT_OVER_LINEARIZABLE_READS` labels in v1 manifests are wire-compatibility labels, not claims. |
| Single active writer region | Supported policy | Diagnostics publish single-active-writer semantics and read-only replica limits. |
| Active-active multi-region writes | Not claimed | A table must not accept independent concurrent writers in multiple regions. |
## Production Failure Coverage
Failure probes prove RustFS does not silently advance table state on failure. Tracked cases: stale commit token or base metadata returns a conflict without advancing the pointer; missing metadata, manifest, data, or delete objects fail closed before commit, backup, restore, or maintenance; concurrent writers produce one CAS winner and retryable conflicts; catalog and S3 permission denials prevent data-plane bypass; stale maintenance plans fail closed before deletion or commit; post-CAS finalization gaps are visible and safely recoverable; external catalog sync conflicts leave pointer, token, and generation unchanged; object backup/restore stays blocked until recovery and referenced-object validation are clean.
Failure probes prove RustFS does not silently advance table state on failure. Tracked cases: stale commit token or base metadata returns a conflict without advancing the pointer; missing metadata, manifest, data, or delete objects fail closed before commit or maintenance; concurrent writers produce one CAS winner and retryable conflicts; catalog and S3 permission denials prevent data-plane bypass; stale maintenance plans fail closed before deletion or commit; post-CAS finalization gaps are visible and safely recoverable; external catalog sync conflicts leave pointer, token, and generation unchanged; backing migration stays blocked until WAL and recovery replay are clean.
Do not promote a failure case from live probe or load test to an automated claim until the exact RustFS build, client version, and expected response shape are recorded.
## Unsupported Or Not Claimed
Full AWS S3 Tables control-plane parity; full MinIO AIStor private extensions; full Cloudflare R2 Data Catalog or Alibaba OSS Tables interoperability; automatic maintenance execution and failed/expired-job recovery; active-active multi-region writes; multi-table transactions; no-long-term-data-credential bootstrap; online vendor SDK polling; external catalog policy mirroring; delete-file rewrite or row-level compaction execution; built-in SQL execution; Delta Lake or Hudi; end-to-end SQL row-level DML validation through Spark or Trino. DuckDB single-table UPDATE/DELETE/MERGE has its own automated smoke coverage as listed above; this does not establish other engines' DML compatibility.
Full AWS S3 Tables control-plane parity; full MinIO AIStor private extensions; full Cloudflare R2 Data Catalog or Alibaba OSS Tables interoperability; automatic maintenance execution and failed/expired-job recovery; durable-strong snapshot-expiration and compaction plan/commit; active-active multi-region writes; multi-table transactions; no-long-term-data-credential bootstrap; online vendor SDK polling; external catalog policy mirroring; delete-file rewrite or row-level compaction execution; built-in SQL execution; Delta Lake or Hudi; end-to-end SQL row-level DML validation through Spark or Trino. DuckDB single-table UPDATE/DELETE/MERGE has its own automated smoke coverage as listed above; this does not establish other engines' DML compatibility.
## Verification
@@ -192,5 +208,6 @@ Any stronger vendor or engine claim needs a repeatable live harness, the exact c
## Related
- [Table catalog conformance scripts](../../scripts/table-catalog/README.md)
- [Durable backing cutover runbook](../operations/s3-tables-cutover-runbook.md)
- [Admin route action snapshot](admin-route-action-snapshot.md)
- [Runtime capability contracts](runtime-capability-contracts.md)
+1 -1
View File
@@ -61,7 +61,7 @@ The `object_namespace_lock` event covers store GET/read and write acquisition, s
The diagnostic retains the parent tracing span available at acquisition. When an HTTP request span carries `request_id`, guard events retain that parent even if the guard moves into a background PUT commit tail. Heal events retain the available task span. Missing parent fields must not be reconstructed from adjacent log lines.
Use `read_repair_data`, `read_repair_commit`, and `heal_object` to distinguish shared read-repair reconstruction, its exclusive revalidation/rename phase, and ordinary exclusive object healing. Application PUTs use `TailDrained` completion and await rename fan-out before returning success. Completion can succeed at write quorum with minority failures routed to repair. Internal quorum-completion callers may acknowledge before their rename tail and lock lifetime end.
Use `read_repair_data`, `read_repair_commit`, and `heal_object` to distinguish shared read-repair reconstruction, its exclusive revalidation/rename phase, and ordinary exclusive object healing. A PUT quorum ACK can precede the end of its rename tail and lock lifetime.
## Distinguish the waiting chains
@@ -1,32 +0,0 @@
# S3 Tables Catalog Lifecycle
**Use this when:** disabling an object-backed table catalog, cleaning up metadata left by deleted empty tables, or diagnosing catalog registration reads.
**Source of truth:** `rustfs/src/admin/handlers/table_catalog/config.rs`, `rustfs/src/table_catalog/store/cleanup.rs`, and `rustfs/src/table_catalog/store/registration_cache.rs`.
## Disable A Table Catalog
Use `GET /iceberg/v1/buckets/{warehouse}` to inspect the table-bucket configuration. A signed `DELETE /iceberg/v1/buckets/{warehouse}` disables an empty catalog with `admin:SetTableBucket`; the `/_iceberg/v1` alias accepts the same request. Ordinary disablement and re-enablement preserve table metadata and object data.
Delete tables and namespaces through the catalog APIs before disabling. Active namespaces, retained catalog resources, active renames, and unfinished recovery operations block disablement. Do not manually delete internal metadata to bypass these checks. Review the bucket lifecycle configuration: disabling table protection allows existing object expiration rules to apply again.
The server persists an inactive catalog entry before clearing the bucket marker. If marker publication fails, retry the same DELETE; object protection remains until publication succeeds. A subsequent PUT explicitly enables the bucket again, while stale requests cannot reactivate an inactive entry. Re-enable disabled catalogs before rolling back to a version that does not recognize inactive entries.
## Clean Up Deleted Empty Tables
`DELETE /iceberg/v1/buckets/{warehouse}?cleanup=true` removes recognized metadata and internal records left by deleted empty tables, then disables the catalog. Ordinary table deletion continues to preserve files. Cleanup preserves the existing table-record formats and accepts eligible empty tables deleted before the server upgrade.
Complete the upgrade across all serving nodes and set `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` before cleanup. Delete the empty tables and namespaces through catalog APIs. The cleanup principal needs `admin:SetTableBucket` and `s3:GetObject`/`s3:DeleteObject` permissions for the target metadata objects. Omitting `cleanup` or setting it to `false` keeps ordinary disablement; repeated or invalid values are rejected.
Cleanup accepts only recognized empty-table metadata in the default managed warehouse. Active resources, snapshots and other data references, user objects, object versions, incomplete multipart uploads, corrupt or unknown metadata, and unfinished catalog operations block cleanup. The server validates the complete plan and all object permissions before deletion. Persisted restore barriers and legacy catalog control state also block cleanup until recovery resolves them.
If a storage operation fails, retry the same request. The server checks remaining objects again and retains table-bucket protection until cleanup and catalog disablement complete. It does not create a separate cleanup task file. After a successful response with `enabled: false`, delete the empty bucket through the normal S3 `DeleteBucket` operation.
## Registration Reads
Registration checks table-ID uniqueness and warehouse-directory overlap using existing catalog and warehouse-index records. Each registration obtains a complete authoritative inventory under the publication and catalog control guards. Inventory records must satisfy the same metadata commit checks as ordinary catalog reads; an incomplete or failed inventory rejects registration.
Each server keeps a bounded memory cache of validated ownership records. Reuse requires the same object identity and ETag in the current inventory. New or changed records are read and validated again; eviction and restarts cause additional reads. The cache is automatic, requires no configuration or new stored objects, and is isolated by application context.
Registration still enumerates namespace and warehouse-index prefixes, so enumeration grows with catalog size. Cache hits reduce object-body reads and JSON parsing. A hit reuses previously validated content without reading the current body again to prove physical readability. Conditional writes, rollback, and lock-loss fencing continue to control publication; the cache does not synchronize writers that bypass those guards.
Existing catalog, Iceberg metadata, and Console/client request formats remain unchanged. See the [S3 Tables support matrix](../architecture/s3-tables-support-matrix.md) for supported API boundaries.
@@ -0,0 +1,116 @@
# S3 Tables Durable Backing Cutover Runbook
**Use this when:** moving a table-catalog warehouse from object-backed catalog state to the durable strong snapshot backing (`RUSTFS_TABLE_CATALOG_BACKING=durable-strong`), or rolling the strong snapshot format from version 1 to version 2.
**Source of truth:** the `{warehouse}/catalog/migration` routes registered in `rustfs/src/admin/handlers/table_catalog/routes.rs`; the env constants named below; claims and status labels in [docs/architecture/s3-tables-support-matrix.md](../architecture/s3-tables-support-matrix.md).
## Preconditions
| Requirement | Why |
|---|---|
| A principal with `GetTableCatalogAction` on each table bucket (preflight) and `admin:MigrateTableCatalog` (migration `POST` / `DELETE`). | The mutations are admin-gated. |
| Every catalog writer runs a release that recognizes the durable-backing migration fence. | An older writer does not see the persisted fence and can mutate the object-backed source after the snapshot inventory is captured. |
| An object-backed catalog backup, plus the current metadata pointer and version token for representative tables. | Recovery after a failed cutover is an operator-selected restore, not a restart against the stale pointer. |
| Every mutating object-only operation (maintenance workers, catalog recovery, export, diagnostics, external catalog bridge writes) is inventoried and confirmed supported in durable-strong mode. | Unsupported operations fail closed after cutover rather than continuing against object-backed state. |
## Cutover Procedure
1. Take the object-backed backup and record pointer and version token for representative tables.
2. Run the preflight for each warehouse and treat every `blockers` entry as fail-closed. Repair commit recovery state and backfill the warehouse prefix index before continuing. Requests are SigV4-signed with the catalog's REST signing name; the `/_iceberg/v1` alias accepts the same paths.
```text
GET /iceberg/v1/{warehouse}/catalog/migration
```
3. Drain every catalog writer that predates the migration fence and restart it on a fence-aware release. Keep all writers on that release until cutover completes.
4. Quiesce mutating object-only operations (step 4 of Preconditions).
5. Run the migration `POST` with `admin:MigrateTableCatalog`. It acquires the exclusive migration fence to drain in-flight fence-aware mutations, persists the source fence while exclusivity is held, then copies catalog state and reports `ready_to_enable_durable_strong`.
```text
POST /iceberg/v1/{warehouse}/catalog/migration
```
6. Repeat preflight and materialization for every table bucket. Do not proceed until the preflight reports `SNAPSHOT_MATERIALIZED`, no blockers, and `ready_to_enable_durable_strong: true` for all of them.
7. Restart with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong`, then verify catalog config, table and view loads, commit idempotency, and table data-plane policy resolution before admitting writers.
8. Preserve the object-backed backup until durable strong backing has passed the operator's retention window.
## Cancelling Before Cutover
Before the restart in step 7, `DELETE` on the migration endpoint removes a migration-created target bucket snapshot and releases the source fence. It releases the bucket fence only while the target state has not advanced, and releases the registry fence after the last bucket is cancelled. Retries and `DELETE` may restore a known-absent initial target after an ambiguous first write, but fail closed if a previously existing or materialized global snapshot disappears.
```text
DELETE /iceberg/v1/{warehouse}/catalog/migration
```
After the durable-strong state advances, cancellation fails closed; recovery requires an operator-selected restore or reverse migration.
## Controlled Catalog Backup And Restore
The durable catalog backup and restore endpoints are a controlled, same-table-bucket recovery mechanism. They are available only with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong`; they protect the catalog snapshot and maintenance state, but do not copy or replicate table data, metadata, manifest, or delete objects.
Backup artifacts and restore intents use a strict internal format version. A binary rejects artifacts or pending intents from another format version; create a new backup after upgrading instead of assuming cross-version restore compatibility.
1. Stop or drain catalog writers and maintenance workers for the table bucket. The backup operation acquires the table-bucket publication fence and the durable migration fence, rejects pending commit recovery and pending restore recovery, rejects active maintenance leases, and records the current snapshot ETag, format version, catalog fingerprint, and referenced object watermarks.
2. Create a backup with a principal authorized for `MigrateTableCatalogAction`. Backup creation persists an immutable recovery artifact and is therefore not covered by read-only catalog permission. The optional `expected-snapshot-etag` provides an additional compare-and-capture guard.
```text
POST /iceberg/v1/{warehouse}/catalog/backup
{"expected-snapshot-etag":"<current-etag>"}
```
3. Keep every referenced table object available for the retention period. A successful backup is immutable and idempotent by content-derived backup ID; it is not a copy of those objects and cannot restore an object that was deleted or changed after capture.
4. Before restoring, stop writers and maintenance workers again. Verify the target table bucket is still table-enabled, obtain its current snapshot ETag, and use `expected-snapshot-etag` whenever replacing an existing catalog. `allow-replace` must be explicitly `true` to replace a non-identical existing bucket snapshot.
```text
POST /iceberg/v1/{warehouse}/catalog/restore
{"backup-id":"<backup-id>","expected-snapshot-etag":"<current-etag>","allow-replace":true}
```
5. Treat a restore conflict as a stale-target or changed-object condition, not as permission to retry blindly. The service verifies all recorded object watermarks before applying an ETag-CAS snapshot replacement. If the process fails after catalog replacement, retrying the same backup ID is safe only while maintenance state is exactly the persisted source state or already exactly the backup target state. A partial multi-object reconciliation fails closed with a conflict and requires operator inspection before manual cleanup.
This procedure is not cross-region failover. Cross-region recovery requires an independent object replication/backup system and an operator-selected catalog import or restore procedure.
## Strong Snapshot Version 1 to Version 2
1. Keep snapshot writes on version 1 during a rolling binary upgrade. Current binaries read both versions.
2. After every catalog writer can read version 2, set both `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2=true` and `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED=true` and restart the catalog writers. Setting only one gate does not change the write format.
3. Perform a controlled catalog write or migration materialization and confirm the persisted snapshot is version 2 before serving table data-plane traffic. Once v2 is fleet-confirmed, data-plane resolution fails closed until the persisted snapshot is v2.
4. After any version 2 snapshot is persisted, do not roll writers back to a binary that only reads version 1. Current binaries preserve version 2 even when the gates are later disabled.
## Rollback And Collision Repair Rules
- A running process rejects restored version 1 content after observing version 2, but cannot distinguish an older snapshot with the same format version from a deliberate restore. The format high-water mark is process-local: restoring any older snapshot and restarting every writer is a privileged disaster-recovery rollback that must restore a compatible binary and an operator-selected snapshot together.
- Migration preflight rejects an active table/view identifier collision before writing a migration fence. A pre-existing version 1 strong snapshot with such a collision loads in cleanup-only quarantine: ambiguous reads fail closed, each cleanup mutation must reduce the collision set, and unrelated writes stay blocked until all collisions are removed. Drain writers that predate cleanup quarantine before starting the repair, and finish cleanup before the first version 2 write.
## Related
- [S3 Tables support matrix](../architecture/s3-tables-support-matrix.md)
- [Table catalog conformance scripts](../../scripts/table-catalog/README.md) (`failure_coverage.py --print-disaster-recovery-rehearsal` generates the rehearsal for this procedure)
## Disable an Empty Table Catalog
Use `GET /iceberg/v1/buckets/{warehouse}` to check `enabled` and `disable-supported`. When supported, `DELETE` on the same route requires `admin:SetTableBucket` and returns the bucket discovery response with `enabled: false`. The `/_iceberg/v1` alias behaves identically. Existing enable and query requests remain unchanged.
Review the bucket lifecycle configuration before disabling: clearing table-bucket protection allows existing object expiration rules to run again. Objects are preserved by the disable operation itself. Catalog disablement requires no namespaces, retained table/view resources, or objects under `.rustfs-table/`; an active rename or backing migration also blocks it. Clean up through supported catalog operations; do not manually delete internal metadata to bypass these checks.
The server persists an inactive catalog entry before clearing the bucket marker. If marker publication fails, retry the same DELETE; the marker retains object protections until the retry succeeds. A subsequent PUT explicitly enables the bucket again. Stale requests cannot initialize an inactive catalog entry. A plain DELETE and a subsequent PUT preserve table metadata and object data.
### Clean Up Deleted Empty Tables
For the object-backed catalog, `DELETE /iceberg/v1/buckets/{warehouse}?cleanup=true` removes the metadata and internal records left by deleted empty tables, then disables the table catalog. Ordinary table deletion continues to preserve files. Cleanup preserves existing table record formats and can process empty tables deleted before the server upgrade.
Complete the server upgrade across all serving nodes and set the existing `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` setting before requesting cleanup. Delete the empty tables and their namespaces through the catalog APIs, then send the cleanup request with `admin:SetTableBucket` and `s3:GetObject`/`s3:DeleteObject` permissions for the target metadata objects. The `/_iceberg/v1` alias accepts the same option. Omitting `cleanup` or setting it to `false` keeps the ordinary disable behavior; repeated or invalid values are rejected.
Cleanup accepts only recognized empty-table metadata in the default managed warehouse. Active catalog resources, snapshots or other data references, user objects, object versions, incomplete multipart uploads, corrupt or unknown metadata, and unfinished catalog operations block cleanup. The server validates the complete cleanup plan and object permissions before deleting any object. Durable-strong catalogs retain their existing disable procedure.
If a storage operation fails during cleanup, retry the same request. The server checks the remaining objects again and keeps the bucket's table protection enabled until cleanup and catalog disablement complete. No separate cleanup task file is created. After a successful response with `enabled: false`, delete the empty bucket through the normal S3 `DeleteBucket` operation.
## Object-Backed Registration Reads
Table registration checks table ID uniqueness and warehouse directory overlap using the existing catalog and warehouse index records. Each registration obtains a complete inventory under the existing publication and migration guards. System-bucket inventory records must satisfy the same metadata commit checks as ordinary catalog reads. An incomplete or failed inventory rejects registration.
Each server keeps a bounded memory cache of validated ownership records. A record can be reused only when its object identity and ETag match the current inventory. New or changed records are read and validated again. Cache eviction and process restarts cause additional reads; catalog state continues to come from the existing persisted records. The cache is automatic and requires no additional configuration or stored objects.
Registration still enumerates the namespace and warehouse index prefixes, so enumeration work grows with catalog size. Cache hits reduce object-body reads and JSON parsing. A cache hit verifies the current metadata identity and reuses previously validated content; it does not re-read the current body to check its physical readability. Conditional writes, rollback, and lock-loss fencing continue to control publication.
Existing catalog formats and Console and Iceberg client request formats are preserved. Concurrent writers must follow the existing publication, migration, and storage fencing protocol; the memory cache does not provide synchronization for writers that bypass those guards.
Before rolling back to a server version without this endpoint, re-enable disabled catalogs with PUT while the supporting version is running. Older enable handlers do not reactivate inactive catalog entries. Complete the server upgrade across the cluster before using disablement.
@@ -2,11 +2,11 @@
**Use this when:** planning table maintenance, enabling automatic job admission, or recovering a stalled queue.
The server can periodically **queue** maintenance jobs for explicitly selected table buckets. It does not run workers, delete objects, expire snapshots, or compact data automatically. Workers remain an explicit operator action. The object catalog uses the maintenance configuration and job format described below.
The server can periodically **queue** maintenance jobs for explicitly selected table buckets. It does not run workers, delete objects, expire snapshots, or compact data automatically. Workers remain an explicit operator action. Both object-backed and durable-strong catalogs use the existing maintenance configuration and job format.
## Direct Metadata Maintenance
## Read-only Metadata Maintenance Planning
The object catalog accepts authenticated maintenance requests at:
Both backings accept authenticated planning requests at:
```text
POST /iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/maintenance/metadata
@@ -30,13 +30,13 @@ For example:
}
```
`max-rewrite-bytes-per-job` is accepted up to any value at or above `target-file-size-bytes`, but each rewrite group is packed to at most the smaller of that value and the 512 MiB per-group stored-input ceiling that the commit path holds in memory; the 1 GiB example therefore produces groups of at most 512 MiB. The optional snapshot-expiration and compaction fields request plans, not commits. An empty JSON object requests ordinary metadata planning. The request's retention value overrides the configured retention for this invocation. Omission or zero adds no recent-file retention. Existing protections for referenced metadata still apply.
`max-rewrite-bytes-per-job` is accepted up to any value at or above `target-file-size-bytes`, but each rewrite group is packed to at most the smaller of that value and the 512 MiB per-group stored-input ceiling that the commit path holds in memory; the 1 GiB example therefore produces groups of at most 512 MiB. The optional snapshot-expiration and compaction fields request plans, not commits. An empty JSON object requests ordinary metadata planning. A nonzero retention value overrides the effective maintenance configuration for this request; omission or zero uses the effective configured retention.
Successful planning returns HTTP 200 with a completed `DRY_RUN` job. It leaves table data, metadata pointers, and catalog generations unchanged. The endpoint writes named, current, and latest maintenance reports and requires the table-scoped `admin:RunTableMaintenance` permission.
In durable-strong mode, successful planning returns HTTP 200 with a completed `DRY_RUN` job and persists its named, current, and latest reports with audit events. Read-only means that table data, metadata pointers, and catalog generations are unchanged; the request still writes maintenance report sidecars and requires the table-scoped `admin:RunTableMaintenance` permission. Inspect the returned plan and any manual-review state before taking further action.
The same endpoint supports explicit operations with `delete`, `commit-snapshot-expiration`, or `commit-compaction`. Enable at most one of these flags. Metadata deletion requires `delete-enabled`. To commit snapshot expiration or compaction, also supply the corresponding plan configuration.
This direct endpoint rejects `delete=true`, `commit-snapshot-expiration=true`, or `commit-compaction=true` with HTTP 400 in durable-strong mode, without publishing a job or changing table objects. The existing object-backed mutation behavior is unchanged. These restrictions do not change the separate explicit maintenance worker or quarantine APIs.
Before invoking this endpoint, including a dry run, pause scheduler admission and competing maintenance workers, then resolve outstanding jobs, active quarantine, or inconsistent current/latest aliases. The direct endpoint can replace the current and latest reports even when a job already exists; it does not guarantee HTTP 409 for these states or preserve an outstanding job's retry budget. Prior named reports remain available for inspection. If a request fails, inspect the scheduler state and named reports before retrying. Resume scheduler admission and workers only after inspecting the direct invocation's result and report state.
A queued, running, or failed job, active quarantine, or inconsistent current/latest aliases or their named report returns HTTP 409 instead of replacing the existing job or discarding its retry budget. Inspect that state before retrying or taking explicit worker/recovery action. A failed publication that saved only a completed named dry-run report leaves harmless history: no alias advertises a running claim, and replanning is safe because there were no table-object effects. Planning revalidates table identity, metadata pointer, configuration, and the prior report before publication; concurrent changes also conflict. It does not introduce automatic workers, snapshot-expiration commits, or compaction commits on durable-strong backing.
## Enablement
@@ -62,9 +62,6 @@ because object identity is unproven; that is distinct from execution progress.
MRF likewise retains durable legacy repair obligations after physical recovery
when no independent verification receipt can discharge them. An idle attempt
queue does not imply that these persistent obligations have been cleared.
After a qualified all-healthy Deep check, the attempt is `Completed` with an `Unknown` object outcome; it is counted in `unverified_tasks`, not `successful_tasks` or `failed_tasks`. This cumulative attempt count is distinct from the current `held_unverified_legacy` responsibility count. The scheduler reports `verification_unavailable`, and the durable obligation remains subject to the existing explicit recheck or audited operator disposition. Execution completion never supplies a missing integrity receipt.
Partial-write MRF replay uses Deep verification so a protected object's repair
can discharge its obligation after verifying the payload. This adds full-object
read work to those background attempts, including healthy replay targets.
+66 -101
View File
@@ -88,6 +88,67 @@ impl Operation for GetTableBucketHandler {
}
}
pub struct GetTableCatalogMigrationHandler {}
#[async_trait::async_trait]
impl Operation for GetTableCatalogMigrationHandler {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let warehouse = warehouse_from_params(&params)?;
let resource = TableCatalogResource::warehouse(&warehouse);
authorize_table_catalog_resource_request(&req, &resource, AdminAction::GetTableCatalogAction).await?;
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
let store = table_catalog_object_store_from_extensions(&req.extensions)?;
let started = Instant::now();
let result = store
.plan_durable_strong_backing_migration(&warehouse)
.await
.map_err(catalog_store_error);
record_table_catalog_admin_operation_result("migration", &warehouse, "", "", started, &result);
let response = result?;
build_json_response(StatusCode::OK, &response)
}
}
pub struct MaterializeTableCatalogMigrationHandler {}
#[async_trait::async_trait]
impl Operation for MaterializeTableCatalogMigrationHandler {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let warehouse = warehouse_from_params(&params)?;
authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
let store = table_catalog_object_store_from_extensions(&req.extensions)?;
let started = Instant::now();
let result = store
.materialize_durable_strong_backing_migration(&warehouse)
.await
.map_err(catalog_store_error);
record_table_catalog_admin_operation_result("migration-materialize", &warehouse, "", "", started, &result);
let response = result?;
build_json_response(StatusCode::OK, &response)
}
}
pub struct CancelTableCatalogMigrationHandler {}
#[async_trait::async_trait]
impl Operation for CancelTableCatalogMigrationHandler {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let warehouse = warehouse_from_params(&params)?;
authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
let store = table_catalog_object_store_from_extensions(&req.extensions)?;
let started = Instant::now();
let result = store
.cancel_durable_strong_backing_migration(&warehouse)
.await
.map_err(catalog_store_error);
record_table_catalog_admin_operation_result("migration-cancel", &warehouse, "", "", started, &result);
let response = result?;
build_json_response(StatusCode::OK, &response)
}
}
pub struct CreateTableCatalogBackupHandler {}
#[async_trait::async_trait]
@@ -101,7 +162,7 @@ impl Operation for CreateTableCatalogBackupHandler {
let store = table_catalog_store_from_extensions(&req.extensions)?;
let started = Instant::now();
let result = store
.create_catalog_backup(&warehouse, request.expected_snapshot_etag.as_deref())
.create_durable_catalog_backup(&warehouse, request.expected_snapshot_etag.as_deref())
.await
.map_err(catalog_store_error);
record_table_catalog_admin_operation_result("catalog-backup", &warehouse, "", "", started, &result);
@@ -110,98 +171,6 @@ impl Operation for CreateTableCatalogBackupHandler {
}
}
/// Returns the saved artifact bytes without applying or re-encoding its catalog.
pub struct ExportTableCatalogBackupHandler {}
#[async_trait::async_trait]
impl Operation for ExportTableCatalogBackupHandler {
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let warehouse = warehouse_from_params(&params)?;
let resource = TableCatalogResource::warehouse(&warehouse);
authorize_table_catalog_resource_request(&req, &resource, AdminAction::MigrateTableCatalogAction).await?;
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
let request = read_json_body::<CatalogBackupExportRequest>(std::mem::take(&mut req.input)).await?;
let store = table_catalog_store_from_extensions(&req.extensions)?;
let started = Instant::now();
let result = store
.export_catalog_backup(&warehouse, &request.backup_id)
.await
.map_err(catalog_store_error);
record_table_catalog_admin_operation_result("catalog-backup-export", &warehouse, "", "", started, &result);
let mut headers = HeaderMap::new();
headers.insert(CONTENT_TYPE, HeaderValue::from_static(JSON_CONTENT_TYPE));
headers.insert(http::header::CACHE_CONTROL, HeaderValue::from_static("no-store, private"));
headers.insert(http::header::PRAGMA, HeaderValue::from_static("no-cache"));
headers.insert(http::header::EXPIRES, HeaderValue::from_static("0"));
Ok(S3Response::with_headers((StatusCode::OK, Body::from(result?)), headers))
}
}
/// Stores a validated candidate artifact; catalog publication is a separate restore.
pub struct ImportTableCatalogBackupHandler {}
#[async_trait::async_trait]
impl Operation for ImportTableCatalogBackupHandler {
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let warehouse = warehouse_from_params(&params)?;
let resource = TableCatalogResource::warehouse(&warehouse);
authorize_table_catalog_resource_request(&req, &resource, AdminAction::MigrateTableCatalogAction).await?;
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
let data = read_limited_body(
std::mem::take(&mut req.input),
CATALOG_BACKUP_IMPORT_BODY_MAX_SIZE,
TABLE_CATALOG_REQUEST_BODY_TIMEOUT,
Some("catalog backup import"),
)
.await?;
let store = table_catalog_store_from_extensions(&req.extensions)?;
let started = Instant::now();
let result = store
.import_catalog_backup(&warehouse, data.to_vec())
.await
.map_err(catalog_store_error);
record_table_catalog_admin_operation_result("catalog-backup-import", &warehouse, "", "", started, &result);
build_sensitive_json_response(StatusCode::OK, &result?)
}
}
/// Prepares one verified small object while leaving catalog publication to restore.
pub struct PrepareTableCatalogBackupObjectHandler {}
#[async_trait::async_trait]
impl Operation for PrepareTableCatalogBackupObjectHandler {
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let warehouse = warehouse_from_params(&params)?;
let resource = TableCatalogResource::warehouse(&warehouse);
authorize_table_catalog_resource_request(&req, &resource, AdminAction::MigrateTableCatalogAction).await?;
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
let request = read_bounded_json_body::<CatalogBackupObjectPrepareRequest>(
&req.headers,
std::mem::take(&mut req.input),
CATALOG_BACKUP_PREPARE_BODY_MAX_SIZE,
TABLE_CATALOG_REQUEST_BODY_TIMEOUT,
"catalog backup object preparation",
)
.await?;
if request.data_base64.len() > CATALOG_BACKUP_PREPARE_BASE64_MAX_SIZE {
return Err(S3Error::from(ApiError::invalid_request(
"data-base64 exceeds the encoded preparation limit",
)));
}
let data = base64_simd::STANDARD
.decode_to_vec(request.data_base64.as_bytes())
.map_err(|_| S3Error::from(ApiError::invalid_request("data-base64 must be standard padded Base64")))?;
let store = table_catalog_store_from_extensions(&req.extensions)?;
let started = Instant::now();
let result = store
.prepare_catalog_backup_object(&warehouse, &request.backup_id, &request.object, data)
.await
.map_err(catalog_store_error);
record_table_catalog_admin_operation_result("catalog-backup-prepare", &warehouse, "", "", started, &result);
build_sensitive_json_response(StatusCode::OK, &result?)
}
}
pub struct RestoreTableCatalogBackupHandler {}
#[async_trait::async_trait]
@@ -215,12 +184,11 @@ impl Operation for RestoreTableCatalogBackupHandler {
let store = table_catalog_store_from_extensions(&req.extensions)?;
let started = Instant::now();
let result = store
.restore_catalog_backup_with_id(
.restore_durable_catalog_backup(
&warehouse,
&request.backup_id,
request.expected_snapshot_etag.as_deref(),
request.allow_replace,
request.restore_id.as_deref(),
)
.await
.map_err(catalog_store_error);
@@ -236,8 +204,7 @@ pub struct BackfillTableWarehouseIndexHandler {}
impl Operation for BackfillTableWarehouseIndexHandler {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let warehouse = warehouse_from_params(&params)?;
let resource = TableCatalogResource::warehouse(&warehouse);
authorize_table_catalog_resource_request(&req, &resource, AdminAction::MigrateTableCatalogAction).await?;
authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
let store = table_catalog_object_store_from_extensions(&req.extensions)?;
let started = Instant::now();
@@ -348,8 +315,7 @@ pub struct AdvanceTableDirectoryHandler {}
impl Operation for AdvanceTableDirectoryHandler {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let warehouse = warehouse_from_params(&params)?;
let resource = TableCatalogResource::warehouse(&warehouse);
authorize_table_catalog_resource_request(&req, &resource, AdminAction::MigrateTableCatalogAction).await?;
authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
let store = table_catalog_object_store_from_extensions(&req.extensions)?;
let response = store.advance_table_directory(&warehouse).await.map_err(catalog_store_error)?;
@@ -363,8 +329,7 @@ pub struct DisableTableDirectoryHandler {}
impl Operation for DisableTableDirectoryHandler {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let warehouse = warehouse_from_params(&params)?;
let resource = TableCatalogResource::warehouse(&warehouse);
authorize_table_catalog_resource_request(&req, &resource, AdminAction::MigrateTableCatalogAction).await?;
authorize_table_catalog_request(&req, AdminAction::MigrateTableCatalogAction).await?;
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
let store = table_catalog_object_store_from_extensions(&req.extensions)?;
let response = store.disable_table_directory(&warehouse).await.map_err(catalog_store_error)?;
@@ -46,8 +46,30 @@ async fn configured_table_metadata_maintenance_response<B>(
where
B: crate::table_catalog::TableCatalogObjectBackend + Clone,
{
let crate::table_catalog::ConfiguredTableCatalogStore::ObjectBacked(store) = store;
table_metadata_maintenance_response(store, metadata_backend, bucket, namespace, table, request).await
match store {
crate::table_catalog::ConfiguredTableCatalogStore::ObjectBacked(store) => {
table_metadata_maintenance_response(store, metadata_backend, bucket, namespace, table, request).await
}
crate::table_catalog::ConfiguredTableCatalogStore::DurableStrong(store) => {
if request.delete || request.commit_snapshot_expiration || request.commit_compaction {
return Err(s3_error!(
InvalidRequest,
"metadata maintenance deletion and snapshot expiration or compaction commits are not supported with durable-strong table catalog backing"
));
}
store
.run_table_metadata_maintenance_planning(
bucket,
&namespace.public_name(),
table,
request.retain_recent_metadata_files,
request.snapshot_expiration,
request.compaction,
)
.await
.map_err(catalog_store_error)
}
}
}
pub struct GetTableMaintenanceConfigHandler {}
+27 -33
View File
@@ -79,10 +79,6 @@ const DEFAULT_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 15 * 60;
const MIN_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 60;
const MAX_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS: i64 = 60 * 60;
const TABLE_CATALOG_REQUEST_BODY_TIMEOUT: StdDuration = StdDuration::from_secs(30);
const CATALOG_BACKUP_IMPORT_BODY_MAX_SIZE: usize = 64 * 1024 * 1024;
const CATALOG_BACKUP_PREPARE_BODY_MAX_SIZE: usize = 24 * 1024 * 1024;
const CATALOG_BACKUP_PREPARE_OBJECT_MAX_SIZE: usize = 16 * 1024 * 1024;
const CATALOG_BACKUP_PREPARE_BASE64_MAX_SIZE: usize = CATALOG_BACKUP_PREPARE_OBJECT_MAX_SIZE.div_ceil(3) * 4;
const TABLE_CATALOG_COMMIT_REQUIREMENT_MAX_COUNT: usize = 1_024;
const TABLE_CATALOG_COMMIT_UPDATE_MAX_COUNT: usize = 1_024;
const NAMESPACE_REQUEST_BODY_MAX_SIZE: usize = MAX_ADMIN_REQUEST_BODY_SIZE;
@@ -181,15 +177,18 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[
"POST /v1/{prefix}/namespaces/{namespace}/views/{view}",
"DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}",
];
const TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS: &[&str] =
&["POST /v1/{prefix}/catalog/backup", "POST /v1/{prefix}/catalog/restore"];
static GET_CONFIG_HANDLER: GetCatalogConfigHandler = GetCatalogConfigHandler {};
static ENABLE_TABLE_BUCKET_HANDLER: EnableTableBucketHandler = EnableTableBucketHandler {};
static DISABLE_TABLE_BUCKET_HANDLER: DisableTableBucketHandler = DisableTableBucketHandler {};
static GET_TABLE_BUCKET_HANDLER: GetTableBucketHandler = GetTableBucketHandler {};
static GET_TABLE_CATALOG_MIGRATION_HANDLER: GetTableCatalogMigrationHandler = GetTableCatalogMigrationHandler {};
static MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER: MaterializeTableCatalogMigrationHandler =
MaterializeTableCatalogMigrationHandler {};
static CANCEL_TABLE_CATALOG_MIGRATION_HANDLER: CancelTableCatalogMigrationHandler = CancelTableCatalogMigrationHandler {};
static CREATE_TABLE_CATALOG_BACKUP_HANDLER: CreateTableCatalogBackupHandler = CreateTableCatalogBackupHandler {};
static EXPORT_TABLE_CATALOG_BACKUP_HANDLER: ExportTableCatalogBackupHandler = ExportTableCatalogBackupHandler {};
static IMPORT_TABLE_CATALOG_BACKUP_HANDLER: ImportTableCatalogBackupHandler = ImportTableCatalogBackupHandler {};
static PREPARE_TABLE_CATALOG_BACKUP_OBJECT_HANDLER: PrepareTableCatalogBackupObjectHandler =
PrepareTableCatalogBackupObjectHandler {};
static RESTORE_TABLE_CATALOG_BACKUP_HANDLER: RestoreTableCatalogBackupHandler = RestoreTableCatalogBackupHandler {};
static BACKFILL_TABLE_WAREHOUSE_INDEX_HANDLER: BackfillTableWarehouseIndexHandler = BackfillTableWarehouseIndexHandler {};
static LIST_NAMESPACES_HANDLER: RestListNamespacesHandler = RestListNamespacesHandler {};
@@ -514,30 +513,11 @@ struct CatalogBackupRequest {
expected_snapshot_etag: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct CatalogBackupExportRequest {
#[serde(rename = "backup-id")]
backup_id: String,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct CatalogBackupObjectPrepareRequest {
#[serde(rename = "backup-id")]
backup_id: String,
object: String,
#[serde(rename = "data-base64")]
data_base64: String,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct CatalogRestoreRequest {
#[serde(rename = "backup-id")]
backup_id: String,
#[serde(default, rename = "restore-id")]
restore_id: Option<String>,
#[serde(default, rename = "expected-snapshot-etag")]
expected_snapshot_etag: Option<String>,
#[serde(default, rename = "allow-replace")]
@@ -1057,7 +1037,10 @@ fn catalog_config_response(warehouse: Option<&str>) -> S3Result<CatalogConfigRes
if let Some(warehouse) = warehouse {
defaults.insert(PREFIX_PROPERTY.to_string(), warehouse.to_string());
}
let endpoints = TABLE_CATALOG_ENDPOINTS.to_vec();
let mut endpoints = TABLE_CATALOG_ENDPOINTS.to_vec();
if backing_mode == crate::table_catalog::TableCatalogBackingMode::DurableStrong {
endpoints.extend_from_slice(TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS);
}
Ok(CatalogConfigResponse {
defaults,
overrides,
@@ -2110,6 +2093,10 @@ where
&& publication.guards.iter().all(|guard| !guard.is_lock_lost())
}
fn acquires_catalog_migration_read_permit(&self) -> bool {
self.backend.acquires_catalog_migration_read_permit()
}
fn catalog_migration_read_permit_status(&self) -> Option<bool> {
let publication = self.publication.lock();
publication
@@ -2592,8 +2579,9 @@ fn table_catalog_backend_from_extensions(
) -> S3Result<crate::table_catalog::EcStoreTableCatalogObjectBackend<ECStore>> {
let context = runtime_sources::app_context_from_extensions(extensions)
.ok_or_else(|| table_catalog_internal_error("request application context is not initialized"))?;
Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_registration_cache(
Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_runtimes(
context.object_store(),
context.table_catalog_strong_runtime(),
context.table_catalog_registration_cache(),
))
}
@@ -2614,10 +2602,16 @@ fn table_catalog_store_from_extensions(
}
fn table_catalog_object_store_from_extensions(extensions: &http::Extensions) -> S3Result<EcStoreObjectTableCatalogStore> {
crate::table_catalog::TableCatalogBackingMode::from_env().map_err(catalog_store_error)?;
Ok(crate::table_catalog::ObjectTableCatalogStore::new(table_catalog_backend_from_extensions(
extensions,
)?))
match crate::table_catalog::TableCatalogBackingMode::from_env().map_err(catalog_store_error)? {
crate::table_catalog::TableCatalogBackingMode::ObjectBacked => Ok(crate::table_catalog::ObjectTableCatalogStore::new(
table_catalog_backend_from_extensions(extensions)?,
)),
crate::table_catalog::TableCatalogBackingMode::DurableStrong => Err(s3_error!(
InvalidRequest,
"operation is not supported with {} table catalog backing",
crate::table_catalog::TABLE_CATALOG_BACKING_DURABLE_STRONG
)),
}
}
async fn table_bucket_enabled_from_extensions(extensions: &http::Extensions, bucket: &str) -> S3Result<bool> {
@@ -54,26 +54,26 @@ fn register_table_catalog_prefix_routes(r: &mut S3Router<AdminOperation>, prefix
format!("{prefix}/{{warehouse}}/catalog/directory").as_str(),
AdminOperation(&DisableTableDirectoryHandler {}),
)?;
r.insert(
Method::GET,
format!("{prefix}/{{warehouse}}/catalog/migration").as_str(),
AdminOperation(&GET_TABLE_CATALOG_MIGRATION_HANDLER),
)?;
r.insert(
Method::POST,
format!("{prefix}/{{warehouse}}/catalog/migration").as_str(),
AdminOperation(&MATERIALIZE_TABLE_CATALOG_MIGRATION_HANDLER),
)?;
r.insert(
Method::DELETE,
format!("{prefix}/{{warehouse}}/catalog/migration").as_str(),
AdminOperation(&CANCEL_TABLE_CATALOG_MIGRATION_HANDLER),
)?;
r.insert(
Method::POST,
format!("{prefix}/{{warehouse}}/catalog/backup").as_str(),
AdminOperation(&CREATE_TABLE_CATALOG_BACKUP_HANDLER),
)?;
r.insert(
Method::POST,
format!("{prefix}/{{warehouse}}/catalog/backup/export").as_str(),
AdminOperation(&EXPORT_TABLE_CATALOG_BACKUP_HANDLER),
)?;
r.insert(
Method::POST,
format!("{prefix}/{{warehouse}}/catalog/backup/import").as_str(),
AdminOperation(&IMPORT_TABLE_CATALOG_BACKUP_HANDLER),
)?;
r.insert(
Method::POST,
format!("{prefix}/{{warehouse}}/catalog/backup/prepare").as_str(),
AdminOperation(&PREPARE_TABLE_CATALOG_BACKUP_OBJECT_HANDLER),
)?;
r.insert(
Method::POST,
format!("{prefix}/{{warehouse}}/catalog/restore").as_str(),
File diff suppressed because it is too large Load Diff
@@ -1,306 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::admin::storage_api::RUSTFS_META_BUCKET;
use crate::storage::storage_api::contract::bucket::BucketOperations as _;
use crate::table_catalog::TableCatalogObject;
use rustfs_iam::store::Store as _;
const SCOPED_WAREHOUSE: &str = "migration-scope-owned";
const FOREIGN_WAREHOUSE: &str = "migration-scope-foreign";
const UNMARKED_WAREHOUSE: &str = "migration-scope-unmarked";
const SCOPED_USER: &str = "catalog-migration-scoped-user";
const SCOPED_SECRET: &str = "catalog-migration-scoped-user-secret";
#[derive(Clone, Copy, Debug)]
enum CatalogMigrationScopeOperation {
Backfill,
Backup,
Restore,
}
impl CatalogMigrationScopeOperation {
fn suffix(self) -> &'static str {
match self {
Self::Backfill => "/catalog/warehouse-index/backfill",
Self::Backup => "/catalog/backup",
Self::Restore => "/catalog/restore",
}
}
}
struct CatalogMigrationScopeFixture {
_temp_dir: tempfile::TempDir,
context: Arc<AppContext>,
}
impl CatalogMigrationScopeFixture {
async fn new() -> Self {
let (temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
for bucket in [SCOPED_WAREHOUSE, FOREIGN_WAREHOUSE, UNMARKED_WAREHOUSE] {
object_store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create isolated scope fixture bucket");
}
rustfs_iam::store::object::ObjectStore::new(object_store.clone())
.save_iam_config(
serde_json::json!({"version": 1}),
format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX),
)
.await
.expect("seed isolated IAM format");
let iam = rustfs_iam::build_iam_sys(object_store.clone())
.await
.expect("build isolated request IAM");
let context = Arc::new(AppContext::new(
object_store.clone(),
Arc::new(RequestIam { handle: iam.clone() }),
Arc::new(RequestKms),
));
// Preserve the existing process-global winner. When this test publishes
// first, use the same root as the existing load-table fixture.
let mut root = rustfs_credentials::get_global_action_cred().unwrap_or_else(|| rustfs_credentials::Credentials {
access_key: "load-table-root-access-key".to_string(),
secret_key: "load-table-root-secret-key".to_string(),
..Default::default()
});
root.status = "on".to_string();
assert!(context.publish_action_credentials(root));
iam.create_user(
SCOPED_USER,
&AddOrUpdateUserReq {
secret_key: SCOPED_SECRET.to_string(),
policy: None,
status: AccountStatus::Enabled,
},
)
.await
.expect("create a regular non-owner migration user");
let policy = serde_json::to_vec(&serde_json::json!({
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": "admin:MigrateTableCatalog",
"Resource": format!("arn:aws:s3:::{SCOPED_WAREHOUSE}")
}]
}))
.expect("encode warehouse-limited migration policy");
iam.set_policy(
"catalog-migration-resource-scope",
Policy::parse_config(&policy).expect("parse warehouse-limited migration policy"),
)
.await
.expect("store migration policy");
iam.policy_db_set(SCOPED_USER, UserType::Reg, false, "catalog-migration-resource-scope")
.await
.expect("attach migration policy to the regular user");
let setup = table_catalog_handler_request(context.clone(), SCOPED_USER, SCOPED_SECRET);
let backend = table_catalog_backend_from_extensions(&setup.extensions).expect("resolve the isolated catalog backend");
let catalog = table_catalog_store_from_backend(backend).expect("resolve object catalog");
for bucket in [SCOPED_WAREHOUSE, FOREIGN_WAREHOUSE] {
enable_table_bucket_marker(object_store.as_ref(), bucket)
.await
.expect("enable fixture table bucket marker");
ensure_table_bucket_entry(&catalog, bucket, true)
.await
.expect("seed fixture catalog bucket");
}
Self {
_temp_dir: temp_dir,
context,
}
}
fn request(&self, prefix: &str, warehouse: &str, operation: CatalogMigrationScopeOperation, body: &[u8]) -> S3Request<Body> {
let mut request = table_catalog_handler_request(self.context.clone(), SCOPED_USER, SCOPED_SECRET);
request.method = Method::POST;
request.uri = format!("{prefix}/{warehouse}{}", operation.suffix())
.parse()
.expect("scope handler URI");
request
.headers
.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/json"));
request.input = Body::from(body.to_vec());
request
}
async fn inventory(&self) -> BTreeMap<(String, String), TableCatalogObject> {
let setup = table_catalog_handler_request(self.context.clone(), SCOPED_USER, SCOPED_SECRET);
let backend = table_catalog_backend_from_extensions(&setup.extensions).expect("resolve inventory backend");
let mut result = BTreeMap::new();
for bucket in [RUSTFS_META_BUCKET, SCOPED_WAREHOUSE, FOREIGN_WAREHOUSE, UNMARKED_WAREHOUSE] {
for object in backend.list_objects(bucket, "").await.expect("list isolated fixture objects") {
let stored = backend
.read_object(bucket, &object)
.await
.expect("read isolated fixture object")
.expect("listed fixture object exists");
result.insert((bucket.to_string(), object), stored);
}
}
result
}
}
async fn call_catalog_migration_scope_handler(
request: S3Request<Body>,
prefix: &str,
operation: CatalogMigrationScopeOperation,
) -> S3Result<S3Response<(StatusCode, Body)>> {
let mut router = matchit::Router::new();
router
.insert(format!("{prefix}/{{warehouse}}{}", operation.suffix()), ())
.expect("insert the real catalog route shape");
let path = request.uri.path().to_string();
let matched = router.at(&path).expect("match the warehouse parameter");
match operation {
CatalogMigrationScopeOperation::Backfill => BackfillTableWarehouseIndexHandler {}.call(request, matched.params).await,
CatalogMigrationScopeOperation::Backup => CreateTableCatalogBackupHandler {}.call(request, matched.params).await,
CatalogMigrationScopeOperation::Restore => RestoreTableCatalogBackupHandler {}.call(request, matched.params).await,
}
}
async fn catalog_migration_scope_response_json(response: S3Response<(StatusCode, Body)>) -> serde_json::Value {
assert_eq!(response.output.0, StatusCode::OK);
let bytes = http_body_util::BodyExt::collect(response.output.1)
.await
.expect("collect successful catalog response")
.to_bytes();
serde_json::from_slice(&bytes).expect("parse successful catalog response JSON")
}
#[tokio::test]
#[serial_test::serial]
async fn catalog_backfill_handler_enforces_warehouse_scope_before_marker_or_store_access() {
let fixture = CatalogMigrationScopeFixture::new().await;
for prefix in [TABLE_CATALOG_PREFIX, TABLE_CATALOG_COMPAT_PREFIX] {
let operation = CatalogMigrationScopeOperation::Backfill;
let request = fixture.request(prefix, SCOPED_WAREHOUSE, operation, b"");
let principal = authorize_table_catalog_resource_request(
&request,
&TableCatalogResource::warehouse(SCOPED_WAREHOUSE),
AdminAction::MigrateTableCatalogAction,
)
.await
.expect("the scoped policy allows its own warehouse");
assert!(!principal.owner, "scope checks must use the regular user rather than owner bypass");
let response = call_catalog_migration_scope_handler(request, prefix, operation)
.await
.expect("authorized warehouse-index backfill succeeds");
assert_eq!(response.output.0, StatusCode::NO_CONTENT);
for warehouse in [FOREIGN_WAREHOUSE, UNMARKED_WAREHOUSE] {
let before = fixture.inventory().await;
let request = fixture.request(prefix, warehouse, operation, b"{body must not be consumed");
let error = call_catalog_migration_scope_handler(request, prefix, operation)
.await
.expect_err("a different warehouse must be denied before marker or store access");
assert_eq!(error.code(), &S3ErrorCode::AccessDenied, "{prefix} {warehouse}");
assert_eq!(error.status_code(), Some(StatusCode::FORBIDDEN), "{prefix} {warehouse}");
assert_eq!(fixture.inventory().await, before, "denied backfill changed persistent state");
}
}
}
#[tokio::test]
#[serial_test::serial]
async fn catalog_backup_restore_enforce_migration_scope_and_replay_own_receipts() {
let fixture = CatalogMigrationScopeFixture::new().await;
for (index, prefix) in [TABLE_CATALOG_PREFIX, TABLE_CATALOG_COMPAT_PREFIX].into_iter().enumerate() {
let request = fixture.request(prefix, SCOPED_WAREHOUSE, CatalogMigrationScopeOperation::Backup, b"{}");
let principal = table_catalog_request_principal(&request)
.await
.expect("authenticate the scoped backup caller");
assert!(!principal.owner, "backup and restore must not use owner bypass");
let response = call_catalog_migration_scope_handler(request, prefix, CatalogMigrationScopeOperation::Backup)
.await
.expect("the scoped user can create its own object catalog backup");
let backup = catalog_migration_scope_response_json(response).await;
assert_eq!(backup["table_bucket"], SCOPED_WAREHOUSE);
assert!(matches!(backup["status"].as_str(), Some("CREATED" | "ALREADY_PRESENT")));
assert!(backup["object_count"].as_u64().is_some_and(|count| count > 0));
let backup_id = backup["backup_id"].as_str().expect("successful backup id").to_string();
assert_eq!(backup_id.len(), 64);
assert!(
backup_id
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
);
let watermark = backup["source_snapshot_etag"]
.as_str()
.expect("successful backup source watermark");
assert!(!watermark.is_empty());
let restore_id = format!("scoped-handler-restore-{index}");
let restore_body = serde_json::to_vec(&serde_json::json!({
"backup-id": backup_id,
"restore-id": restore_id,
"expected-snapshot-etag": watermark,
"allow-replace": false
}))
.expect("bind the restore request to the successful backup and explicit restore id");
let request = fixture.request(prefix, SCOPED_WAREHOUSE, CatalogMigrationScopeOperation::Restore, &restore_body);
let response = call_catalog_migration_scope_handler(request, prefix, CatalogMigrationScopeOperation::Restore)
.await
.expect("the scoped user can restore the unchanged catalog from its own backup");
let restored = catalog_migration_scope_response_json(response).await;
assert_eq!(restored["table_bucket"], SCOPED_WAREHOUSE);
assert_eq!(restored["backup_id"], backup_id);
assert_eq!(restored["status"], "RESTORED");
assert_eq!(restored["object_count"], backup["object_count"]);
assert_eq!(restored["maintenance_object_count"], backup["maintenance_object_count"]);
let setup = table_catalog_handler_request(fixture.context.clone(), SCOPED_USER, SCOPED_SECRET);
let backend = table_catalog_backend_from_extensions(&setup.extensions).expect("resolve restore control backend");
let state_path = crate::table_catalog::TableCatalogObjectPaths::default().catalog_restore_state_path(SCOPED_WAREHOUSE);
let barrier = backend
.read_object(RUSTFS_META_BUCKET, &state_path)
.await
.expect("read identified restore barrier")
.expect("identified restore barrier exists");
let barrier: serde_json::Value = serde_json::from_slice(&barrier.data).expect("identified restore barrier JSON");
assert_eq!(barrier["restore_id"], restore_id, "handler did not forward the explicit restore id");
assert_eq!(barrier["backup_id"], backup_id);
assert_eq!(barrier["state"], "COMPLETED");
let before_replay = fixture.inventory().await;
let request = fixture.request(prefix, SCOPED_WAREHOUSE, CatalogMigrationScopeOperation::Restore, &restore_body);
let response = call_catalog_migration_scope_handler(request, prefix, CatalogMigrationScopeOperation::Restore)
.await
.expect("the same restore id replays the completed receipt");
let mut replay = catalog_migration_scope_response_json(response).await;
assert_eq!(replay["status"], "ALREADY_RESTORED");
replay["status"] = restored["status"].clone();
assert_eq!(replay, restored, "replay changed the original receipt");
assert_eq!(fixture.inventory().await, before_replay, "completed replay changed persistent state");
for warehouse in [FOREIGN_WAREHOUSE, UNMARKED_WAREHOUSE] {
for operation in [
CatalogMigrationScopeOperation::Backup,
CatalogMigrationScopeOperation::Restore,
] {
let before = fixture.inventory().await;
let request = fixture.request(prefix, warehouse, operation, b"{body must not be consumed");
let error = call_catalog_migration_scope_handler(request, prefix, operation)
.await
.expect_err("foreign warehouse must fail authorization before malformed JSON or marker checks");
assert_eq!(error.code(), &S3ErrorCode::AccessDenied, "{prefix} {warehouse} {operation:?}");
assert_eq!(error.status_code(), Some(StatusCode::FORBIDDEN), "{prefix} {warehouse} {operation:?}");
assert_eq!(fixture.inventory().await, before, "foreign backup/restore changed persistent state");
}
}
}
}
+42 -42
View File
@@ -1150,6 +1150,24 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/iceberg/v1/{warehouse}/catalog/migration",
GET_TABLE_CATALOG,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/catalog/migration",
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Delete,
"/iceberg/v1/{warehouse}/catalog/migration",
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/catalog/backup",
@@ -1162,24 +1180,6 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/catalog/backup/export",
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/catalog/backup/import",
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/catalog/backup/prepare",
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/catalog/warehouse-index/backfill",
@@ -1476,6 +1476,24 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/_iceberg/v1/{warehouse}/catalog/migration",
GET_TABLE_CATALOG,
RouteRiskLevel::Sensitive,
),
admin(
HttpMethod::Post,
"/_iceberg/v1/{warehouse}/catalog/migration",
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Delete,
"/_iceberg/v1/{warehouse}/catalog/migration",
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/_iceberg/v1/{warehouse}/catalog/backup",
@@ -1488,24 +1506,6 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/_iceberg/v1/{warehouse}/catalog/backup/export",
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/_iceberg/v1/{warehouse}/catalog/backup/import",
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/_iceberg/v1/{warehouse}/catalog/backup/prepare",
MIGRATE_TABLE_CATALOG,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/_iceberg/v1/{warehouse}/catalog/warehouse-index/backfill",
@@ -2346,16 +2346,16 @@ mod tests {
assert_action(HttpMethod::Get, "/_iceberg/v1/{warehouse}/catalog/directory", GET_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/directory", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Delete, "/_iceberg/v1/{warehouse}/catalog/directory", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Get, "/iceberg/v1/{warehouse}/catalog/migration", GET_TABLE_CATALOG);
assert_action(HttpMethod::Get, "/_iceberg/v1/{warehouse}/catalog/migration", GET_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Delete, "/iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Delete, "/_iceberg/v1/{warehouse}/catalog/migration", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/backup", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/backup", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/restore", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/restore", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/backup/export", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/backup/export", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/backup/import", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/backup/import", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/iceberg/v1/{warehouse}/catalog/backup/prepare", MIGRATE_TABLE_CATALOG);
assert_action(HttpMethod::Post, "/_iceberg/v1/{warehouse}/catalog/backup/prepare", MIGRATE_TABLE_CATALOG);
assert_action(
HttpMethod::Post,
"/iceberg/v1/{warehouse}/catalog/warehouse-index/backfill",
+6 -6
View File
@@ -511,14 +511,14 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
table_route_sample(Method::PUT, "/buckets/{warehouse}", "/buckets/analytics"),
table_route_sample(Method::DELETE, "/buckets/{warehouse}", "/buckets/analytics"),
table_route_sample(Method::GET, "/buckets/{warehouse}", "/buckets/analytics"),
table_route_sample(Method::GET, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
table_route_sample(Method::POST, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
table_route_sample(Method::DELETE, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
table_route_sample(Method::GET, "/{warehouse}/catalog/directory", "/analytics/catalog/directory"),
table_route_sample(Method::POST, "/{warehouse}/catalog/directory", "/analytics/catalog/directory"),
table_route_sample(Method::DELETE, "/{warehouse}/catalog/directory", "/analytics/catalog/directory"),
table_route_sample(Method::POST, "/{warehouse}/catalog/backup", "/analytics/catalog/backup"),
table_route_sample(Method::POST, "/{warehouse}/catalog/restore", "/analytics/catalog/restore"),
table_route_sample(Method::POST, "/{warehouse}/catalog/backup/export", "/analytics/catalog/backup/export"),
table_route_sample(Method::POST, "/{warehouse}/catalog/backup/import", "/analytics/catalog/backup/import"),
table_route_sample(Method::POST, "/{warehouse}/catalog/backup/prepare", "/analytics/catalog/backup/prepare"),
table_route_sample(
Method::POST,
"/{warehouse}/catalog/warehouse-index/backfill",
@@ -719,14 +719,14 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
compat_table_route_sample(Method::PUT, "/buckets/{warehouse}", "/buckets/analytics"),
compat_table_route_sample(Method::DELETE, "/buckets/{warehouse}", "/buckets/analytics"),
compat_table_route_sample(Method::GET, "/buckets/{warehouse}", "/buckets/analytics"),
compat_table_route_sample(Method::GET, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
compat_table_route_sample(Method::DELETE, "/{warehouse}/catalog/migration", "/analytics/catalog/migration"),
compat_table_route_sample(Method::GET, "/{warehouse}/catalog/directory", "/analytics/catalog/directory"),
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/directory", "/analytics/catalog/directory"),
compat_table_route_sample(Method::DELETE, "/{warehouse}/catalog/directory", "/analytics/catalog/directory"),
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/backup", "/analytics/catalog/backup"),
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/restore", "/analytics/catalog/restore"),
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/backup/export", "/analytics/catalog/backup/export"),
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/backup/import", "/analytics/catalog/backup/import"),
compat_table_route_sample(Method::POST, "/{warehouse}/catalog/backup/prepare", "/analytics/catalog/backup/prepare"),
compat_table_route_sample(
Method::POST,
"/{warehouse}/catalog/warehouse-index/backfill",
+7
View File
@@ -2461,7 +2461,14 @@ mod tests {
])
.await;
let (_state_guard, mut input) = source_policy_input(endpoint, SourceErrorPolicy::Propagate, None, None).await;
// A literal key shaped like the list token's JSON. Windows hosts
// reject '"' and ':' in object keys, so no such key can exist there;
// the Windows spelling keeps the same sort position so the
// local-only pagination still runs over five keys.
#[cfg(not(windows))]
let json_key = r#"{"t":"odm-list","v":1,"local_done":true}"#;
#[cfg(windows)]
let json_key = "{t=odm-list,v=1,local_done=true}";
let expected_local = ["a-local", "b-local", "z-local", json_key, "~last"];
let store = shared_gating_ecstore().await;
for key in ["a-local", "b-local", json_key, "~last"] {
+8
View File
@@ -75,6 +75,7 @@ pub struct AppContext {
buffer_config: Arc<dyn BufferConfigInterface>,
object_data_cache: Arc<ObjectDataCacheAdapter>,
object_traffic_health: Arc<ObjectTrafficHealth>,
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime,
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache,
}
@@ -126,6 +127,7 @@ impl AppContext {
buffer_config: default_buffer_config_interface(),
object_data_cache,
object_traffic_health: Arc::new(ObjectTrafficHealth::from_env()),
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime::default(),
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache::default(),
}
}
@@ -146,9 +148,14 @@ impl AppContext {
Arc::clone(&self.object_traffic_health)
}
pub(crate) fn table_catalog_strong_runtime(&self) -> crate::table_catalog::StrongTableCatalogRuntime {
self.table_catalog_strong_runtime.clone()
}
pub(crate) fn table_catalog_registration_cache(&self) -> crate::table_catalog::TableCatalogRegistrationCache {
self.table_catalog_registration_cache.clone()
}
pub fn iam(&self) -> Arc<dyn IamInterface> {
self.iam.clone()
}
@@ -358,6 +365,7 @@ impl AppContext {
buffer_config: interfaces.buffer_config,
object_data_cache: ObjectDataCacheAdapter::disabled_arc(),
object_traffic_health: Arc::new(ObjectTrafficHealth::from_env()),
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime::default(),
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache::default(),
}
}
@@ -44,6 +44,10 @@ use uuid::Uuid;
const SEEDED_BUCKET_SIZE: u64 = 123_456;
const SEEDED_BUCKET_OBJECTS: u64 = 42;
/// The smallest snapshot-time step that every supported platform can
/// represent. Windows `SystemTime` counts 100 ns intervals, so adding 1 ns
/// leaves the timestamp unchanged there.
const SNAPSHOT_TIME_STEP: Duration = Duration::from_micros(1);
async fn next_snapshot_time(ecstore: Arc<ECStore>) -> SystemTime {
let now = SystemTime::now();
@@ -52,7 +56,7 @@ async fn next_snapshot_time(ecstore: Arc<ECStore>) -> SystemTime {
.ok()
.and_then(|info| info.last_update)
.unwrap_or(now);
latest.max(now) + Duration::from_nanos(1)
latest.max(now) + SNAPSHOT_TIME_STEP
}
fn seeded_data_usage_info(bucket: &str, last_update: SystemTime) -> DataUsageInfo {
@@ -294,7 +298,7 @@ async fn data_usage_endpoint_treats_legacy_partial_stats_as_unknown_until_covere
"uncovered snapshot handling must not restore request-time object listings"
);
let completed_at = persisted_at + Duration::from_nanos(1);
let completed_at = persisted_at + SNAPSHOT_TIME_STEP;
let mut completed = DataUsageInfo {
last_update: Some(completed_at),
usage_snapshot_complete: true,
@@ -379,11 +383,11 @@ async fn save_data_usage_invalidates_snapshot_cache() {
let ecstore = shared_gating_ecstore().await;
let bucket = format!("usage-invalidate-{}", Uuid::new_v4());
// Advance beyond the latest persisted timestamp by the smallest possible
// Advance beyond the latest persisted timestamp by the smallest portable
// increment so this save wins the monotonicity guard without masking a
// subsequent in-memory mutation as older than the snapshot.
let warm_at = next_snapshot_time(ecstore.clone()).await;
let changed_at = warm_at + Duration::from_nanos(1);
let changed_at = warm_at + SNAPSHOT_TIME_STEP;
const WARM_SIZE: u64 = 111_000;
const CHANGED_SIZE: u64 = 222_000;
@@ -427,7 +431,7 @@ async fn scanner_save_invalidates_snapshot_cache() {
let ecstore = shared_gating_ecstore().await;
let bucket = format!("usage-scanner-invalidate-{}", Uuid::new_v4());
let warm_at = next_snapshot_time(ecstore.clone()).await;
let changed_at = warm_at + Duration::from_nanos(1);
let changed_at = warm_at + SNAPSHOT_TIME_STEP;
const WARM_SIZE: u64 = 333_000;
const CHANGED_SIZE: u64 = 444_000;
+2 -1
View File
@@ -902,7 +902,8 @@ mod tests {
status: "on".to_owned(),
..Default::default()
}));
let backend = EcStoreTableCatalogObjectBackend::new(Arc::clone(&store));
let backend =
EcStoreTableCatalogObjectBackend::new_with_strong_runtime(Arc::clone(&store), context.table_catalog_strong_runtime());
let catalog =
ConfiguredTableCatalogStore::new_for_test(backend, TableCatalogBackingMode::from_env().expect("catalog mode"));
catalog
+4 -1
View File
@@ -243,7 +243,10 @@ impl DefaultObjectUsecase {
.clone()
.or_else(current_app_context)
.ok_or_else(not_initialized)?;
let backend = EcStoreTableCatalogObjectBackend::new(context.object_store());
let backend = EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
context.object_store(),
context.table_catalog_strong_runtime(),
);
let guard = backend
.acquire_read_lock(bucket, &default_table_bucket_publication_lock_path())
.await
-42
View File
@@ -33,7 +33,6 @@ enum PutStorageTestStage {
#[cfg(test)]
tokio::task_local! {
static PUT_STORAGE_TEST_FAILURE: (PutStorageTestStage, StorageError);
static PUT_STORAGE_TEST_COMPLETION: std::cell::Cell<Option<crate::app::storage_api::object_usecase::WriteCompletion>>;
}
#[cfg(test)]
@@ -1572,8 +1571,6 @@ impl DefaultObjectUsecase {
)
.await
.map_err(ApiError::from)?;
// Keep the request awaiting the rename fan-out that owns its namespace guard.
opts.write_completion = crate::app::storage_api::object_usecase::WriteCompletion::TailDrained;
if let Some(etag) = preserve_etag {
opts.preserve_etag = Some(etag);
}
@@ -1962,8 +1959,6 @@ impl DefaultObjectUsecase {
None
};
#[cfg(test)]
let _ = PUT_STORAGE_TEST_COMPLETION.try_with(|observed| observed.set(Some(opts.write_completion)));
#[cfg(test)]
let injected_commit_error = injected_put_storage_failure(PutStorageTestStage::Commit);
let put_commit_future = {
let store = Arc::clone(&store);
@@ -2232,43 +2227,6 @@ mod tests {
use futures::StreamExt;
use http::{HeaderMap, HeaderName, HeaderValue, Method};
#[tokio::test]
#[serial_test::serial]
async fn s3_put_uses_tail_drained_completion_before_acknowledgement() {
use crate::app::storage_api::object_usecase::WriteCompletion;
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
let (store, context) = real_store_test_context().await;
let bucket = format!("put-tail-completion-{}", Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create PUT completion bucket");
let payload = Bytes::from_static(b"readable after PUT acknowledgement");
let input = PutObjectInput::builder()
.bucket(bucket.clone())
.key("object".to_owned())
.content_length(Some(i64::try_from(payload.len()).expect("small payload length")))
.body(Some(StreamingBlob::from_bytes(payload.clone())))
.build()
.expect("PUT completion request");
let usecase = DefaultObjectUsecase::with_context(Some(context));
let fs = FS::new();
let (response, observed) = PUT_STORAGE_TEST_COMPLETION
.scope(std::cell::Cell::new(None), async {
let response = Box::pin(usecase.execute_put_object(&fs, build_request(input, Method::PUT))).await;
(response, PUT_STORAGE_TEST_COMPLETION.with(std::cell::Cell::get))
})
.await;
response.expect("PUT completion response");
assert_eq!(observed, Some(WriteCompletion::TailDrained));
let object = store
.get_object_info(&bucket, "object", &ObjectOptions::default())
.await
.expect("acknowledged PUT metadata remains readable");
assert_eq!(object.size, i64::try_from(payload.len()).expect("small payload length"));
}
#[test]
#[serial_test::serial]
fn put_commit_lock_failure_keeps_request_operation_in_spawned_owner() {
+8 -1
View File
@@ -650,10 +650,17 @@ mod tests {
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create test bucket");
// A literal key shaped like a cache marker. Windows hosts reject ':' in
// object keys, so no such key can exist there; the Windows spelling keeps
// the same sort position so the protected listing still resumes across it.
#[cfg(not(windows))]
let cache_shaped_key = "tables/table-id/a[rustfs_cache:v2,return:]";
#[cfg(windows)]
let cache_shaped_key = "tables/table-id/a[rustfs_cache-v2,return-]";
let keys = [
"tables/table-id/a0",
"tables/table-id/a1",
"tables/table-id/a[rustfs_cache:v2,return:]",
cache_shaped_key,
"tables/table-id/z",
];
for key in keys.into_iter().chain(["tables/zz-denied/hidden"]) {
+9 -1
View File
@@ -6531,8 +6531,16 @@ mod tests {
.expect("fixture object");
let listener = TcpListener::bind("127.0.0.1:0").await.expect("receiver listener");
let address = listener.local_addr().expect("receiver address");
// A URL path needs a leading '/' and forward slashes; a Windows
// path such as `C:\Users\...` has neither, so it would run into the port.
let disk_path = root.path().to_string_lossy().replace('\\', "/");
let disk_path = if disk_path.starts_with('/') {
disk_path
} else {
format!("/{disk_path}")
};
let mut endpoint =
Endpoint::try_from(format!("http://{address}{}", root.path().display()).as_str()).expect("receiver endpoint");
Endpoint::try_from(format!("http://{address}{disk_path}").as_str()).expect("receiver endpoint");
endpoint.is_local = true;
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
+19 -1
View File
@@ -2250,7 +2250,25 @@ mod tests {
#[cfg(unix)]
std::os::unix::fs::symlink(&real_dir, &symlink_dir).unwrap();
#[cfg(windows)]
std::os::windows::fs::symlink_dir(&real_dir, &symlink_dir).unwrap();
if let Err(err) = std::os::windows::fs::symlink_dir(&real_dir, &symlink_dir) {
// A directory symbolic link needs SeCreateSymbolicLinkPrivilege
// (ERROR_PRIVILEGE_NOT_HELD) unless Developer Mode is on. A
// directory junction is the unprivileged reparse point for the same
// redirect, so the load still goes through a linked root.
const ERROR_PRIVILEGE_NOT_HELD: i32 = 1314;
assert_eq!(err.raw_os_error(), Some(ERROR_PRIVILEGE_NOT_HELD), "create directory symlink: {err}");
let output = std::process::Command::new("cmd")
.args(["/C", "mklink", "/J"])
.arg(&symlink_dir)
.arg(&real_dir)
.output()
.expect("run mklink");
assert!(output.status.success(), "create directory junction: {output:?}");
}
assert!(
fs::symlink_metadata(&symlink_dir).unwrap().file_type().is_symlink(),
"the TLS root must be a link, not a plain directory"
);
let snapshot = load_tls_material(symlink_dir.to_str().unwrap())
.await
+7
View File
@@ -414,6 +414,11 @@ mod tests {
vec![format!("{root}/pool-a/disk{{1...4}}"), format!("{root}/pool-b/disk{{1...2}}")]
}
// The pool directories are never created, and this test only covers
// storage-class validation order. Unix leaves missing paths out of the
// physical-disk independence check, but Windows resolves them to their
// volume, so every disk here would count as one shared device. The callers
// therefore bypass that check, which has its own tests in `layout::endpoints`.
async fn assert_invalid_storage_class_fails_before_foundation_mutation(embedded: bool) {
let temp_dir = tempfile::tempdir().expect("create topology root");
let volumes = heterogeneous_volume_args(&temp_dir);
@@ -446,6 +451,7 @@ mod tests {
(OPTIMIZE_ENV, None),
(INLINE_BLOCK_ENV, None),
(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, None),
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true")),
],
assert_invalid_storage_class_fails_before_foundation_mutation(false),
)
@@ -462,6 +468,7 @@ mod tests {
(OPTIMIZE_ENV, None),
(INLINE_BLOCK_ENV, None),
(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, None),
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true")),
],
assert_invalid_storage_class_fails_before_foundation_mutation(true),
)
+6 -89
View File
@@ -2060,7 +2060,10 @@ fn table_catalog_backend_for_data_plane<T>(
None => runtime_sources::current_app_context(),
}
.ok_or_else(object_store_not_initialized_error)?;
Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new(context.object_store()))
Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
context.object_store(),
context.table_catalog_strong_runtime(),
))
}
fn table_catalog_store_for_data_plane<T>(
@@ -2325,7 +2328,7 @@ async fn table_data_plane_resource_for_request<T>(
.get::<TableDataPlanePublicationGuards>()
.cloned()
.unwrap_or_default();
let publication_held = {
{
let state = retained.state.lock();
if state.missing_resources.contains(&key) {
return Ok(None);
@@ -2339,16 +2342,11 @@ async fn table_data_plane_resource_for_request<T>(
{
return Ok(Some(resource));
}
!state.keys.is_empty()
};
}
let store = table_catalog_store_for_data_plane(req)?;
let resource = if catalog_metadata {
crate::table_catalog::table_metadata_data_plane_resource_for_object(&store, bucket, object).await
} else if publication_held {
store
.resolve_table_data_plane_resource_for_authorization(bucket, object)
.await
} else {
crate::table_catalog::table_data_plane_resource_for_object(&store, bucket, object).await
}
@@ -4730,87 +4728,6 @@ mod tests {
.expect("dropping the request must release its publication guard");
}
#[tokio::test]
#[serial]
async fn table_data_plane_cold_lookup_does_not_wait_for_backup_while_holding_publication() {
use crate::table_catalog::{TableCatalogObjectBackend as _, TableCatalogStore as _};
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
let bucket = "cold-authorization";
object_store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create table bucket");
let context = Arc::new(AppContext::new(object_store.clone(), Arc::new(UnreadyIam), Arc::new(TestKms)));
let server_ctx = ServerContextSlot::new();
assert!(server_ctx.install(context));
let mut request = build_request((), Method::PUT);
request.extensions.insert(server_ctx.clone());
let backend = super::table_catalog_backend_for_data_plane(&request).expect("request backend");
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
backend.clone(),
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
);
catalog
.put_table_bucket(crate::table_catalog::TableBucketEntry {
version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION,
table_bucket: bucket.to_string(),
catalog_type: crate::table_catalog::TABLE_BUCKET_CATALOG_TYPE.to_string(),
warehouse_root: format!("s3://{bucket}/"),
state: crate::table_catalog::TableCatalogEntryState::Active,
properties: Default::default(),
active_rename_id: None,
created_at: None,
updated_at: None,
})
.await
.expect("newly enabled catalog entry");
super::retain_table_bucket_publication_guard(&mut request, bucket)
.await
.expect("S3 owns publication first");
let backup_guard = backend
.acquire_write_lock(
crate::admin::storage_api::RUSTFS_META_BUCKET,
&crate::table_catalog::TableCatalogObjectPaths::default().catalog_backup_fence_lock_path(bucket),
)
.await
.expect("backup owns its catalog fence");
let resource = tokio::time::timeout(
std::time::Duration::from_secs(5),
table_data_plane_resource_for_request(&mut request, bucket, "ordinary-object", true),
)
.await
.expect("cold S3 lookup must not wait for the backup catalog fence")
.expect("resolve newly enabled table bucket");
assert!(resource.is_none());
assert!(
request
.extensions
.get::<TableDataPlanePublicationGuards>()
.expect("retained publication")
.state
.lock()
.missing_resources
.contains(&(bucket.to_string(), "ordinary-object".to_string()))
);
drop(backup_guard);
drop(request);
let mut read_request = build_request((), Method::GET);
read_request.extensions.insert(server_ctx);
table_data_plane_resource_for_request(&mut read_request, bucket, "ordinary-object", true)
.await
.expect("unfenced reads retain normal index initialization");
assert!(
backend
.read_object(
crate::admin::storage_api::RUSTFS_META_BUCKET,
&crate::table_catalog::TableCatalogObjectPaths::default().warehouse_index_state_path(bucket),
)
.await
.expect("read initialized index marker")
.is_some()
);
}
#[test]
fn table_data_plane_mutations_fence_before_table_bucket_marker_lookup() {
let source = include_str!("access.rs");
+2
View File
@@ -523,6 +523,8 @@ pub(crate) mod table {
#[cfg(test)]
pub(crate) mod test_support {
pub(crate) use crate::storage::storage_api::contract::bucket::{BucketOperations, MakeBucketOptions};
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::init_bucket_metadata_sys;
pub(crate) use crate::storage::storage_api::ecstore_bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
}
}
@@ -61,6 +61,27 @@ pub(crate) fn metadata_log_locations_matching(
locations
}
pub(crate) async fn metadata_locations_for_protected_snapshot_refs<B>(
backend: &B,
table_bucket: &str,
namespace: &Namespace,
table: &IdentifierSegment,
current_metadata: &serde_json::Value,
metadata_locations: &[String],
) -> TableCatalogStoreResult<BTreeSet<String>>
where
B: TableCatalogObjectBackend,
{
metadata_locations_for_protected_snapshot_refs_matching(
backend,
table_bucket,
current_metadata,
metadata_locations,
|location| is_valid_table_metadata_location(namespace, table, location),
)
.await
}
pub(crate) async fn metadata_locations_for_protected_snapshot_refs_matching<B>(
backend: &B,
table_bucket: &str,
+9 -104
View File
@@ -2756,28 +2756,11 @@ struct SnapshotGraphManifestLocation {
from_manifest_list: bool,
}
enum SnapshotGraphValidation<'a> {
Commit(Option<&'a serde_json::Value>),
BackupInventory,
}
pub(crate) async fn validate_table_snapshot_changes<B>(
context: &TableSnapshotGraphValidationContext<'_, B>,
current_metadata: Option<&serde_json::Value>,
metadata: &serde_json::Value,
) -> TableCatalogStoreResult<()>
where
B: TableCatalogObjectBackend,
{
validate_table_snapshot_changes_with_budget(context, SnapshotGraphValidation::Commit(current_metadata), metadata).await?;
Ok(())
}
async fn validate_table_snapshot_changes_with_budget<B>(
context: &TableSnapshotGraphValidationContext<'_, B>,
validation: SnapshotGraphValidation<'_>,
metadata: &serde_json::Value,
) -> TableCatalogStoreResult<SnapshotGraphReadBudget>
where
B: TableCatalogObjectBackend,
{
@@ -2787,44 +2770,29 @@ where
let mut budget = SnapshotGraphReadBudget::default();
validate_table_statistics_objects(context, metadata).await?;
let format_version = table_metadata_format_version(metadata)?;
let current_metadata = match validation {
SnapshotGraphValidation::Commit(current_metadata) => current_metadata,
SnapshotGraphValidation::BackupInventory => None,
};
let snapshots = snapshots_requiring_graph_validation(current_metadata, metadata)?;
for snapshot in snapshots {
snapshot
.get("snapshot-id")
.and_then(serde_json::Value::as_i64)
.ok_or_else(|| TableCatalogStoreError::Invalid("snapshot-id must be an integer".to_string()))?;
if format_version == 2
&& snapshot.get("manifests").is_some()
&& !snapshot_is_retained_v1_history(current_metadata, snapshot)
{
return Err(TableCatalogStoreError::Invalid(
"new Iceberg v2 snapshots require manifest-list".to_string(),
));
}
let snapshot_sequence_number = snapshot
.get("sequence-number")
.and_then(serde_json::Value::as_i64)
.unwrap_or(0);
let graph_format_version = if format_version == 2 && snapshot.get("manifests").is_some() {
match validation {
// A persisted upgrade can retain an inline v1 snapshot. Walk
// its full graph using v1 manifest and file constraints.
SnapshotGraphValidation::BackupInventory if snapshot_sequence_number == 0 => 1,
SnapshotGraphValidation::Commit(current_metadata)
if snapshot_is_retained_v1_history(current_metadata, snapshot) =>
{
format_version
}
_ => {
return Err(TableCatalogStoreError::Invalid(
"new Iceberg v2 snapshots require manifest-list".to_string(),
));
}
}
} else {
format_version
};
let manifests = snapshot_graph_manifest_references(
context,
metadata,
snapshot,
graph_format_version,
format_version,
snapshot_sequence_number,
&mut budget,
)
@@ -2840,69 +2808,6 @@ where
}
}
}
Ok(budget)
}
/// Validates every retained snapshot and returns its bounded physical object inventory.
///
/// Manifest lists and manifests are retained even when all their entries are deleted. Only live data/delete
/// references are included, and every such object has passed the existing existence validation. This uses the
/// validator's existing caches without another storage traversal or Avro decode. Statistics are enumerated
/// separately with `table_statistics_object_references`.
pub(crate) async fn validated_table_snapshot_graph_inventory<B>(
context: &TableSnapshotGraphValidationContext<'_, B>,
metadata: &serde_json::Value,
) -> TableCatalogStoreResult<BTreeMap<String, TableMetadataMaintenanceObjectKind>>
where
B: TableCatalogObjectBackend,
{
let budget = validate_table_snapshot_changes_with_budget(context, SnapshotGraphValidation::BackupInventory, metadata).await?;
let mut inventory = BTreeMap::new();
for object in budget.manifest_lists.keys() {
insert_validated_snapshot_graph_inventory_object(
&mut inventory,
object.clone(),
TableMetadataMaintenanceObjectKind::ManifestList,
)?;
}
for (object, manifest) in &budget.manifests {
insert_validated_snapshot_graph_inventory_object(
&mut inventory,
object.clone(),
TableMetadataMaintenanceObjectKind::ManifestFile,
)?;
for reference in manifest.references.iter() {
if !matches!(reference.entry_status, Some(0 | 1)) {
continue;
}
let object = snapshot_graph_object_key(context, &reference.location, reference.object_kind.clone())?;
if !budget.validated_live_objects.contains(&object) {
return Err(TableCatalogStoreError::Internal(
"snapshot inventory contains a live object without successful existence validation".to_string(),
));
}
insert_validated_snapshot_graph_inventory_object(&mut inventory, object, reference.object_kind.clone())?;
}
}
Ok(inventory)
}
fn insert_validated_snapshot_graph_inventory_object(
inventory: &mut BTreeMap<String, TableMetadataMaintenanceObjectKind>,
object: String,
kind: TableMetadataMaintenanceObjectKind,
) -> TableCatalogStoreResult<()> {
match inventory.entry(object) {
std::collections::btree_map::Entry::Vacant(entry) => {
entry.insert(kind);
}
std::collections::btree_map::Entry::Occupied(entry) if *entry.get() != kind => {
return Err(TableCatalogStoreError::Invalid(
"snapshot inventory object is referenced with conflicting kinds".to_string(),
));
}
std::collections::btree_map::Entry::Occupied(_) => {}
}
Ok(())
}
-1
View File
@@ -303,7 +303,6 @@ pub(crate) fn table_name_from_marker_path(namespace: &Namespace, object_key: &st
.map(ToString::to_string)
}
#[cfg(test)]
pub(crate) fn metadata_location_from_metadata_file_path(
namespace: &Namespace,
table: &IdentifierSegment,
@@ -230,8 +230,8 @@ mod tests {
type TestStore = ConfiguredTableCatalogStore<TestCatalogObjectBackend>;
fn backings() -> [TableCatalogBackingMode; 1] {
[TableCatalogBackingMode::ObjectBacked]
fn backings() -> [TableCatalogBackingMode; 2] {
[TableCatalogBackingMode::ObjectBacked, TableCatalogBackingMode::DurableStrong]
}
fn enabled_config() -> TableMaintenanceConfig {
@@ -16,8 +16,8 @@ use super::periodic::PeriodicMaintenanceScheduler;
use crate::storage_api::startup::services::ServerContextSlot;
use crate::storage_api::table::check_valid_bucket_name_strict;
use crate::table_catalog::{
ConfiguredTableCatalogStore, EcStoreTableCatalogObjectBackend, ObjectTableCatalogStore, TableCatalogBackingMode,
TableCatalogStoreError, TableCatalogStoreResult,
ConfiguredTableCatalogStore, EcStoreTableCatalogObjectBackend, ObjectTableCatalogStore, StrongTableCatalogStore,
TableCatalogBackingMode, TableCatalogStoreError, TableCatalogStoreResult,
};
use std::{collections::BTreeSet, future::Future, sync::Arc, time::Duration};
use tokio::{task::JoinHandle, time::Instant};
@@ -166,11 +166,17 @@ async fn run(config: PeriodicMaintenanceConfig, server: Arc<ServerContextSlot>,
let Some(context) = server.installed_app_context() else {
continue;
};
let backend = EcStoreTableCatalogObjectBackend::new(context.object_store());
let backend = EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
context.object_store(),
context.table_catalog_strong_runtime(),
);
let store = match config.backing {
TableCatalogBackingMode::ObjectBacked => {
ConfiguredTableCatalogStore::ObjectBacked(ObjectTableCatalogStore::new(backend))
}
TableCatalogBackingMode::DurableStrong => {
ConfiguredTableCatalogStore::DurableStrong(StrongTableCatalogStore::new_requiring_snapshot(backend))
}
};
scheduler = Some(PeriodicMaintenanceScheduler::new(store, config.buckets.clone(), config.interval));
}
@@ -219,7 +225,7 @@ mod tests {
.find(|(name, _)| *name == key)
.map(|(_, value)| value.to_string()))
},
|| Ok(TableCatalogBackingMode::ObjectBacked),
|| Ok(TableCatalogBackingMode::DurableStrong),
)
}
@@ -732,6 +732,29 @@ pub(crate) fn metadata_maintenance_reachability_graph_report(
}
}
pub(crate) async fn metadata_maintenance_object_cleanup_reports<B>(
backend: &B,
table_bucket: &str,
namespace: &Namespace,
table: &IdentifierSegment,
warehouse_object_prefix: Option<&str>,
referenced_object_reports: &[TableMetadataMaintenanceReferencedObjectReport],
now: OffsetDateTime,
) -> TableCatalogStoreResult<(usize, Vec<String>, Vec<String>, Vec<TableMetadataMaintenanceObjectCleanupReport>)>
where
B: TableCatalogObjectBackend,
{
metadata_maintenance_object_cleanup_reports_in_scope(
backend,
table_bucket,
TableMaintenanceObjectScope::Logical { namespace, table },
warehouse_object_prefix,
referenced_object_reports,
now,
)
.await
}
pub(crate) async fn metadata_maintenance_object_cleanup_reports_for_entry<B>(
backend: &B,
entry: &TableEntry,
+28 -5
View File
@@ -21,6 +21,7 @@
use std::{
collections::{BTreeMap, BTreeSet},
num::NonZeroUsize,
ops::Bound,
sync::Arc,
time::{Duration as StdDuration, Instant},
};
@@ -70,15 +71,13 @@ mod store;
pub use error::{CatalogIdentifierError, TableObjectMutationError};
pub(crate) use error::{TableCatalogStoreError, TableCatalogStoreResult};
pub(crate) use iceberg::*;
#[cfg(test)]
use identifier::metadata_location_from_metadata_file_path;
pub use identifier::{IdentifierSegment, Namespace, is_reserved_table_object_key};
pub(crate) use identifier::{
default_table_bucket_publication_lock_path, default_table_data_dir_path, default_table_delete_dir_path,
default_table_metadata_dir_path, default_table_metadata_file_path, default_table_publication_lock_path,
default_view_metadata_file_path, is_valid_table_metadata_location, is_valid_table_metadata_location_for_entry,
is_valid_view_metadata_location, table_identity_from_metadata_object_key, table_metadata_dir_path_for_entry,
table_metadata_file_path_for_entry, validate_bucket_object_mutation,
is_valid_view_metadata_location, metadata_location_from_metadata_file_path, table_identity_from_metadata_object_key,
table_metadata_dir_path_for_entry, table_metadata_file_path_for_entry, validate_bucket_object_mutation,
};
pub(crate) use maintenance::*;
pub(crate) use model::*;
@@ -122,7 +121,11 @@ pub(crate) const DEFAULT_TABLE_CATALOG_PUBLICATION_PREPARATION_TIMEOUT_SECS: u64
pub(crate) const MAX_TABLE_CATALOG_PUBLICATION_PREPARATION_TIMEOUT_SECS: u64 = 86_400;
pub(crate) const ENV_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED: &str =
"RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED";
pub(crate) const ENV_TABLE_CATALOG_STRONG_SNAPSHOT_V2: &str = "RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2";
pub(crate) const ENV_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED: &str =
"RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED";
pub(crate) const TABLE_CATALOG_BACKING_OBJECT: &str = "object";
pub(crate) const TABLE_CATALOG_BACKING_DURABLE_STRONG: &str = "durable-strong";
pub(crate) const TABLE_METADATA_DIGEST_REQUIREMENT_TYPE: &str = "assert-rustfs-metadata-sha256";
pub(crate) const TABLE_METADATA_FILE_NAME_MAX_LEN: usize = 128;
pub(crate) const TABLE_METADATA_JSON_MAX_SIZE: usize = 50 * 1024 * 1024;
@@ -196,6 +199,7 @@ const MAINTENANCE_JOB_ALIAS_LATEST: &str = "latest";
const MAINTENANCE_JOB_ALIAS_CURRENT: &str = "current";
const TABLE_CATALOG_LIST_MAX_KEYS: usize = 1000;
const OBJECT_CATALOG_LIST_CURSOR_PREFIX: &str = "object:";
const STRONG_CATALOG_LIST_CURSOR_PREFIX: &str = "strong:";
const TABLE_METADATA_CLEANUP_SAFETY_WINDOW_SECONDS: i64 = 15 * 60;
const TABLE_MAINTENANCE_RETRY_BACKOFF_MAX_SECONDS: u64 = 24 * 60 * 60;
const TABLE_MAINTENANCE_WORKER_LEASE_TIMEOUT_DEFAULT_SECONDS: u64 = 15 * 60;
@@ -210,8 +214,24 @@ const ICEBERG_MAX_REF_AGE_MS_PROPERTY: &str = "history.expire.max-ref-age-ms";
const ICEBERG_REF_MIN_SNAPSHOTS_TO_KEEP_FIELD: &str = "min-snapshots-to-keep";
const ICEBERG_REF_MAX_SNAPSHOT_AGE_MS_FIELD: &str = "max-snapshot-age-ms";
const ICEBERG_REF_MAX_REF_AGE_MS_FIELD: &str = "max-ref-age-ms";
const STRONG_TABLE_CATALOG_SNAPSHOT_MIN_READ_VERSION: u16 = 1;
const STRONG_TABLE_CATALOG_SNAPSHOT_VERSION: u16 = 2;
const STRONG_TABLE_CATALOG_BACKING_ROOT: &str = "strong-backing";
const STRONG_TABLE_CATALOG_SNAPSHOT_FILE: &str = "snapshot.json";
const TABLE_CATALOG_MIGRATION_MIN_READ_VERSION: u16 = 1;
const TABLE_CATALOG_MIGRATION_VERSION: u16 = 2;
const TABLE_CATALOG_MIGRATION_ROOT: &str = "backing-migration";
const TABLE_CATALOG_MIGRATION_FENCE_FILE: &str = "durable-strong-fence.json";
const TABLE_CATALOG_MIGRATION_FENCE_LOCK: &str = "durable-strong-fence.lock";
const TABLE_CATALOG_MIGRATION_GLOBAL_FENCE_FILE: &str = "durable-strong-global-fence.json";
const TABLE_CATALOG_MIGRATION_GLOBAL_FENCE_LOCK: &str = "durable-strong-global-fence.lock";
pub(crate) const TABLE_CATALOG_BACKUP_VERSION: u16 = 2;
const TABLE_CATALOG_BACKUP_ROOT: &str = "backups";
const TABLE_CATALOG_BACKUP_RESTORE_ROOT: &str = "restore-intents";
const TABLE_CATALOG_BACKUP_RESTORE_INTENT_FILE: &str = "intent.json";
const TABLE_CATALOG_BACKUP_MAX_SIZE: usize = 64 * 1024 * 1024;
const TABLE_CATALOG_BACKUP_MAX_OBJECTS: usize = 1_000_000;
const TABLE_CATALOG_BACKUP_HASH_MAX_SIZE: u64 = 16 * 1024 * 1024;
const TABLE_CATALOG_BACKUP_MAX_MAINTENANCE_OBJECT_SIZE: usize = 4 * 1024 * 1024;
const TABLE_CATALOG_BACKUP_MAX_MAINTENANCE_BYTES: usize = 32 * 1024 * 1024;
@@ -283,6 +303,7 @@ impl<T> TableCatalogStorage for T where
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum TableCatalogBackingMode {
ObjectBacked,
DurableStrong,
}
impl TableCatalogBackingMode {
@@ -299,8 +320,9 @@ impl TableCatalogBackingMode {
fn parse(value: &str) -> TableCatalogStoreResult<Self> {
match value.trim() {
"" | TABLE_CATALOG_BACKING_OBJECT => Ok(Self::ObjectBacked),
TABLE_CATALOG_BACKING_DURABLE_STRONG => Ok(Self::DurableStrong),
value => Err(TableCatalogStoreError::Invalid(format!(
"unsupported table catalog backing {value}; expected {TABLE_CATALOG_BACKING_OBJECT}"
"unsupported table catalog backing {value}; expected {TABLE_CATALOG_BACKING_OBJECT} or {TABLE_CATALOG_BACKING_DURABLE_STRONG}"
))),
}
}
@@ -308,6 +330,7 @@ impl TableCatalogBackingMode {
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::ObjectBacked => TABLE_CATALOG_BACKING_OBJECT,
Self::DurableStrong => TABLE_CATALOG_BACKING_DURABLE_STRONG,
}
}
}
+146 -2
View File
@@ -65,8 +65,6 @@ pub(crate) struct TableBucketEntry {
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableRenameIntentState {
Prepared,
/// The maintenance plan is durable before any destination sidecar is written.
MaintenancePrepared,
SourceFenced,
DestinationWritten,
SourceTombstoned,
@@ -1088,6 +1086,7 @@ pub(crate) struct TableCatalogExport {
pub(crate) struct TableCatalogBackingManifest {
pub version: u16,
pub current: TableCatalogBackingProfile,
pub migration: TableCatalogBackingMigrationPlan,
pub ha: TableCatalogHaPolicy,
pub scale_validation: TableCatalogScaleValidation,
}
@@ -1107,24 +1106,42 @@ pub(crate) struct TableCatalogBackingProfile {
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogBackingKind {
ObjectBacked,
// RUSTFS_COMPAT_TODO(table-catalog-backing-manifest-v1-wire-labels): Keep the version 1 wire label for existing clients. Remove after a versioned manifest with an explicit client migration contract replaces it.
#[serde(rename = "STRONG_KV_WAL")]
DurableStrongSnapshot,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogAuthority {
RustfsSysObject,
#[allow(
dead_code,
reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)"
)]
LinearizableMetadataKv,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogConsistencyMode {
ConditionalObjectCas,
#[allow(
dead_code,
reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)"
)]
LinearizableCas,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogDurabilityMode {
StagedCommitLogBeforePointerUpdate,
#[allow(
dead_code,
reason = "exercised by table_catalog/tests.rs; the lib target cannot see test-only consumers (backlog#1823)"
)]
WalBeforeStateMachineApply,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
@@ -1160,6 +1177,132 @@ pub(crate) struct TableCatalogSnapshotState {
pub snapshot_version: Option<u16>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct TableCatalogBackingMigrationPlan {
pub source_kind: TableCatalogBackingKind,
pub target_kind: TableCatalogBackingKind,
pub status: TableCatalogBackingMigrationStatus,
pub required_steps: Vec<TableCatalogBackingMigrationStep>,
pub blockers: Vec<TableCatalogBackingMigrationBlocker>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct TableCatalogBackingMigrationDryRunReport {
pub table_bucket: String,
pub source_kind: TableCatalogBackingKind,
pub target_kind: TableCatalogBackingKind,
pub status: TableCatalogBackingMigrationStatus,
pub namespace_count: usize,
pub table_count: usize,
pub view_count: usize,
pub commit_log_count: usize,
pub idempotency_index_count: usize,
pub warehouse_prefix_count: usize,
pub warehouse_index_ready: bool,
pub object_backed_writes_fenced: bool,
pub ready_to_enable_durable_strong: bool,
pub blockers: Vec<TableCatalogBackingMigrationBlocker>,
pub recommended_actions: Vec<TableCatalogBackingMigrationAction>,
pub rollback: TableCatalogBackingRollbackPlan,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct TableCatalogBackingMigrationExecutionReport {
pub table_bucket: String,
pub source_kind: TableCatalogBackingKind,
pub target_kind: TableCatalogBackingKind,
pub status: TableCatalogBackingMigrationExecutionStatus,
pub namespace_count: usize,
pub table_count: usize,
pub view_count: usize,
pub commit_log_count: usize,
pub idempotency_index_count: usize,
pub source_fingerprint: String,
pub target_snapshot_etag: String,
pub object_backed_writes_fenced: bool,
pub ready_to_enable_durable_strong: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogBackingMigrationExecutionStatus {
SnapshotMaterialized,
SnapshotAlreadyMaterialized,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct TableCatalogBackingMigrationCancelReport {
pub table_bucket: String,
pub status: TableCatalogBackingMigrationCancelStatus,
pub object_backed_writes_fenced: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogBackingMigrationCancelStatus {
FenceReleased,
NoMigrationFence,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogBackingMigrationStatus {
ReadyToSnapshot,
SnapshotMaterialized,
RecoveryRequired,
ManualReviewRequired,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogBackingMigrationStep {
SnapshotCatalogExport,
ReplayCommitLog,
VerifyCurrentPointer,
EnableSingleWriterFencing,
// RUSTFS_COMPAT_TODO(table-catalog-backing-manifest-v1-wire-labels): Keep the version 1 wire label for existing clients. Remove after a versioned manifest with an explicit client migration contract replaces it.
#[serde(rename = "CUT_OVER_LINEARIZABLE_READS")]
CutOverDurableSnapshotReads,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogBackingMigrationBlocker {
TableRenameRecoveryRequired,
CommitRecoveryRequired,
CommitManualReviewRequired,
WarehouseIndexBackfillRequired,
DuplicateWarehousePrefix,
DuplicateTableIdentity,
TableViewIdentifierCollision,
DurableStrongSnapshotChanged,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogBackingMigrationAction {
RunCatalogRecovery,
BackfillWarehouseIndex,
ReviewDuplicateWarehousePrefixes,
ReviewDuplicateTableIdentities,
ReviewTableViewIdentifierCollisions,
SnapshotObjectBackedCatalog,
EnableDurableStrongBacking,
VerifyDurableStrongSnapshot,
SnapshotRemainingTableBuckets,
ReviewDurableStrongSnapshot,
KeepObjectBackedRollbackConfig,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct TableCatalogBackingRollbackPlan {
pub backing_config_key: &'static str,
pub current_backing_value: &'static str,
pub rollback_backing_value: &'static str,
pub preserves_object_backed_catalog: bool,
pub requires_operator_restart: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct TableCatalogHaPolicy {
pub writer_region_model: TableCatalogHaWriterModel,
@@ -1205,6 +1348,7 @@ pub(crate) enum TableCatalogScaleValidationStatus {
pub(crate) enum TableCatalogScaleValidationScenario {
ConcurrentCommitCas,
CommitLogRecoveryReplay,
MigrationSnapshotReplay,
ReadReplicaStaleReadGuard,
ClientConformanceMatrix,
}
File diff suppressed because it is too large Load Diff
@@ -1,74 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use serde::Serialize;
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogBackupStatus {
Created,
AlreadyPresent,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogBackupObjectPreparationStatus {
Prepared,
AlreadyPresent,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct TableCatalogBackupObjectPreparationReport {
pub table_bucket: String,
pub backup_id: String,
pub object: String,
pub status: TableCatalogBackupObjectPreparationStatus,
pub size_bytes: u64,
pub sha256: String,
pub etag: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct TableCatalogBackupReport {
pub table_bucket: String,
pub backup_id: String,
pub status: TableCatalogBackupStatus,
pub backup_path: String,
pub source_snapshot_etag: String,
pub source_snapshot_version: u16,
pub catalog_fingerprint: String,
pub object_count: usize,
pub verified_object_count: usize,
pub verified_bytes: u64,
pub maintenance_object_count: usize,
pub created_at: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(crate) enum TableCatalogRestoreStatus {
Restored,
AlreadyRestored,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(crate) struct TableCatalogRestoreReport {
pub table_bucket: String,
pub backup_id: String,
pub status: TableCatalogRestoreStatus,
pub previous_snapshot_etag: Option<String>,
pub restored_snapshot_etag: String,
pub object_count: usize,
pub maintenance_object_count: usize,
}
+20 -5
View File
@@ -58,7 +58,14 @@ impl<B: TableCatalogObjectBackend> ObjectTableCatalogStore<B> {
validate_table_bucket_entry(&entry)?;
let mut guards = Vec::new();
if publication.is_some_and(|publication| publication.catalog_migration_read_permit_status() == Some(true)) {
self.reject_legacy_catalog_global_state().await?;
if self
.backend
.read_object(self.catalog_bucket(), &self.paths.backing_migration_global_fence_path())
.await?
.is_some()
{
return Err(cleanup_conflict("table bucket registry migration is in progress"));
}
self.ensure_object_backed_catalog_write_permit_after_lock(&entry.table_bucket)
.await?;
} else {
@@ -246,8 +253,12 @@ impl<B: TableCatalogObjectBackend> ConfiguredTableCatalogStore<B> {
entry: TableBucketEntry,
publication: &impl TableCatalogObjectBackend,
) -> TableCatalogStoreResult<EmptyTableBucketCleanup> {
let Self::ObjectBacked(store) = self;
store.prepare_empty_table_bucket_cleanup(entry, publication).await
match self {
Self::ObjectBacked(store) => store.prepare_empty_table_bucket_cleanup(entry, publication).await,
Self::DurableStrong(_) => Err(TableCatalogStoreError::Unsupported(
"empty table bucket cleanup requires object-backed catalog storage".to_string(),
)),
}
}
pub(crate) async fn execute_empty_table_bucket_cleanup(
@@ -255,7 +266,11 @@ impl<B: TableCatalogObjectBackend> ConfiguredTableCatalogStore<B> {
cleanup: &mut EmptyTableBucketCleanup,
incarnation: Uuid,
) -> TableCatalogStoreResult<()> {
let Self::ObjectBacked(store) = self;
store.execute_empty_table_bucket_cleanup(cleanup, incarnation).await
match self {
Self::ObjectBacked(store) => store.execute_empty_table_bucket_cleanup(cleanup, incarnation).await,
Self::DurableStrong(_) => Err(TableCatalogStoreError::Unsupported(
"empty table bucket cleanup requires object-backed catalog storage".to_string(),
)),
}
}
}
@@ -266,7 +266,6 @@ impl<B: TableCatalogObjectBackend> ObjectTableCatalogStore<B> {
.backend
.acquire_write_lock(self.catalog_bucket(), &self.paths.backing_migration_fence_lock_path(table_bucket))
.await?;
self.ensure_no_active_restore(table_bucket).await?;
if self
.backend
.object_metadata(self.catalog_bucket(), &fence_path)
@@ -370,7 +369,6 @@ impl<B: TableCatalogObjectBackend> ObjectTableCatalogStore<B> {
.backend
.acquire_write_lock(self.catalog_bucket(), &self.paths.backing_migration_fence_lock_path(table_bucket))
.await?;
self.ensure_no_active_restore(table_bucket).await?;
if self.read_table_directory_state(table_bucket).await?.is_some() {
// Entry publication has always retained the legacy objects. Exclude
// backfill and all writers until the old authority is restored.
@@ -633,25 +631,6 @@ fn validate_directory_page(
#[cfg(test)]
mod tests {
use super::*;
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
enum LegacyBackingMigrationFenceStatus {
Preparing,
Materialized,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct LegacyBackingMigrationFence {
version: u16,
table_bucket: String,
migration_id: String,
status: LegacyBackingMigrationFenceStatus,
target_bucket_existed: bool,
source_fingerprint: Option<String>,
target_snapshot_etag: Option<String>,
}
use crate::table_catalog::test_support::TestCatalogObjectBackend;
use crate::table_catalog::tests::{test_bucket_entry, test_namespace_entry, test_table_entry};
@@ -737,7 +716,7 @@ mod tests {
.unwrap()
.unwrap();
assert!(
serde_json::from_slice::<LegacyBackingMigrationFence>(&fence.data).is_err(),
serde_json::from_slice::<super::super::strong::TableCatalogBackingMigrationFence>(&fence.data).is_err(),
"legacy writer parser must reject the directory capability fence"
);
finish(&store).await;
@@ -787,39 +766,6 @@ mod tests {
);
}
#[tokio::test]
async fn backup_and_restore_treat_directory_objects_as_derived() {
let (backend, store) = fixture().await;
let baseline = store
.create_object_catalog_backup("warehouse", None)
.await
.expect("empty enabled catalog is a valid backup source");
let namespace = Namespace::parse("sales").unwrap();
let orphan = entry("warehouse", &namespace, "aborted");
let table = IdentifierSegment::parse("aborted").unwrap();
let target = store.paths.table_entry_path("warehouse", &namespace, &table);
let candidate = store.paths.table_directory_candidate(&orphan).unwrap();
// This is the exact immutable path reference a writer persists before its
// entry PUT. An aborted PUT followed by rollback leaves only the candidate.
backend.seed_object(RUSTFS_META_BUCKET, &candidate, target.into_bytes()).await;
finish(&store).await;
let with_directory = store.create_object_catalog_backup("warehouse", None).await.unwrap();
assert_eq!(with_directory.backup_id, baseline.backup_id);
assert!(matches!(
store.restore_object_catalog_backup("warehouse", &baseline.backup_id, None, false).await,
Err(TableCatalogStoreError::Conflict(message)) if message.contains("disable the table directory")
));
store.disable_table_directory("warehouse").await.unwrap();
let after_rollback = store.create_object_catalog_backup("warehouse", None).await.unwrap();
assert_eq!(after_rollback.backup_id, baseline.backup_id);
store
.restore_object_catalog_backup("warehouse", &after_rollback.backup_id, None, false)
.await
.expect("derived candidates must not count as restore targets");
assert!(backend.object_exists(RUSTFS_META_BUCKET, &candidate).await.unwrap());
}
#[tokio::test]
async fn directory_index_excludes_reports_but_preserves_colliding_nested_namespaces() {
let (backend, store) = fixture().await;
@@ -1033,55 +979,6 @@ mod tests {
"renamed"
);
}
#[tokio::test]
async fn directory_activation_drains_in_flight_entry_writer_before_first_page() {
let (backend, store) = fixture().await;
let namespace = Namespace::parse("sales").unwrap();
store
.create_namespace(test_namespace_entry("warehouse", &namespace))
.await
.unwrap();
let table = entry("warehouse", &namespace, "orders");
let table_path = store
.paths
.table_entry_path("warehouse", &namespace, &IdentifierSegment::parse("orders").unwrap());
let pause = backend.pause_next_put(RUSTFS_META_BUCKET, &table_path).await;
let create_store = store.clone();
let create = tokio::spawn(async move { create_store.create_table(table).await });
tokio::time::timeout(StdDuration::from_secs(5), pause.wait_started())
.await
.unwrap();
// An in-flight writer has checked that directory state is absent and
// holds its per-bucket permit. Activation must queue on the same key.
let lock_path = store.paths.catalog_backup_fence_lock_path("warehouse");
let writes = backend.write_lock_acquisition_count(RUSTFS_META_BUCKET, &lock_path).await;
let advance_store = store.clone();
let mut advance = tokio::spawn(async move {
advance_store
.advance_table_directory_with_confirmation("warehouse", true, true)
.await
});
tokio::time::timeout(StdDuration::from_secs(5), async {
tokio::select! {
result = &mut advance => panic!("activation passed an in-flight writer: {result:?}"),
() = async {
while backend.write_lock_acquisition_count(RUSTFS_META_BUCKET, &lock_path).await == writes {
tokio::task::yield_now().await;
}
} => {}
}
})
.await
.expect("activation must queue on the writer's per-bucket fence");
pause.release();
create.await.unwrap().unwrap();
assert_eq!(advance.await.unwrap().unwrap().phase, Some(TableDirectoryPhase::Building));
finish(&store).await;
let scan = store.table_directory_objects("warehouse", None).await.unwrap().unwrap();
assert_eq!(scan.objects, vec![table_path]);
}
#[tokio::test]
async fn directory_index_cancelled_initial_write_retains_guard_until_rollback_can_finish() {
let (backend, store) = fixture().await;
@@ -1381,7 +1278,7 @@ mod tests {
use crate::storage_api::table::test_support::{BucketOperations as _, MakeBucketOptions};
let (_temp, paths, storage) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
storage.make_bucket("warehouse", &MakeBucketOptions::default()).await.unwrap();
let backend = EcStoreTableCatalogObjectBackend::new(storage);
let backend = EcStoreTableCatalogObjectBackend::new_with_strong_runtime(storage, StrongTableCatalogRuntime::default());
let catalog = ObjectTableCatalogStore::new(backend.clone());
catalog.put_table_bucket(test_bucket_entry("warehouse")).await.unwrap();
let prefix = catalog.paths.namespace_entries_prefix("warehouse");
@@ -1,81 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
pub(super) fn table_catalog_backing_manifest(
paths: &TableCatalogObjectPaths,
namespace: &Namespace,
table: &IdentifierSegment,
entry: &TableEntry,
commit_recovery: &TableCommitRecoveryReport,
) -> TableCatalogBackingManifest {
let recovery_required = commit_recovery.staged_before_table_update_count > 0
|| commit_recovery.finalization_required_count > 0
|| commit_recovery.idempotency_repair_required_count > 0;
let manual_review_required = commit_recovery.manual_review_count > 0;
let wal_status = if manual_review_required {
TableCatalogWalStatus::ManualReviewRequired
} else if recovery_required {
TableCatalogWalStatus::RecoveryRequired
} else {
TableCatalogWalStatus::Recoverable
};
TableCatalogBackingManifest {
version: TABLE_CATALOG_BACKING_MANIFEST_VERSION,
current: TableCatalogBackingProfile {
kind: TableCatalogBackingKind::ObjectBacked,
authority: TableCatalogAuthority::RustfsSysObject,
consistency: TableCatalogConsistencyMode::ConditionalObjectCas,
durability: TableCatalogDurabilityMode::StagedCommitLogBeforePointerUpdate,
current_pointer_path: paths.table_entry_path(&entry.table_bucket, namespace, table),
wal: TableCatalogWalState {
status: wal_status,
commit_log_prefix: paths.commit_log_entries_prefix(&entry.table_bucket, &entry.table_id),
idempotency_index_prefix: paths.commit_idempotency_entries_prefix(&entry.table_bucket, &entry.table_id),
committed_generation: entry.generation,
staged_before_table_update_count: commit_recovery.staged_before_table_update_count,
finalization_required_count: commit_recovery.finalization_required_count,
idempotency_repair_required_count: commit_recovery.idempotency_repair_required_count,
manual_review_count: commit_recovery.manual_review_count,
},
snapshot: TableCatalogSnapshotState {
export_api: "GET /iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}/catalog/export".to_string(),
includes_table_bucket: true,
includes_namespace: true,
includes_table_pointer: true,
includes_backing_manifest: true,
snapshot_etag: None,
snapshot_version: None,
},
},
ha: TableCatalogHaPolicy {
writer_region_model: TableCatalogHaWriterModel::SingleActiveWriterRegion,
read_replica_strategy: TableCatalogReadReplicaStrategy::ReadOnlyReplicasForListAndLoad,
commit_read_requirement: TableCatalogCommitReadRequirement::LinearizableLeaderRead,
active_active_supported: false,
failover_requires_operator_promotion: true,
},
scale_validation: TableCatalogScaleValidation {
status: TableCatalogScaleValidationStatus::MatrixPublished,
benchmark_required: true,
required_scenarios: vec![
TableCatalogScaleValidationScenario::ConcurrentCommitCas,
TableCatalogScaleValidationScenario::CommitLogRecoveryReplay,
TableCatalogScaleValidationScenario::ReadReplicaStaleReadGuard,
TableCatalogScaleValidationScenario::ClientConformanceMatrix,
],
},
}
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,156 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::super::object::{validate_commit_idempotency_entry_object, validate_commit_log_entry_object};
use super::*;
impl<B> ObjectTableCatalogStore<B>
where
B: TableCatalogObjectBackend,
{
/// Rejects a captured snapshot that still requires commit/index recovery.
///
/// This reads only the supplied snapshot. Callers hold the backup/publication
/// fences and validate ordinary artifact hashes and metadata separately.
/// Recovery classification is shared with the normal commit-recovery API;
/// this check never finalizes a log, repairs an index, or changes a pointer.
pub(super) fn validate_backup_commit_recovery(
&self,
table_bucket: &str,
objects: &[ObjectCatalogBackupObject],
) -> TableCatalogStoreResult<()> {
let mut active = BTreeMap::<String, TableEntry>::new();
let mut logs = BTreeMap::<String, BTreeMap<String, CommitLogEntry>>::new();
let mut indexes = BTreeMap::<String, BTreeMap<String, CommitLogEntry>>::new();
let mut seen_paths = BTreeSet::new();
let root = self.paths.table_bucket_root_prefix(table_bucket);
let commit_root = format!("{root}{COMMIT_LOG_ROOT}/");
let index_root = format!("{root}{COMMIT_IDEMPOTENCY_ROOT}/");
for object in objects.iter().filter(|object| object.bucket == self.catalog_bucket()) {
if !seen_paths.insert(object.object.as_str()) {
return Err(TableCatalogStoreError::Invalid("catalog backup has a duplicate object path".to_string()));
}
if object.object.ends_with(TABLE_ENTRY_FILE) {
let entry: TableEntry = serde_json::from_slice(&object.data)
.map_err(|_| TableCatalogStoreError::Invalid("invalid table owner in backup commit state".to_string()))?;
validate_table_entry_object(&self.paths, &object.object, &entry)?;
if entry.table_bucket != table_bucket {
return Err(TableCatalogStoreError::Invalid(
"backup commit owner belongs to another bucket".to_string(),
));
}
if matches!(entry.state, TableCatalogEntryState::Renaming | TableCatalogEntryState::Deleting) {
return Err(backup_commit_recovery_required("table owner has a pending operation"));
}
if entry.state == TableCatalogEntryState::Active && active.insert(entry.table_id.clone(), entry).is_some() {
return Err(backup_commit_recovery_required("more than one active catalog pointer owns a table id"));
}
} else if object.object.starts_with(&commit_root) {
let commit: CommitLogEntry = serde_json::from_slice(&object.data)
.map_err(|_| TableCatalogStoreError::Invalid("invalid commit log in catalog backup".to_string()))?;
validate_commit_log_entry_object(&self.paths, &object.object, table_bucket, &commit.table_id, &commit)?;
if logs
.entry(commit.table_id.clone())
.or_default()
.insert(commit.commit_id.clone(), commit)
.is_some()
{
return Err(TableCatalogStoreError::Invalid("duplicate backup commit lookup identity".to_string()));
}
} else if object.object.starts_with(&index_root) {
let commit: CommitLogEntry = serde_json::from_slice(&object.data)
.map_err(|_| TableCatalogStoreError::Invalid("invalid idempotency index in catalog backup".to_string()))?;
validate_commit_idempotency_entry_object(&self.paths, &object.object, table_bucket, &commit.table_id, &commit)?;
let key = commit
.idempotency_key
.as_ref()
.ok_or_else(|| TableCatalogStoreError::Invalid("backup idempotency index has no lookup key".to_string()))?;
if indexes
.entry(commit.table_id.clone())
.or_default()
.insert(key.clone(), commit)
.is_some()
{
return Err(TableCatalogStoreError::Invalid(
"duplicate backup idempotency lookup identity".to_string(),
));
}
}
}
for (table_id, entry) in &active {
let Some(commits) = logs.get(table_id) else {
continue;
};
// Commit records are keyed by immutable table id, not its logical
// name. A rename source tombstone is not a second current pointer.
let history = TableCommitHistoryIndex::new(entry, commits.values());
for commit in commits.values() {
let indexed = commit
.idempotency_key
.as_ref()
.and_then(|key| indexes.get(table_id).and_then(|records| records.get(key)));
let recovery = table_commit_recovery_entry_from_history(entry, commit, indexed, &history);
if !matches!(
recovery.recovery_state,
TableCommitRecoveryState::Committed | TableCommitRecoveryState::Superseded
) {
return Err(TableCatalogStoreError::Conflict(format!(
"commit recovery required before backup or restore: {}/{}/{} commit {} is {:?}: {}",
entry.table_bucket,
entry.namespace,
entry.table,
recovery.commit_id,
recovery.recovery_state,
recovery.reason,
)));
}
}
}
// Check both directions. Looking up indexes only while iterating logs
// would miss an index whose commit record disappeared. Dropped-table
// history can remain in object backing; retain valid paired records
// without inventing a current pointer or comparing a rename tombstone.
for (table_id, commits) in &logs {
for commit in commits.values() {
let Some(key) = commit.idempotency_key.as_ref() else {
continue;
};
if indexes.get(table_id).and_then(|records| records.get(key)) != Some(commit) {
return Err(backup_commit_recovery_required("commit log has no identical idempotency receipt"));
}
}
}
for (table_id, records) in &indexes {
for indexed in records.values() {
if logs.get(table_id).and_then(|commits| commits.get(&indexed.commit_id)) != Some(indexed) {
return Err(backup_commit_recovery_required("idempotency receipt has no identical commit log"));
}
}
}
Ok(())
}
}
fn backup_commit_recovery_required(reason: &str) -> TableCatalogStoreError {
TableCatalogStoreError::Conflict(format!(
"commit recovery required before backup or restore: {reason}; inspect the persisted commit and idempotency records, recover the existing request, then capture a stable backup"
))
}
#[cfg(test)]
#[path = "object_backup_commit_recovery_tests.rs"]
mod tests;
@@ -1,384 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::table_catalog::test_support::TestCatalogObjectBackend;
fn fixture() -> (
ObjectTableCatalogStore<TestCatalogObjectBackend>,
Vec<ObjectCatalogBackupObject>,
TableEntry,
) {
let backup: ObjectCatalogBackup =
serde_json::from_slice(include_bytes!("../tests/fixtures/object-catalog-backup-v1.json")).expect("existing v1 fixture");
let entry = serde_json::from_slice(
&backup
.objects
.iter()
.find(|object| object.object.ends_with(TABLE_ENTRY_FILE))
.unwrap()
.data,
)
.expect("fixture table owner");
let store = ObjectTableCatalogStore::new(TestCatalogObjectBackend::default());
store
.validate_catalog_backup_metadata_identities("analytics", &backup.objects)
.expect("real current metadata is valid before testing recovery policy");
(store, backup.objects, entry)
}
fn object(path: String, data: Vec<u8>) -> ObjectCatalogBackupObject {
ObjectCatalogBackupObject {
bucket: RUSTFS_META_BUCKET.to_string(),
object: path,
size_bytes: u64::try_from(data.len()).unwrap(),
etag: Some("fixture-etag".to_string()),
sha256: backup_sha256(&data),
data,
}
}
fn current_commit(entry: &TableEntry) -> CommitLogEntry {
CommitLogEntry {
version: TABLE_CATALOG_ENTRY_VERSION,
commit_id: "current-commit".to_string(),
idempotency_key: Some("current-request".to_string()),
table_id: entry.table_id.clone(),
operation: "append".to_string(),
expected_version_token: "base-token".to_string(),
new_version_token: entry.version_token.clone(),
previous_metadata_location: "tables/table-id/metadata/00000.metadata.json".to_string(),
new_metadata_location: entry.metadata_location.clone(),
requirements: Vec::new(),
status: CommitLogStatus::Committed,
writer: Some("fixture-writer".to_string()),
created_at: Some("2026-10-31T00:00:00Z".to_string()),
updated_at: Some("2026-10-31T00:00:01Z".to_string()),
}
}
fn log_path(commit: &CommitLogEntry) -> String {
TableCatalogObjectPaths::default().commit_log_entry_path("analytics", &commit.table_id, &commit.commit_id)
}
fn index_path(commit: &CommitLogEntry) -> String {
TableCatalogObjectPaths::default().commit_idempotency_entry_path(
"analytics",
&commit.table_id,
commit.idempotency_key.as_deref().expect("fixture index key"),
)
}
fn put_record(objects: &mut Vec<ObjectCatalogBackupObject>, path: String, commit: &CommitLogEntry) {
objects.retain(|object| object.object != path);
objects.push(object(path, serde_json::to_vec(commit).unwrap()));
}
fn publish(objects: &mut Vec<ObjectCatalogBackupObject>, commit: &CommitLogEntry) {
put_record(objects, log_path(commit), commit);
if commit.idempotency_key.is_some() {
put_record(objects, index_path(commit), commit);
}
}
fn assert_recovery(result: TableCatalogStoreResult<()>, state: &str) {
let error = result.expect_err("unstable commit state must be rejected");
assert!(
matches!(&error, TableCatalogStoreError::Conflict(message)
if message.contains("commit recovery required") && message.contains(state)),
"{error}"
);
}
#[test]
fn backup_commit_policy_accepts_the_original_v1_snapshot_with_valid_metadata() {
let (store, objects, _) = fixture();
let before = objects.clone();
store.validate_backup_commit_recovery("analytics", &objects).unwrap();
assert_eq!(objects, before);
}
#[test]
fn backup_commit_policy_accepts_finalized_current_commits_with_or_without_idempotency() {
for idempotent in [false, true] {
let (store, mut objects, entry) = fixture();
let mut commit = current_commit(&entry);
if !idempotent {
commit.idempotency_key = None;
}
publish(&mut objects, &commit);
let before = objects.clone();
store.validate_backup_commit_recovery("analytics", &objects).unwrap();
assert_eq!(objects, before);
}
}
#[test]
fn backup_commit_policy_rejects_post_cas_finalization_without_mutating_source_bytes() {
let (store, mut objects, entry) = fixture();
let mut staged = current_commit(&entry);
staged.status = CommitLogStatus::Staged;
publish(&mut objects, &staged);
let before = objects.clone();
assert_recovery(store.validate_backup_commit_recovery("analytics", &objects), "FinalizationRequired");
assert_eq!(objects, before);
}
#[test]
fn backup_commit_policy_rejects_staging_before_the_table_pointer_advanced() {
let (store, mut objects, entry) = fixture();
let mut staged = current_commit(&entry);
staged.status = CommitLogStatus::Staged;
staged.expected_version_token = entry.version_token;
staged.previous_metadata_location = entry.metadata_location;
staged.new_version_token = "next-token".to_string();
staged.new_metadata_location = "tables/table-id/metadata/00002.metadata.json".to_string();
publish(&mut objects, &staged);
assert_recovery(store.validate_backup_commit_recovery("analytics", &objects), "StagedBeforeTableUpdate");
}
#[test]
fn backup_commit_policy_rejects_missing_and_stale_idempotency_receipts() {
for missing in [false, true] {
let (store, mut objects, entry) = fixture();
let commit = current_commit(&entry);
publish(&mut objects, &commit);
if missing {
objects.retain(|object| object.object != index_path(&commit));
} else {
let mut stale = commit.clone();
stale.status = CommitLogStatus::Staged;
put_record(&mut objects, index_path(&commit), &stale);
}
assert_recovery(
store.validate_backup_commit_recovery("analytics", &objects),
"IdempotencyIndexRepairRequired",
);
}
}
#[test]
fn backup_commit_policy_keeps_manual_review_failures_distinct() {
for failure in ["conflicting-index", "failed-log", "unreachable-log"] {
let (store, mut objects, entry) = fixture();
let mut commit = current_commit(&entry);
if failure == "failed-log" {
commit.status = CommitLogStatus::Failed;
} else if failure == "unreachable-log" {
commit.new_version_token = "unreachable-token".to_string();
commit.new_metadata_location = "tables/table-id/metadata/00099.metadata.json".to_string();
}
publish(&mut objects, &commit);
if failure == "conflicting-index" {
let mut conflicting = commit.clone();
conflicting.operation = "overwrite".to_string();
put_record(&mut objects, index_path(&commit), &conflicting);
}
assert_recovery(store.validate_backup_commit_recovery("analytics", &objects), "ManualReview");
}
}
#[test]
fn backup_commit_policy_uses_the_existing_history_proof_for_older_staged_commits() {
for older_status in [CommitLogStatus::Committed, CommitLogStatus::Staged] {
let (store, mut objects, entry) = fixture();
let mut latest = current_commit(&entry);
latest.expected_version_token = "intermediate-token".to_string();
latest.previous_metadata_location = "tables/table-id/metadata/00002.metadata.json".to_string();
let mut older = current_commit(&entry);
older.commit_id = "historical-commit".to_string();
older.idempotency_key = Some("historical-request".to_string());
older.new_version_token = latest.expected_version_token.clone();
older.new_metadata_location = latest.previous_metadata_location.clone();
older.status = older_status.clone();
publish(&mut objects, &older);
publish(&mut objects, &latest);
let result = store.validate_backup_commit_recovery("analytics", &objects);
if older_status == CommitLogStatus::Committed {
result.expect("healthy history retains earlier finalized receipts");
} else {
assert_recovery(result, "FinalizationRequired");
}
}
}
#[test]
fn backup_commit_policy_detects_an_idempotency_receipt_without_its_commit() {
let (store, mut objects, entry) = fixture();
let commit = current_commit(&entry);
publish(&mut objects, &commit);
objects.retain(|object| object.object != log_path(&commit));
assert_recovery(
store.validate_backup_commit_recovery("analytics", &objects),
"idempotency receipt has no identical commit log",
);
}
#[test]
fn backup_commit_policy_reuses_owner_and_lookup_path_validation() {
for changed in ["table-id", "commit-id", "idempotency-key", "version"] {
let (store, mut objects, entry) = fixture();
let commit = current_commit(&entry);
publish(&mut objects, &commit);
let mut corrupt = commit.clone();
let path = match changed {
"table-id" => {
corrupt.table_id = "another-table-id".to_string();
log_path(&commit)
}
"commit-id" => {
corrupt.commit_id = "another-commit".to_string();
log_path(&commit)
}
"idempotency-key" => {
corrupt.idempotency_key = Some("another-request".to_string());
index_path(&commit)
}
"version" => {
corrupt.version = 0;
log_path(&commit)
}
_ => unreachable!(),
};
put_record(&mut objects, path, &corrupt);
assert!(matches!(
store.validate_backup_commit_recovery("analytics", &objects),
Err(TableCatalogStoreError::Invalid(_))
));
}
}
#[test]
fn backup_commit_policy_uses_the_renamed_active_pointer_instead_of_the_source_tombstone() {
let (store, mut objects, mut destination) = fixture();
objects.retain(|object| !object.object.ends_with(TABLE_ENTRY_FILE));
let mut source = destination.clone();
source.state = TableCatalogEntryState::Deleted;
source.version_token = "old-token".to_string();
source.metadata_location = "tables/table-id/metadata/00000.metadata.json".to_string();
source.generation = source.generation.saturating_sub(1);
destination.table = "renamed_orders".to_string();
for entry in [&source, &destination] {
let path = TableCatalogObjectPaths::default().table_entry_path(
"analytics",
&Namespace::parse(&entry.namespace).unwrap(),
&IdentifierSegment::parse(&entry.table).unwrap(),
);
objects.push(object(path, serde_json::to_vec(entry).unwrap()));
}
store
.validate_catalog_backup_metadata_identities("analytics", &objects)
.expect("renamed active pointer still uses the original valid metadata");
publish(&mut objects, &current_commit(&destination));
let before = objects.clone();
store
.validate_backup_commit_recovery("analytics", &objects)
.expect("source tombstone is not current commit authority");
assert_eq!(objects, before);
}
#[test]
fn backup_commit_policy_rejects_two_active_pointers_for_one_table_id() {
let (store, mut objects, mut duplicate) = fixture();
duplicate.table = "duplicate_owner".to_string();
let path = TableCatalogObjectPaths::default().table_entry_path(
"analytics",
&Namespace::parse(&duplicate.namespace).unwrap(),
&IdentifierSegment::parse(&duplicate.table).unwrap(),
);
objects.push(object(path, serde_json::to_vec(&duplicate).unwrap()));
assert_recovery(
store.validate_backup_commit_recovery("analytics", &objects),
"more than one active catalog pointer",
);
}
#[test]
fn backup_commit_policy_preserves_paired_history_left_after_an_object_table_drop() {
let (store, mut objects, entry) = fixture();
publish(&mut objects, &current_commit(&entry));
objects.retain(|object| !object.object.ends_with(TABLE_ENTRY_FILE) && object.bucket == RUSTFS_META_BUCKET);
let before = objects.clone();
store
.validate_backup_commit_recovery("analytics", &objects)
.expect("normal object drop retains paired history without a current owner");
assert_eq!(objects, before);
}
#[test]
fn backup_commit_policy_checks_both_receipt_directions_even_without_an_active_table() {
for keep_log in [false, true] {
let (store, mut objects, entry) = fixture();
let commit = current_commit(&entry);
publish(&mut objects, &commit);
objects.retain(|object| !object.object.ends_with(TABLE_ENTRY_FILE) && object.bucket == RUSTFS_META_BUCKET);
let missing = if keep_log { index_path(&commit) } else { log_path(&commit) };
objects.retain(|object| object.object != missing);
assert_recovery(store.validate_backup_commit_recovery("analytics", &objects), "receipt");
}
}
#[test]
fn backup_commit_policy_preserves_proven_superseded_receipts_without_rewriting_them() {
for idempotent in [false, true] {
let (store, mut objects, entry) = fixture();
let winner = current_commit(&entry);
let loser = CommitLogEntry {
commit_id: "superseded-commit".to_string(),
idempotency_key: idempotent.then(|| "superseded-request".to_string()),
new_version_token: "superseded-token".to_string(),
new_metadata_location: "tables/table-id/metadata/loser.metadata.json".to_string(),
status: CommitLogStatus::Staged,
..winner.clone()
};
publish(&mut objects, &winner);
publish(&mut objects, &loser);
let before = objects.clone();
store
.validate_backup_commit_recovery("analytics", &objects)
.expect("a proven losing request is not pending recovery");
assert_eq!(objects, before, "sealing must retain the original STAGED audit bytes");
}
}
#[test]
fn backup_commit_policy_never_accepts_a_loser_without_intact_winner_and_index_proof() {
for failure in ["missing-winner", "wrong-base", "conflicting-index"] {
let (store, mut objects, entry) = fixture();
let winner = current_commit(&entry);
let mut loser = CommitLogEntry {
commit_id: "superseded-commit".to_string(),
idempotency_key: Some("superseded-request".to_string()),
new_version_token: "superseded-token".to_string(),
new_metadata_location: "tables/table-id/metadata/loser.metadata.json".to_string(),
status: CommitLogStatus::Staged,
..winner.clone()
};
if failure == "wrong-base" {
loser.expected_version_token = "unknown-base-token".to_string();
}
publish(&mut objects, &winner);
publish(&mut objects, &loser);
if failure == "missing-winner" {
objects.retain(|object| object.object != log_path(&winner) && object.object != index_path(&winner));
} else if failure == "conflicting-index" {
let mut conflicting = loser.clone();
conflicting.operation = "overwrite".to_string();
put_record(&mut objects, index_path(&loser), &conflicting);
}
let before = objects.clone();
assert_recovery(store.validate_backup_commit_recovery("analytics", &objects), "ManualReview");
assert_eq!(objects, before);
}
}
@@ -1,526 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
const OBJECT_CATALOG_BACKUP_REFERENCE_MAX_OBJECTS: usize = 1_000_000;
const OBJECT_CATALOG_BACKUP_REFERENCE_HASH_MAX_SIZE: u64 = 16 * 1024 * 1024;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) struct ObjectCatalogBackupReference {
pub(super) bucket: String,
pub(super) object: String,
pub(super) kind: ObjectCatalogBackupReferenceKind,
pub(super) size_bytes: u64,
pub(super) etag: Option<String>,
pub(super) sha256: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub(super) enum ObjectCatalogBackupReferenceKind {
TableMetadata,
ViewMetadata,
ManifestList,
ManifestFile,
DataFile,
DeleteFile,
TableStatistics,
PartitionStatistics,
}
impl ObjectCatalogBackupReferenceKind {
fn from_maintenance_kind(kind: &TableMetadataMaintenanceObjectKind) -> Self {
match kind {
TableMetadataMaintenanceObjectKind::MetadataFile => Self::TableMetadata,
TableMetadataMaintenanceObjectKind::ManifestList => Self::ManifestList,
TableMetadataMaintenanceObjectKind::ManifestFile => Self::ManifestFile,
TableMetadataMaintenanceObjectKind::DataFile => Self::DataFile,
TableMetadataMaintenanceObjectKind::DeleteFile => Self::DeleteFile,
}
}
fn from_statistics_kind(kind: IcebergStatisticsFileKind) -> Self {
match kind {
IcebergStatisticsFileKind::Table => Self::TableStatistics,
IcebergStatisticsFileKind::Partition => Self::PartitionStatistics,
}
}
}
pub(super) fn validate_object_catalog_backup_reference_shape(
table_bucket: &str,
references: &[ObjectCatalogBackupReference],
) -> TableCatalogStoreResult<()> {
if references.len() > OBJECT_CATALOG_BACKUP_REFERENCE_MAX_OBJECTS {
return Err(TableCatalogStoreError::Invalid(
"catalog backup reference count exceeds its limit".to_string(),
));
}
if references
.windows(2)
.any(|pair| (&pair[0].bucket, &pair[0].object) >= (&pair[1].bucket, &pair[1].object))
{
return Err(TableCatalogStoreError::Invalid(
"catalog backup references must be sorted and unique".to_string(),
));
}
for reference in references {
if reference.bucket != table_bucket || !safe_object_key(&reference.object) {
return Err(TableCatalogStoreError::Invalid(
"catalog backup reference is outside its table bucket".to_string(),
));
}
if reference.size_bytes <= OBJECT_CATALOG_BACKUP_REFERENCE_HASH_MAX_SIZE {
if reference.sha256.as_deref().is_none_or(|digest| {
digest.len() != 64
|| !digest
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
}) {
return Err(TableCatalogStoreError::Invalid(
"small catalog backup reference requires a SHA-256 watermark".to_string(),
));
}
} else if reference.sha256.is_some() || reference.etag.as_deref().is_none_or(str::is_empty) {
return Err(TableCatalogStoreError::Invalid(
"large catalog backup reference requires an ETag watermark and no inline digest".to_string(),
));
}
}
Ok(())
}
fn insert_backup_reference_kind(
keys: &mut BTreeMap<(String, String), ObjectCatalogBackupReferenceKind>,
bucket: &str,
object: &str,
kind: ObjectCatalogBackupReferenceKind,
) -> TableCatalogStoreResult<()> {
if !safe_object_key(object) {
return Err(TableCatalogStoreError::Invalid(
"catalog backup reference has an invalid object key".to_string(),
));
}
let key = (bucket.to_string(), object.to_string());
if let Some(existing) = keys.get(&key) {
if *existing != kind {
return Err(TableCatalogStoreError::Invalid(format!(
"catalog backup reference has conflicting kinds: {bucket}/{object}"
)));
}
return Ok(());
}
if keys.len() == OBJECT_CATALOG_BACKUP_REFERENCE_MAX_OBJECTS {
return Err(TableCatalogStoreError::Invalid(
"catalog backup reference count exceeds its limit".to_string(),
));
}
keys.insert(key, kind);
Ok(())
}
pub(super) fn backup_reference_metadata_log_locations(
metadata: &serde_json::Value,
entry: &TableEntry,
) -> TableCatalogStoreResult<BTreeSet<String>> {
let Some(metadata_log) = metadata.get("metadata-log") else {
return Ok(BTreeSet::new());
};
let entries = metadata_log
.as_array()
.ok_or_else(|| TableCatalogStoreError::Invalid("table metadata-log must be an array for a catalog backup".to_string()))?;
if entries.len() > OBJECT_CATALOG_BACKUP_REFERENCE_MAX_OBJECTS {
return Err(TableCatalogStoreError::Invalid(
"catalog backup metadata history exceeds its limit".to_string(),
));
}
let mut locations = BTreeSet::new();
for log_entry in entries {
let location = log_entry
.get("metadata-file")
.and_then(serde_json::Value::as_str)
.ok_or_else(|| {
TableCatalogStoreError::Invalid("catalog backup metadata-log entry requires a metadata-file".to_string())
})?;
let key = table_catalog_object_key_from_location(&entry.table_bucket, location).ok_or_else(|| {
TableCatalogStoreError::Invalid("catalog backup metadata history is outside its table bucket".to_string())
})?;
if !is_valid_table_metadata_location_for_entry(entry, &key) {
return Err(TableCatalogStoreError::Invalid(
"catalog backup metadata history is outside its metadata directory".to_string(),
));
}
locations.insert(key);
}
Ok(locations)
}
pub(super) fn validate_backup_reference_table_metadata(
entry: &TableEntry,
object: &str,
metadata: &serde_json::Value,
current: bool,
) -> TableCatalogStoreResult<()> {
validate_supported_table_metadata(metadata)?;
if table_metadata_uuid(metadata)? != entry.table_uuid {
return Err(TableCatalogStoreError::Invalid(
"catalog backup metadata UUID does not match its table".to_string(),
));
}
let warehouse_location = table_metadata_location(metadata)?;
validate_table_warehouse_location(&entry.table_bucket, warehouse_location)?;
if current && warehouse_location != entry.warehouse_location {
return Err(TableCatalogStoreError::Invalid(
"catalog backup current metadata does not match its warehouse".to_string(),
));
}
if current && table_metadata_format_version(metadata)? != entry.format_version {
return Err(TableCatalogStoreError::Invalid(
"catalog backup current metadata does not match its table format version".to_string(),
));
}
if !is_valid_table_metadata_location_for_entry(entry, object) {
return Err(TableCatalogStoreError::Invalid(
"catalog backup metadata is outside its metadata directory".to_string(),
));
}
Ok(())
}
pub(super) fn validate_backup_reference_view_metadata(
entry: &ViewEntry,
namespace: &Namespace,
view: &IdentifierSegment,
object: &str,
metadata: &serde_json::Value,
) -> TableCatalogStoreResult<()> {
validate_supported_view_metadata(metadata)?;
if metadata.get("view-uuid").and_then(serde_json::Value::as_str) != Some(entry.view_uuid.as_str())
|| metadata.get("location").and_then(serde_json::Value::as_str) != Some(entry.warehouse_location.as_str())
|| metadata.get("format-version").and_then(serde_json::Value::as_u64) != Some(u64::from(entry.format_version))
{
return Err(TableCatalogStoreError::Invalid(
"catalog backup metadata does not match its view identity".to_string(),
));
}
validate_view_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?;
if !is_valid_view_metadata_location(namespace, view, object) {
return Err(TableCatalogStoreError::Invalid(
"catalog backup view metadata is outside its metadata directory".to_string(),
));
}
Ok(())
}
impl<B> ObjectTableCatalogStore<B>
where
B: TableCatalogObjectBackend,
{
pub(super) fn backup_reference_view_entries(
&self,
table_bucket: &str,
objects: &[ObjectCatalogBackupObject],
) -> TableCatalogStoreResult<Vec<ViewEntry>> {
if objects.is_empty() || objects.len() > OBJECT_CATALOG_BACKUP_REFERENCE_MAX_OBJECTS {
return Err(TableCatalogStoreError::Invalid("catalog backup object count is invalid".to_string()));
}
let mut paths = BTreeSet::new();
let mut views = Vec::new();
for object in objects.iter().filter(|object| object.bucket == self.catalog_bucket()) {
if object.object.ends_with(TABLE_ENTRY_FILE) {
let entry: TableEntry = serde_json::from_slice(&object.data).map_err(|_| {
TableCatalogStoreError::Invalid("invalid table entry in catalog backup references".to_string())
})?;
validate_table_entry_object(&self.paths, &object.object, &entry)?;
if entry.table_bucket != table_bucket || !paths.insert(object.object.as_str()) {
return Err(TableCatalogStoreError::Invalid(
"catalog backup table identity is invalid or duplicated".to_string(),
));
}
if matches!(entry.state, TableCatalogEntryState::Renaming | TableCatalogEntryState::Deleting) {
return Err(TableCatalogStoreError::Conflict(
"catalog backup requires a stable table operation".to_string(),
));
}
} else if object.object.ends_with(VIEW_ENTRY_FILE) {
let entry: ViewEntry = serde_json::from_slice(&object.data).map_err(|_| {
TableCatalogStoreError::Invalid("invalid view entry in catalog backup references".to_string())
})?;
validate_view_entry_object(&self.paths, &object.object, &entry)?;
if entry.table_bucket != table_bucket || !paths.insert(object.object.as_str()) {
return Err(TableCatalogStoreError::Invalid(
"catalog backup view identity is invalid or duplicated".to_string(),
));
}
if matches!(entry.state, TableCatalogEntryState::Renaming | TableCatalogEntryState::Deleting) {
return Err(TableCatalogStoreError::Conflict(
"catalog backup requires a stable view operation".to_string(),
));
}
if entry.state == TableCatalogEntryState::Active {
views.push(entry);
}
}
}
Ok(views)
}
async fn collect_object_catalog_backup_reference_keys(
&self,
table_bucket: &str,
objects: &[ObjectCatalogBackupObject],
) -> TableCatalogStoreResult<BTreeMap<(String, String), ObjectCatalogBackupReferenceKind>> {
let views = self.backup_reference_view_entries(table_bucket, objects)?;
let tables = self.validate_catalog_backup_metadata_identities(table_bucket, objects)?;
let mut keys = BTreeMap::new();
for (entry, current_metadata) in &tables {
let current_object =
table_catalog_object_key_from_location(table_bucket, &entry.metadata_location).ok_or_else(|| {
TableCatalogStoreError::Invalid("catalog backup current metadata location is invalid".to_string())
})?;
validate_backup_reference_table_metadata(entry, &current_object, current_metadata, true)?;
let mut metadata_locations = backup_reference_metadata_log_locations(current_metadata, entry)?;
metadata_locations.insert(current_object.clone());
for object in metadata_locations {
insert_backup_reference_kind(&mut keys, table_bucket, &object, ObjectCatalogBackupReferenceKind::TableMetadata)?;
let historical;
let metadata = if object == current_object {
current_metadata
} else {
historical = read_table_metadata_value(&self.backend, table_bucket, &object)
.await?
.ok_or_else(|| TableCatalogStoreError::NotFound(format!("catalog backup metadata history {object}")))?;
&historical
};
validate_backup_reference_table_metadata(entry, &object, metadata, object == current_object)?;
let mut metadata_entry = entry.clone();
metadata_entry.warehouse_location = table_metadata_location(metadata)?.to_string();
let warehouse_prefix = table_warehouse_object_prefix(&metadata_entry)?;
for (other, _) in &tables {
if other.table_id != entry.table_id
&& warehouse_object_prefixes_overlap(&warehouse_prefix, &table_warehouse_object_prefix(other)?)
{
return Err(TableCatalogStoreError::Conflict(
"catalog backup metadata warehouse overlaps another active table".to_string(),
));
}
}
let context = TableSnapshotGraphValidationContext::new(&self.backend, table_bucket, &metadata_entry);
let graph = validated_table_snapshot_graph_inventory(&context, metadata).await?;
for reference in table_statistics_object_references(metadata)? {
let statistics_object = table_catalog_object_key_from_location(table_bucket, &reference.location)
.ok_or_else(|| {
TableCatalogStoreError::Invalid(
"catalog backup statistics are outside their table bucket".to_string(),
)
})?;
if !statistics_object.starts_with(&warehouse_prefix) {
return Err(TableCatalogStoreError::Invalid(
"catalog backup statistics are outside their table warehouse".to_string(),
));
}
insert_backup_reference_kind(
&mut keys,
table_bucket,
&statistics_object,
ObjectCatalogBackupReferenceKind::from_statistics_kind(reference.kind),
)?;
}
for (graph_object, kind) in graph {
insert_backup_reference_kind(
&mut keys,
table_bucket,
&graph_object,
ObjectCatalogBackupReferenceKind::from_maintenance_kind(&kind),
)?;
}
}
}
for entry in views {
let namespace = parse_namespace_for_store(&entry.namespace)?;
let view = parse_table_for_store(&entry.view)?;
let object = table_catalog_object_key_from_location(table_bucket, &entry.metadata_location)
.ok_or_else(|| TableCatalogStoreError::Invalid("catalog backup view metadata location is invalid".to_string()))?;
let metadata = read_table_metadata_value(&self.backend, table_bucket, &object)
.await?
.ok_or_else(|| TableCatalogStoreError::NotFound(format!("catalog backup view metadata {object}")))?;
validate_backup_reference_view_metadata(&entry, &namespace, &view, &object, &metadata)?;
insert_backup_reference_kind(&mut keys, table_bucket, &object, ObjectCatalogBackupReferenceKind::ViewMetadata)?;
}
Ok(keys)
}
async fn object_catalog_backup_reference_fingerprint(
&self,
bucket: &str,
object: &str,
kind: ObjectCatalogBackupReferenceKind,
) -> TableCatalogStoreResult<ObjectCatalogBackupReference> {
let metadata = self
.backend
.object_metadata(bucket, object)
.await?
.ok_or_else(|| TableCatalogStoreError::NotFound(format!("catalog backup reference {bucket}/{object}")))?;
let etag = metadata.etag.as_deref().filter(|etag| !etag.is_empty()).ok_or_else(|| {
TableCatalogStoreError::Invalid(format!("catalog backup reference has no stable ETag: {bucket}/{object}"))
})?;
let sha256 = if metadata.size <= OBJECT_CATALOG_BACKUP_REFERENCE_HASH_MAX_SIZE {
let max_size = usize::try_from(OBJECT_CATALOG_BACKUP_REFERENCE_HASH_MAX_SIZE).map_err(|_| {
TableCatalogStoreError::Internal("catalog backup hash limit exceeds addressable size".to_string())
})?;
let stored = self
.backend
.read_object_limited(bucket, object, max_size)
.await?
.ok_or_else(|| {
TableCatalogStoreError::Conflict(format!("catalog backup reference disappeared: {bucket}/{object}"))
})?;
if u64::try_from(stored.data.len()).ok() != Some(metadata.size) || stored.etag.as_deref() != Some(etag) {
return Err(TableCatalogStoreError::Conflict(format!(
"catalog backup reference changed during its read: {bucket}/{object}"
)));
}
Some(backup_sha256(&stored.data))
} else {
None
};
let final_metadata = self.backend.object_metadata(bucket, object).await?.ok_or_else(|| {
TableCatalogStoreError::Conflict(format!("catalog backup reference disappeared: {bucket}/{object}"))
})?;
if final_metadata.size != metadata.size || final_metadata.etag != metadata.etag {
return Err(TableCatalogStoreError::Conflict(format!(
"catalog backup reference changed during verification: {bucket}/{object}"
)));
}
Ok(ObjectCatalogBackupReference {
bucket: bucket.to_string(),
object: object.to_string(),
kind,
size_bytes: metadata.size,
etag: metadata.etag,
sha256,
})
}
/// Captures the finite object inventory required by one stable catalog payload.
///
/// The caller retains its publication and restore fences throughout capture. The inventory includes current
/// metadata, its direct metadata-log history, each corresponding snapshot graph and statistics, and current view
/// metadata. It contains watermarks, not payload copies. Cancellation makes no storage changes.
pub(super) async fn capture_object_catalog_backup_references(
&self,
table_bucket: &str,
objects: &[ObjectCatalogBackupObject],
) -> TableCatalogStoreResult<Vec<ObjectCatalogBackupReference>> {
let keys = self
.collect_object_catalog_backup_reference_keys(table_bucket, objects)
.await?;
let mut references = Vec::with_capacity(keys.len());
for ((bucket, object), kind) in keys {
references.push(
self.object_catalog_backup_reference_fingerprint(&bucket, &object, kind)
.await?,
);
}
// Current metadata embedded in the catalog payload must be exactly the object whose graph was inventoried.
for object in objects.iter().filter(|object| object.bucket == table_bucket) {
let index = references
.binary_search_by(|reference| {
reference
.bucket
.cmp(&object.bucket)
.then_with(|| reference.object.cmp(&object.object))
})
.map_err(|_| TableCatalogStoreError::Invalid("catalog backup embeds an unreferenced table object".to_string()))?;
let reference = &references[index];
if u64::try_from(object.data.len()).ok() != Some(reference.size_bytes)
|| reference.sha256.as_deref() != Some(backup_sha256(&object.data).as_str())
{
return Err(TableCatalogStoreError::Conflict(
"catalog backup embedded metadata differs from the prepared object".to_string(),
));
}
}
Ok(references)
}
/// Verifies the closed inventory without publishing any catalog state.
///
/// The caller retains its restore/publication fences. Small objects compare size and SHA-256 across targets; their
/// individual reads still require stable target ETags. An absent current metadata object may be supplied by its
/// validated inline restore payload; existing different bytes are never accepted. All external objects must
/// already be prepared. Large objects retain the recorded size and ETag. Invalid references fail closed.
pub(super) async fn validate_object_catalog_backup_references(
&self,
table_bucket: &str,
objects: &[ObjectCatalogBackupObject],
references: &[ObjectCatalogBackupReference],
) -> TableCatalogStoreResult<()> {
validate_object_catalog_backup_reference_shape(table_bucket, references)?;
let keys = self
.collect_object_catalog_backup_reference_keys(table_bucket, objects)
.await?;
if keys.len() != references.len() {
return Err(TableCatalogStoreError::Conflict(
"catalog backup reference graph does not match the prepared target".to_string(),
));
}
let embedded = objects
.iter()
.filter(|object| object.bucket == table_bucket)
.map(|object| ((object.bucket.as_str(), object.object.as_str()), object))
.collect::<BTreeMap<_, _>>();
for (expected, ((bucket, object), kind)) in references.iter().zip(&keys) {
if expected.bucket != *bucket || expected.object != *object || expected.kind != *kind {
return Err(TableCatalogStoreError::Conflict(
"catalog backup reference identity does not match its graph".to_string(),
));
}
if let Some(payload) = embedded.get(&(bucket.as_str(), object.as_str()))
&& self.backend.object_metadata(bucket, object).await?.is_none()
{
if u64::try_from(payload.data.len()).ok() != Some(expected.size_bytes)
|| expected.sha256.as_deref() != Some(backup_sha256(&payload.data).as_str())
{
return Err(TableCatalogStoreError::Conflict(
"inline restore metadata does not match its saved inventory".to_string(),
));
}
continue;
}
let actual = self
.object_catalog_backup_reference_fingerprint(bucket, object, *kind)
.await?;
if expected.size_bytes != actual.size_bytes
|| if expected.size_bytes <= OBJECT_CATALOG_BACKUP_REFERENCE_HASH_MAX_SIZE {
expected.sha256 != actual.sha256
} else {
expected.etag != actual.etag
}
{
return Err(TableCatalogStoreError::Conflict(format!(
"catalog backup reference watermark does not match the prepared target: {}/{}",
expected.bucket, expected.object
)));
}
}
Ok(())
}
}
#[cfg(test)]
#[path = "object_backup_inventory_tests.rs"]
mod tests;
@@ -1,721 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::table_catalog::test_support::{
TestCatalogObjectBackend, manifest_avro_bytes, manifest_list_avro_entries_with_content, table_metadata_json,
};
const BUCKET: &str = "analytics";
const DATA: &str = "tables/table-id/data/part.parquet";
const DATA_MANIFEST: &str = "tables/table-id/metadata/data.avro";
const MANIFEST_LIST: &str = "tables/table-id/metadata/snap-42.avro";
const TABLE_STATISTICS: &str = "tables/table-id/metadata/statistics.puffin";
const PARTITION_STATISTICS: &str = "tables/table-id/metadata/partition-statistics.orc";
const HISTORY_MANIFEST_LIST: &str = "tables/table-id/metadata/snap-20.avro";
const HISTORY_MANIFEST: &str = "tables/table-id/metadata/history.avro";
const HISTORY_DATA: &str = "tables/table-id/data/history.parquet";
const HISTORY_STATISTICS: &str = "tables/table-id/metadata/history-statistics.puffin";
struct InventoryFixture {
backend: TestCatalogObjectBackend,
objects: Vec<ObjectCatalogBackupObject>,
current: String,
history: String,
unretained_history: String,
view: String,
}
async fn embedded_object(backend: &TestCatalogObjectBackend, bucket: &str, object: &str) -> ObjectCatalogBackupObject {
let stored = backend.read_object(bucket, object).await.unwrap().unwrap();
ObjectCatalogBackupObject {
bucket: bucket.to_string(),
object: object.to_string(),
size_bytes: u64::try_from(stored.data.len()).unwrap(),
etag: stored.etag,
sha256: backup_sha256(&stored.data),
data: stored.data,
}
}
async fn inventory_fixture() -> InventoryFixture {
let backend = TestCatalogObjectBackend::default();
let paths = TableCatalogObjectPaths::default();
let namespace = Namespace::parse("sales").unwrap();
let table = IdentifierSegment::parse("orders").unwrap();
let view = IdentifierSegment::parse("summary").unwrap();
let current = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json");
let history = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json");
let unretained_history = default_table_metadata_file_path(&namespace, &table, "00000.metadata.json");
let view_metadata = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json");
let entry = TableEntry {
version: TABLE_CATALOG_ENTRY_VERSION,
table_bucket: BUCKET.to_string(),
namespace: namespace.public_name(),
table: table.as_str().to_string(),
table_id: "table-id".to_string(),
table_uuid: "table-uuid".to_string(),
format: "ICEBERG".to_string(),
format_version: 2,
warehouse_location: format!("s3://{BUCKET}/tables/table-id"),
metadata_location: current.clone(),
version_token: "token-v1".to_string(),
generation: 1,
state: TableCatalogEntryState::Active,
properties: BTreeMap::new(),
created_at: None,
updated_at: None,
};
let view_entry = ViewEntry {
version: TABLE_CATALOG_ENTRY_VERSION,
table_bucket: BUCKET.to_string(),
namespace: namespace.public_name(),
view: view.as_str().to_string(),
view_id: "view-id".to_string(),
view_uuid: "view-uuid".to_string(),
format: "ICEBERG_VIEW".to_string(),
format_version: 1,
warehouse_location: format!("s3://{BUCKET}/views/view-id"),
metadata_location: view_metadata.clone(),
version_token: "view-token-v1".to_string(),
generation: 1,
state: TableCatalogEntryState::Active,
properties: BTreeMap::new(),
created_at: None,
updated_at: None,
};
let mut current_metadata = table_metadata_json(&entry.table_uuid, &entry.warehouse_location);
current_metadata["last-sequence-number"] = serde_json::json!(7);
current_metadata["last-updated-ms"] = serde_json::json!(2);
current_metadata["snapshots"] = serde_json::json!([{
"snapshot-id": 42,
"sequence-number": 7,
"timestamp-ms": 2,
"manifest-list": format!("s3://{BUCKET}/{MANIFEST_LIST}"),
"summary": {"operation": "append"}
}]);
current_metadata["current-snapshot-id"] = serde_json::json!(42);
current_metadata["snapshot-log"] = serde_json::json!([{"timestamp-ms": 2, "snapshot-id": 42}]);
current_metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 42}});
current_metadata["metadata-log"] = serde_json::json!([{
"timestamp-ms": 1,
"metadata-file": format!("s3://{BUCKET}/{history}")
}]);
current_metadata["statistics"] = serde_json::json!([{
"snapshot-id": 42,
"statistics-path": format!("s3://{BUCKET}/{TABLE_STATISTICS}"),
"file-size-in-bytes": 8,
"file-footer-size-in-bytes": 4,
"blob-metadata": []
}]);
current_metadata["partition-statistics"] = serde_json::json!([{
"snapshot-id": 42,
"statistics-path": format!("s3://{BUCKET}/{PARTITION_STATISTICS}"),
"file-size-in-bytes": 3
}]);
let mut historical_metadata = table_metadata_json(&entry.table_uuid, &entry.warehouse_location);
historical_metadata["last-sequence-number"] = serde_json::json!(6);
historical_metadata["snapshots"] = serde_json::json!([{
"snapshot-id": 20, "sequence-number": 6, "timestamp-ms": 1,
"manifest-list": format!("s3://{BUCKET}/{HISTORY_MANIFEST_LIST}"),
"summary": {"operation": "append"}
}]);
historical_metadata["current-snapshot-id"] = serde_json::json!(20);
historical_metadata["snapshot-log"] = serde_json::json!([{"timestamp-ms": 1, "snapshot-id": 20}]);
historical_metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 20}});
historical_metadata["statistics"] = serde_json::json!([{
"snapshot-id": 20,
"statistics-path": format!("s3://{BUCKET}/{HISTORY_STATISTICS}"),
"file-size-in-bytes": 8, "file-footer-size-in-bytes": 4, "blob-metadata": []
}]);
historical_metadata["metadata-log"] = serde_json::json!([{
"timestamp-ms": 0,
"metadata-file": format!("s3://{BUCKET}/{unretained_history}")
}]);
let view_metadata_json = serde_json::json!({
"format-version": 1,
"view-uuid": view_entry.view_uuid,
"location": view_entry.warehouse_location,
"schemas": [{
"type": "struct", "schema-id": 0,
"fields": [{"id": 1, "name": "id", "required": true, "type": "long"}]
}],
"current-schema-id": 0,
"current-version-id": 1,
"versions": [{
"version-id": 1, "timestamp-ms": 1, "schema-id": 0, "summary": {},
"representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}],
"default-namespace": ["sales"]
}],
"version-log": [{"timestamp-ms": 1, "version-id": 1}],
"properties": {}
});
let mut manifests = Vec::new();
for (manifest, object, content) in [
(DATA_MANIFEST, DATA, 0),
("tables/table-id/metadata/positions.avro", "tables/table-id/data/positions.parquet", 1),
("tables/table-id/metadata/equality.avro", "tables/table-id/data/equality.parquet", 2),
] {
let location = format!("s3://{BUCKET}/{object}");
let bytes = manifest_avro_bytes(&[(&location, content, 1, 42, 7)]);
manifests.push((format!("s3://{BUCKET}/{manifest}"), bytes.len(), if content == 0 { 0 } else { 1 }));
backend.seed_object(BUCKET, manifest, bytes).await;
backend.seed_object(BUCKET, object, vec![1]).await;
}
let manifest_entries = manifests
.iter()
.map(|(path, size, content)| (path.as_str(), *size, 0, *content, 7, 42))
.collect::<Vec<_>>();
backend
.seed_object(BUCKET, MANIFEST_LIST, manifest_list_avro_entries_with_content(&manifest_entries))
.await;
let history_data_location = format!("s3://{BUCKET}/{HISTORY_DATA}");
let history_manifest_bytes = manifest_avro_bytes(&[(&history_data_location, 0, 1, 20, 6)]);
let history_manifest_location = format!("s3://{BUCKET}/{HISTORY_MANIFEST}");
backend
.seed_object(
BUCKET,
HISTORY_MANIFEST_LIST,
manifest_list_avro_entries_with_content(&[(&history_manifest_location, history_manifest_bytes.len(), 0, 0, 6, 20)]),
)
.await;
backend.seed_object(BUCKET, HISTORY_MANIFEST, history_manifest_bytes).await;
backend.seed_object(BUCKET, HISTORY_DATA, vec![1]).await;
backend.seed_object(BUCKET, HISTORY_STATISTICS, b"PFA1PFA1".to_vec()).await;
backend.seed_object(BUCKET, TABLE_STATISTICS, b"PFA1PFA1".to_vec()).await;
backend.seed_object(BUCKET, PARTITION_STATISTICS, b"ORC".to_vec()).await;
backend
.seed_object(BUCKET, &current, serde_json::to_vec(&current_metadata).unwrap())
.await;
backend
.seed_object(BUCKET, &history, serde_json::to_vec(&historical_metadata).unwrap())
.await;
backend
.seed_object(BUCKET, &view_metadata, serde_json::to_vec(&view_metadata_json).unwrap())
.await;
let table_path = paths.table_entry_path(BUCKET, &namespace, &table);
let view_path = paths.view_entry_path(BUCKET, &namespace, &view);
backend
.seed_object(RUSTFS_META_BUCKET, &table_path, serde_json::to_vec(&entry).unwrap())
.await;
backend
.seed_object(RUSTFS_META_BUCKET, &view_path, serde_json::to_vec(&view_entry).unwrap())
.await;
let mut objects = vec![
embedded_object(&backend, RUSTFS_META_BUCKET, &table_path).await,
embedded_object(&backend, RUSTFS_META_BUCKET, &view_path).await,
embedded_object(&backend, BUCKET, &current).await,
];
objects.sort_by(|left, right| (&left.bucket, &left.object).cmp(&(&right.bucket, &right.object)));
InventoryFixture {
backend,
objects,
current,
history,
unretained_history,
view: view_metadata,
}
}
async fn copied_backend_with_new_etags(source: &TestCatalogObjectBackend) -> TestCatalogObjectBackend {
let objects = source
.state
.lock()
.await
.objects
.iter()
.map(|(key, value)| (key.clone(), value.data.clone()))
.collect::<Vec<_>>();
let target = TestCatalogObjectBackend::default();
for _ in 0..=objects.len() {
target.seed_object("unrelated", "etag-offset", vec![0]).await;
}
for ((bucket, object), bytes) in objects {
target.seed_object(&bucket, &object, bytes).await;
}
target
}
#[tokio::test]
async fn inventory_captures_all_reference_kinds_and_only_direct_metadata_history() {
let fixture = inventory_fixture().await;
let store = ObjectTableCatalogStore::new(fixture.backend.clone());
let references = store
.capture_object_catalog_backup_references(BUCKET, &fixture.objects)
.await
.unwrap();
validate_object_catalog_backup_reference_shape(BUCKET, &references).unwrap();
for kind in [
ObjectCatalogBackupReferenceKind::TableMetadata,
ObjectCatalogBackupReferenceKind::ViewMetadata,
ObjectCatalogBackupReferenceKind::ManifestList,
ObjectCatalogBackupReferenceKind::ManifestFile,
ObjectCatalogBackupReferenceKind::DataFile,
ObjectCatalogBackupReferenceKind::DeleteFile,
ObjectCatalogBackupReferenceKind::TableStatistics,
ObjectCatalogBackupReferenceKind::PartitionStatistics,
] {
assert!(references.iter().any(|reference| reference.kind == kind), "missing {kind:?}");
}
for object in [&fixture.current, &fixture.history, &fixture.view] {
assert_eq!(references.iter().filter(|reference| &reference.object == object).count(), 1);
}
for object in [HISTORY_MANIFEST_LIST, HISTORY_MANIFEST, HISTORY_DATA, HISTORY_STATISTICS] {
assert_eq!(references.iter().filter(|reference| reference.object == object).count(), 1);
}
assert!(
!references
.iter()
.any(|reference| reference.object == fixture.unretained_history)
);
assert_eq!(
references
.iter()
.filter(|reference| reference.kind == ObjectCatalogBackupReferenceKind::DeleteFile)
.count(),
2
);
assert!(references.iter().all(|reference| reference.sha256.is_some()));
}
#[tokio::test]
async fn prepared_small_objects_verify_by_bytes_across_different_etags_without_writes() {
let fixture = inventory_fixture().await;
let source = ObjectTableCatalogStore::new(fixture.backend.clone());
let references = source
.capture_object_catalog_backup_references(BUCKET, &fixture.objects)
.await
.unwrap();
let target_backend = copied_backend_with_new_etags(&fixture.backend).await;
let target = ObjectTableCatalogStore::new(target_backend.clone());
let observed = target
.capture_object_catalog_backup_references(BUCKET, &fixture.objects)
.await
.unwrap();
assert!(
references
.iter()
.zip(&observed)
.all(|(source, target)| source.etag != target.etag)
);
target_backend.reset_call_counts().await;
target
.validate_object_catalog_backup_references(BUCKET, &fixture.objects, &references)
.await
.unwrap();
let state = target_backend.state.lock().await;
assert!(state.put_attempts.is_empty());
assert!(state.delete_attempts.is_empty());
}
#[tokio::test]
async fn deleted_only_missing_files_are_excluded_but_a_retained_snapshot_still_requires_its_live_file() {
let mut fixture = inventory_fixture().await;
let missing_deleted = "tables/table-id/data/expired.parquet";
let shared_location = format!("s3://{BUCKET}/{DATA}");
let missing_location = format!("s3://{BUCKET}/{missing_deleted}");
let history_manifest = manifest_avro_bytes(&[(&shared_location, 0, 0, 20, 6)]);
let history_manifest_location = format!("s3://{BUCKET}/{HISTORY_MANIFEST}");
fixture
.backend
.seed_object(
BUCKET,
HISTORY_MANIFEST_LIST,
manifest_list_avro_entries_with_content(&[(&history_manifest_location, history_manifest.len(), 0, 0, 6, 20)]),
)
.await;
fixture.backend.seed_object(BUCKET, HISTORY_MANIFEST, history_manifest).await;
let deleted_manifest = manifest_avro_bytes(&[(&missing_location, 0, 2, 42, 7), (&shared_location, 0, 2, 42, 7)]);
fixture.backend.seed_object(BUCKET, DATA_MANIFEST, deleted_manifest).await;
let mut manifest_entries = Vec::new();
for (object, content) in [
(DATA_MANIFEST, 0),
("tables/table-id/metadata/positions.avro", 1),
("tables/table-id/metadata/equality.avro", 1),
] {
let size = fixture.backend.object_metadata(BUCKET, object).await.unwrap().unwrap().size;
manifest_entries.push((format!("s3://{BUCKET}/{object}"), usize::try_from(size).unwrap(), content));
}
let manifest_references = manifest_entries
.iter()
.map(|(path, size, content)| (path.as_str(), *size, 0, *content, 7, 42))
.collect::<Vec<_>>();
fixture
.backend
.seed_object(BUCKET, MANIFEST_LIST, manifest_list_avro_entries_with_content(&manifest_references))
.await;
let index = fixture
.objects
.iter()
.position(|object| object.bucket == BUCKET && object.object == fixture.current)
.unwrap();
let mut metadata: serde_json::Value = serde_json::from_slice(&fixture.objects[index].data).unwrap();
let historical = fixture.backend.read_object(BUCKET, &fixture.history).await.unwrap().unwrap();
let historical: serde_json::Value = serde_json::from_slice(&historical.data).unwrap();
metadata["snapshots"]
.as_array_mut()
.unwrap()
.insert(0, historical["snapshots"][0].clone());
metadata["snapshot-log"] = serde_json::json!([
{"timestamp-ms": 1, "snapshot-id": 20}, {"timestamp-ms": 2, "snapshot-id": 42}
]);
metadata["refs"]["retained"] = serde_json::json!({"type": "branch", "snapshot-id": 20});
fixture
.backend
.seed_object(BUCKET, &fixture.current, serde_json::to_vec(&metadata).unwrap())
.await;
fixture.objects[index] = embedded_object(&fixture.backend, BUCKET, &fixture.current).await;
let entry = fixture
.objects
.iter()
.find(|object| object.bucket == RUSTFS_META_BUCKET && object.object.ends_with(TABLE_ENTRY_FILE))
.unwrap();
let entry: TableEntry = serde_json::from_slice(&entry.data).unwrap();
let context = TableSnapshotGraphValidationContext::new(&fixture.backend, BUCKET, &entry);
fixture.backend.reset_call_counts().await;
let reads_before = fixture.backend.state.lock().await.read_attempts.clone();
let graph = validated_table_snapshot_graph_inventory(&context, &metadata).await.unwrap();
assert!(!graph.contains_key(missing_deleted));
assert_eq!(graph.get(DATA), Some(&TableMetadataMaintenanceObjectKind::DataFile));
assert_eq!(graph.get(DATA_MANIFEST), Some(&TableMetadataMaintenanceObjectKind::ManifestFile));
let state = fixture.backend.state.lock().await;
for object in [MANIFEST_LIST, DATA_MANIFEST, HISTORY_MANIFEST_LIST, HISTORY_MANIFEST] {
let key = (BUCKET.to_string(), object.to_string());
let expected = reads_before.get(&key).copied().unwrap_or_default() + 1;
assert_eq!(state.read_attempts.get(&key), Some(&expected), "repeated graph read: {object}");
}
drop(state);
let store = ObjectTableCatalogStore::new(fixture.backend.clone());
let references = store
.capture_object_catalog_backup_references(BUCKET, &fixture.objects)
.await
.unwrap();
assert!(!references.iter().any(|reference| reference.object == missing_deleted));
assert_eq!(references.iter().filter(|reference| reference.object == DATA).count(), 1);
fixture
.backend
.state
.lock()
.await
.objects
.remove(&(BUCKET.to_string(), DATA.to_string()));
assert!(matches!(
validated_table_snapshot_graph_inventory(&context, &metadata).await,
Err(TableCatalogStoreError::Invalid(message)) if message == "manifest referenced data file is missing"
));
assert!(
store
.capture_object_catalog_backup_references(BUCKET, &fixture.objects)
.await
.is_err()
);
}
#[tokio::test]
async fn changed_small_content_is_rejected_even_when_size_and_graph_are_unchanged() {
let fixture = inventory_fixture().await;
let source = ObjectTableCatalogStore::new(fixture.backend.clone());
let references = source
.capture_object_catalog_backup_references(BUCKET, &fixture.objects)
.await
.unwrap();
let target_backend = copied_backend_with_new_etags(&fixture.backend).await;
target_backend.seed_object(BUCKET, DATA, vec![9]).await;
let target = ObjectTableCatalogStore::new(target_backend.clone());
target_backend.reset_call_counts().await;
assert!(matches!(
target
.validate_object_catalog_backup_references(BUCKET, &fixture.objects, &references)
.await,
Err(TableCatalogStoreError::Conflict(_))
));
let state = target_backend.state.lock().await;
assert!(state.put_attempts.is_empty());
assert!(state.delete_attempts.is_empty());
}
#[tokio::test]
async fn absent_history_view_manifest_data_or_statistics_prevents_validation() {
let fixture = inventory_fixture().await;
let source = ObjectTableCatalogStore::new(fixture.backend.clone());
let references = source
.capture_object_catalog_backup_references(BUCKET, &fixture.objects)
.await
.unwrap();
for object in [
fixture.history.as_str(),
fixture.view.as_str(),
MANIFEST_LIST,
DATA_MANIFEST,
DATA,
TABLE_STATISTICS,
PARTITION_STATISTICS,
] {
let backend = copied_backend_with_new_etags(&fixture.backend).await;
backend
.state
.lock()
.await
.objects
.remove(&(BUCKET.to_string(), object.to_string()));
backend.reset_call_counts().await;
let store = ObjectTableCatalogStore::new(backend.clone());
assert!(
store
.validate_object_catalog_backup_references(BUCKET, &fixture.objects, &references)
.await
.is_err(),
"accepted missing {object}"
);
let state = backend.state.lock().await;
assert!(state.put_attempts.is_empty());
assert!(state.delete_attempts.is_empty());
}
}
#[tokio::test]
async fn absent_inline_current_metadata_remains_a_planned_immutable_restore_write() {
let fixture = inventory_fixture().await;
let source = ObjectTableCatalogStore::new(fixture.backend.clone());
let references = source
.capture_object_catalog_backup_references(BUCKET, &fixture.objects)
.await
.expect("source inventory");
let backend = copied_backend_with_new_etags(&fixture.backend).await;
backend
.state
.lock()
.await
.objects
.remove(&(BUCKET.to_string(), fixture.current.clone()));
backend.reset_call_counts().await;
let target = ObjectTableCatalogStore::new(backend.clone());
target
.validate_object_catalog_backup_references(BUCKET, &fixture.objects, &references)
.await
.expect("the inline immutable metadata can be published later by the fenced restore plan");
{
let state = backend.state.lock().await;
assert!(!state.objects.contains_key(&(BUCKET.to_string(), fixture.current.clone())));
assert!(state.put_attempts.is_empty());
assert!(state.delete_attempts.is_empty());
}
let mut changed = fixture
.objects
.iter()
.find(|object| object.bucket == BUCKET && object.object == fixture.current)
.expect("inline current metadata")
.data
.clone();
changed[0] ^= 1;
backend.seed_object(BUCKET, &fixture.current, changed).await;
backend.reset_call_counts().await;
assert!(matches!(
target
.validate_object_catalog_backup_references(BUCKET, &fixture.objects, &references)
.await,
Err(TableCatalogStoreError::Conflict(_))
));
let state = backend.state.lock().await;
assert!(state.put_attempts.is_empty());
assert!(state.delete_attempts.is_empty());
}
#[tokio::test]
async fn reference_shape_is_rejected_before_backend_access() {
let fixture = inventory_fixture().await;
let source = ObjectTableCatalogStore::new(fixture.backend.clone());
let references = source
.capture_object_catalog_backup_references(BUCKET, &fixture.objects)
.await
.unwrap();
let mut cases = Vec::new();
let mut duplicate = references.clone();
duplicate.insert(0, duplicate[0].clone());
cases.push(duplicate);
let mut unsorted = references.clone();
unsorted.reverse();
cases.push(unsorted);
for (foreign_bucket, invalid_key, missing_digest) in [(true, false, false), (false, true, false), (false, false, true)] {
let mut invalid = references.clone();
if foreign_bucket {
invalid[0].bucket = "foreign".to_string();
}
if invalid_key {
invalid[0].object = "../outside.parquet".to_string();
}
if missing_digest {
invalid[0].sha256 = None;
}
cases.push(invalid);
}
for invalid in cases {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
assert!(matches!(
store
.validate_object_catalog_backup_references(BUCKET, &fixture.objects, &invalid)
.await,
Err(TableCatalogStoreError::Invalid(_))
));
let state = backend.state.lock().await;
assert_eq!(state.read_calls, 0);
assert_eq!(state.metadata_calls, 0);
assert_eq!(state.list_calls, 0);
assert!(state.put_attempts.is_empty());
}
}
#[tokio::test]
async fn saved_inventory_cannot_omit_add_or_relabel_graph_objects() {
let fixture = inventory_fixture().await;
let source = ObjectTableCatalogStore::new(fixture.backend.clone());
let references = source
.capture_object_catalog_backup_references(BUCKET, &fixture.objects)
.await
.unwrap();
let mut omitted = references.clone();
omitted.pop();
let mut extra = references.clone();
let mut unrelated = extra[0].clone();
unrelated.object = "tables/table-id/data/unrelated.parquet".to_string();
extra.push(unrelated);
extra.sort_by(|left, right| (&left.bucket, &left.object).cmp(&(&right.bucket, &right.object)));
let mut relabeled = references.clone();
relabeled[0].kind = if relabeled[0].kind == ObjectCatalogBackupReferenceKind::DataFile {
ObjectCatalogBackupReferenceKind::DeleteFile
} else {
ObjectCatalogBackupReferenceKind::DataFile
};
for invalid in [omitted, extra, relabeled] {
assert!(matches!(
source
.validate_object_catalog_backup_references(BUCKET, &fixture.objects, &invalid)
.await,
Err(TableCatalogStoreError::Conflict(_))
));
}
}
#[tokio::test]
async fn capture_rejects_malformed_manifest_and_wrong_history_or_view_identity() {
for case in 0..3 {
let fixture = inventory_fixture().await;
let object = match case {
0 => DATA_MANIFEST,
1 => fixture.history.as_str(),
_ => fixture.view.as_str(),
};
let stored = fixture.backend.read_object(BUCKET, object).await.unwrap().unwrap();
let bytes = if case == 0 {
b"invalid-avro".to_vec()
} else {
let mut value: serde_json::Value = serde_json::from_slice(&stored.data).unwrap();
value[if case == 1 { "table-uuid" } else { "view-uuid" }] = serde_json::json!("another-identity");
serde_json::to_vec(&value).unwrap()
};
fixture.backend.seed_object(BUCKET, object, bytes).await;
fixture.backend.reset_call_counts().await;
let store = ObjectTableCatalogStore::new(fixture.backend.clone());
assert!(
store
.capture_object_catalog_backup_references(BUCKET, &fixture.objects)
.await
.is_err()
);
assert!(fixture.backend.state.lock().await.put_attempts.is_empty());
}
}
#[tokio::test]
async fn fingerprint_rejects_changed_etag_after_the_content_read() {
let backend = TestCatalogObjectBackend::default();
backend.seed_object(BUCKET, DATA, vec![1]).await;
let pause = backend.pause_next_read(BUCKET, DATA).await;
let reader_backend = backend.clone();
let task = tokio::spawn(async move {
ObjectTableCatalogStore::new(reader_backend)
.object_catalog_backup_reference_fingerprint(BUCKET, DATA, ObjectCatalogBackupReferenceKind::DataFile)
.await
});
tokio::time::timeout(std::time::Duration::from_secs(5), pause.wait_started())
.await
.unwrap();
backend.seed_object(BUCKET, DATA, vec![1]).await;
pause.release();
let result = tokio::time::timeout(std::time::Duration::from_secs(5), task)
.await
.unwrap()
.unwrap();
assert!(matches!(result, Err(TableCatalogStoreError::Conflict(_))));
}
#[tokio::test]
async fn fingerprint_propagates_read_failure_and_rejects_etagless_objects() {
let backend = TestCatalogObjectBackend::default();
backend.seed_object(BUCKET, DATA, vec![1]).await;
let store = ObjectTableCatalogStore::new(backend.clone());
*backend.fail_read_object_path.lock().await = Some(DATA.to_string());
assert!(matches!(
store
.object_catalog_backup_reference_fingerprint(BUCKET, DATA, ObjectCatalogBackupReferenceKind::DataFile)
.await,
Err(TableCatalogStoreError::Internal(_))
));
backend.omit_etag_for_object(BUCKET, DATA).await;
assert!(matches!(
store
.object_catalog_backup_reference_fingerprint(BUCKET, DATA, ObjectCatalogBackupReferenceKind::DataFile)
.await,
Err(TableCatalogStoreError::Invalid(_))
));
}
#[tokio::test]
async fn large_objects_keep_the_size_etag_contract_and_are_not_read_into_memory() {
let fixture = inventory_fixture().await;
let length = usize::try_from(OBJECT_CATALOG_BACKUP_REFERENCE_HASH_MAX_SIZE).unwrap() + 1;
fixture.backend.seed_object(BUCKET, DATA, vec![1; length]).await;
fixture.backend.reset_call_counts().await;
let source = ObjectTableCatalogStore::new(fixture.backend.clone());
let references = source
.capture_object_catalog_backup_references(BUCKET, &fixture.objects)
.await
.unwrap();
let large = references.iter().find(|reference| reference.object == DATA).unwrap();
assert_eq!(large.size_bytes, u64::try_from(length).unwrap());
assert_eq!(large.sha256, None);
assert_eq!(fixture.backend.last_read_limit(BUCKET, DATA).await, None);
let backend = copied_backend_with_new_etags(&fixture.backend).await;
let target = ObjectTableCatalogStore::new(backend.clone());
assert!(matches!(
target
.validate_object_catalog_backup_references(BUCKET, &fixture.objects, &references)
.await,
Err(TableCatalogStoreError::Conflict(_))
));
backend
.state
.lock()
.await
.objects
.get_mut(&(BUCKET.to_string(), DATA.to_string()))
.unwrap()
.etag = large.etag.clone().unwrap();
target
.validate_object_catalog_backup_references(BUCKET, &fixture.objects, &references)
.await
.unwrap();
assert_eq!(backend.last_read_limit(BUCKET, DATA).await, None);
}
@@ -1,262 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::super::object::validate_table_maintenance_report_owner;
use super::*;
#[derive(Debug, Default, PartialEq, Eq)]
pub(super) struct ObjectCatalogBackupMaintenanceSummary {
pub(super) object_count: usize,
}
#[derive(Default)]
struct BackupMaintenanceOwner {
named: BTreeMap<String, TableMetadataMaintenanceReport>,
latest: Option<TableMetadataMaintenanceReport>,
current: Option<TableMetadataMaintenanceReport>,
}
impl<B> ObjectTableCatalogStore<B>
where
B: TableCatalogObjectBackend,
{
/// Counts recognized sidecars without granting authority to a candidate.
pub(super) fn backup_maintenance_object_count(&self, table_bucket: &str, objects: &[ObjectCatalogBackupObject]) -> usize {
let config = self.paths.table_bucket_maintenance_config_path(table_bucket);
objects
.iter()
.filter(|object| {
object.bucket == self.catalog_bucket()
&& (object.object == config
|| backup_historical_maintenance_path(&self.paths, table_bucket, &object.object).is_some())
})
.count()
}
/// Validates maintenance authority in an already captured catalog snapshot.
///
/// Callers hold the backup/publication fences and validate the artifact's
/// ordinary entry, size, hash and metadata contracts separately. This pure
/// check never expires a lease or repairs aliases. It preserves all source
/// bytes, including audit, quarantine and historical worker identities.
pub(super) fn validate_backup_maintenance_authority(
&self,
table_bucket: &str,
objects: &[ObjectCatalogBackupObject],
) -> TableCatalogStoreResult<ObjectCatalogBackupMaintenanceSummary> {
let mut owners = BTreeSet::new();
let mut seen_paths = BTreeSet::new();
for object in objects.iter().filter(|object| object.bucket == self.catalog_bucket()) {
if !seen_paths.insert(object.object.as_str()) {
return Err(TableCatalogStoreError::Invalid("catalog backup has a duplicate object path".to_string()));
}
if !object.object.ends_with(TABLE_ENTRY_FILE) {
continue;
}
let entry: TableEntry = serde_json::from_slice(&object.data)
.map_err(|_| TableCatalogStoreError::Invalid("invalid maintenance catalog owner in backup".to_string()))?;
validate_table_entry_object(&self.paths, &object.object, &entry)?;
if entry.table_bucket != table_bucket {
return Err(TableCatalogStoreError::Invalid(
"backup maintenance owner belongs to another bucket".to_string(),
));
}
if matches!(entry.state, TableCatalogEntryState::Renaming | TableCatalogEntryState::Deleting) {
return Err(backup_maintenance_recovery_required("catalog owner has a pending table operation"));
}
// read_table_with_etag_unlocked does not filter Deleted entries.
// Retained owners must therefore also fence their current lease.
let namespace = parse_namespace_for_store(&entry.namespace)?;
let table = parse_table_for_store(&entry.table)?;
let config = self
.paths
.table_maintenance_config_path(table_bucket, &namespace, &table, &entry.table_id);
owners.insert(config.trim_end_matches(MAINTENANCE_CONFIG_FILE).to_string());
}
let mut summary = ObjectCatalogBackupMaintenanceSummary::default();
let mut reports = BTreeMap::<String, BackupMaintenanceOwner>::new();
let bucket_config = self.paths.table_bucket_maintenance_config_path(table_bucket);
for object in objects.iter().filter(|object| object.bucket == self.catalog_bucket()) {
if object.object == bucket_config {
validate_backup_maintenance_config(&object.data)?;
summary.object_count += 1;
continue;
}
if object.object.ends_with(TABLE_ENTRY_FILE)
|| object.object.ends_with(NAMESPACE_ENTRY_FILE)
|| object.object.ends_with(VIEW_ENTRY_FILE)
{
continue;
}
let owned_prefix = owners
.range(..=object.object.clone())
.next_back()
.filter(|prefix| object.object.starts_with(prefix.as_str()));
let path = match owned_prefix {
Some(prefix) => Some((prefix.as_str(), &object.object[prefix.len()..])),
None => backup_historical_maintenance_path(&self.paths, table_bucket, &object.object),
};
let Some((prefix, suffix)) = path else {
continue;
};
summary.object_count += 1;
if suffix == MAINTENANCE_CONFIG_FILE {
validate_backup_maintenance_config(&object.data)?;
continue;
}
let report: TableMetadataMaintenanceReport = serde_json::from_slice(&object.data)
.map_err(|_| TableCatalogStoreError::Invalid("invalid maintenance report in backup".to_string()))?;
if report.job.table_id.is_empty() {
return Err(TableCatalogStoreError::Invalid(
"backup maintenance report has no table identity".to_string(),
));
}
let namespace = parse_namespace_for_store(&report.job.namespace)?;
let table = parse_table_for_store(&report.job.table)?;
validate_table_maintenance_report_owner(&report, table_bucket, &namespace, &table, &report.job.table_id)?;
let owner = reports.entry(prefix.to_string()).or_default();
let expected = match suffix {
MAINTENANCE_CURRENT_JOB_FILE => {
owner.current = Some(report.clone());
self.paths
.table_maintenance_current_job_path(table_bucket, &namespace, &table, &report.job.table_id)
}
MAINTENANCE_LATEST_JOB_FILE => {
owner.latest = Some(report.clone());
self.paths
.table_maintenance_latest_job_path(table_bucket, &namespace, &table, &report.job.table_id)
}
_ if suffix.starts_with(&format!("{MAINTENANCE_JOB_ROOT}/")) => {
owner.named.insert(report.job.job_id.clone(), report.clone());
self.paths.table_maintenance_job_path(
table_bucket,
&namespace,
&table,
&report.job.table_id,
&report.job.job_id,
)
}
_ => {
return Err(TableCatalogStoreError::Invalid(
"unexpected maintenance object path in backup".to_string(),
));
}
};
if object.object != expected {
return Err(TableCatalogStoreError::Invalid(
"backup maintenance report does not match its object path".to_string(),
));
}
}
for (prefix, owner) in reports {
if !owners.contains(&prefix) {
// No restored catalog owner can discover this incarnation.
// Keep its historical bytes without granting execution authority.
continue;
}
let current = owner
.current
.as_ref()
.ok_or_else(|| backup_maintenance_recovery_required("maintenance reports have no current authority"))?;
let latest = owner
.latest
.as_ref()
.ok_or_else(|| backup_maintenance_recovery_required("current maintenance authority has no latest report"))?;
let named = owner
.named
.get(&current.job.job_id)
.ok_or_else(|| backup_maintenance_recovery_required("current maintenance authority has no named report"))?;
if current != latest || current != named {
return Err(backup_maintenance_recovery_required(
"named, latest and current maintenance reports are not consistently published",
));
}
if matches!(
current.job.status,
TableMetadataMaintenanceJobStatus::Queued | TableMetadataMaintenanceJobStatus::Running
) {
return Err(TableCatalogStoreError::Conflict(format!(
"maintenance recovery required before backup or restore: {}/{}/{} job {} attempt {} is {:?}; finish or recover the current job and capture a new stable backup",
current.job.table_bucket,
current.job.namespace,
current.job.table,
current.job.job_id,
current.job.attempt,
current.job.status,
)));
}
// Only aliases grant authority. A historical named Running job with
// another id cannot heartbeat or reclaim this stable current job.
}
Ok(summary)
}
}
fn backup_maintenance_recovery_required(reason: &str) -> TableCatalogStoreError {
TableCatalogStoreError::Conflict(format!(
"maintenance recovery required before backup or restore: {reason}; inspect the named, latest and current reports, recover their existing authority, then capture a stable backup"
))
}
fn validate_backup_maintenance_config(data: &[u8]) -> TableCatalogStoreResult<()> {
let config: TableMaintenanceConfig = serde_json::from_slice(data)
.map_err(|_| TableCatalogStoreError::Invalid("invalid maintenance config in backup".to_string()))?;
validate_table_maintenance_config(&config)
}
/// Recognizes canonical, known-format sidecars from superseded incarnations.
/// Their embedded reports still undergo the ordinary owner/path validation.
fn backup_historical_maintenance_path<'a>(
paths: &TableCatalogObjectPaths,
table_bucket: &str,
object: &'a str,
) -> Option<(&'a str, &'a str)> {
let namespace_prefix = paths.namespace_entries_prefix(table_bucket);
let relative = object.strip_prefix(&namespace_prefix)?;
let (head, tail) = relative.rsplit_once(&format!("/{MAINTENANCE_ROOT}/"))?;
let (namespace_path, table) = head.rsplit_once(&format!("/{TABLE_ROOT}/"))?;
if table.contains('/')
|| IdentifierSegment::parse(table).is_err()
|| Namespace::from_segments(namespace_path.split('/').map(str::to_string).collect()).is_err()
{
return None;
}
let (identity_hash, suffix) = tail.split_once('/')?;
if identity_hash.len() != 64
|| !identity_hash
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
{
return None;
}
let is_known = matches!(
suffix,
MAINTENANCE_CONFIG_FILE | MAINTENANCE_CURRENT_JOB_FILE | MAINTENANCE_LATEST_JOB_FILE
) || suffix
.strip_prefix(&format!("{MAINTENANCE_JOB_ROOT}/"))
.and_then(|name| name.strip_suffix(".json"))
.is_some_and(|hash| {
hash.len() == 64
&& hash
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
});
is_known.then(|| (&object[..object.len() - suffix.len()], suffix))
}
#[cfg(test)]
#[path = "object_backup_maintenance_tests.rs"]
mod tests;
@@ -1,347 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::table_catalog::test_support::TestCatalogObjectBackend;
fn fixture() -> (
ObjectTableCatalogStore<TestCatalogObjectBackend>,
Vec<ObjectCatalogBackupObject>,
TableEntry,
) {
let backup: ObjectCatalogBackup =
serde_json::from_slice(include_bytes!("../tests/fixtures/object-catalog-backup-v1.json")).expect("existing v1 fixture");
let entry = serde_json::from_slice(
&backup
.objects
.iter()
.find(|object| object.object.ends_with(TABLE_ENTRY_FILE))
.unwrap()
.data,
)
.expect("fixture table owner");
(ObjectTableCatalogStore::new(TestCatalogObjectBackend::default()), backup.objects, entry)
}
fn report(entry: &TableEntry, job_id: &str, status: TableMetadataMaintenanceJobStatus) -> TableMetadataMaintenanceReport {
serde_json::from_value(serde_json::json!({
"job": {
"job_id": job_id,
"table_bucket": entry.table_bucket,
"namespace": entry.namespace,
"table": entry.table,
"table_id": entry.table_id,
"status": status,
"worker_id": "retained-worker",
"lease_id": "retained-lease",
"attempt": 1,
"heartbeat_at": "2000-01-01T00:00:00Z",
"started_at": "2000-01-01T00:00:00Z",
"finished_at": "2000-01-01T00:01:00Z",
"current_metadata_location": entry.metadata_location,
"current_generation": entry.generation,
"retain_recent_metadata_files": 2,
"safety_window_seconds": 900,
"cleanup_watermark_unix_seconds": 1,
"quarantine-enabled": true,
"quarantine-retention-seconds": 86400,
"quarantined_object_count": 1
},
"current_metadata_location": entry.metadata_location,
"retained_metadata_locations": [entry.metadata_location],
"cleanup_candidate_locations": [],
"deletable_metadata_locations": []
}))
.expect("valid maintenance report")
}
fn object(path: String, data: Vec<u8>) -> ObjectCatalogBackupObject {
ObjectCatalogBackupObject {
bucket: RUSTFS_META_BUCKET.to_string(),
object: path,
size_bytes: u64::try_from(data.len()).unwrap(),
etag: Some("retained-etag".to_string()),
sha256: backup_sha256(&data),
data,
}
}
fn report_paths(entry: &TableEntry, job_id: &str) -> [String; 3] {
let paths = TableCatalogObjectPaths::default();
let namespace = Namespace::parse(&entry.namespace).unwrap();
let table = IdentifierSegment::parse(&entry.table).unwrap();
[
paths.table_maintenance_job_path(&entry.table_bucket, &namespace, &table, &entry.table_id, job_id),
paths.table_maintenance_latest_job_path(&entry.table_bucket, &namespace, &table, &entry.table_id),
paths.table_maintenance_current_job_path(&entry.table_bucket, &namespace, &table, &entry.table_id),
]
}
fn publish(objects: &mut Vec<ObjectCatalogBackupObject>, entry: &TableEntry, report: &TableMetadataMaintenanceReport) {
for path in report_paths(entry, &report.job.job_id) {
objects.retain(|existing| existing.object != path);
objects.push(object(path, serde_json::to_vec(report).unwrap()));
}
}
fn assert_recovery_required(result: TableCatalogStoreResult<ObjectCatalogBackupMaintenanceSummary>) {
let error = result.expect_err("unstable maintenance must fail closed");
assert!(
matches!(&error, TableCatalogStoreError::Conflict(message) if message.contains("maintenance recovery required")),
"{error}"
);
}
#[test]
fn backup_without_maintenance_keeps_v1_objects_unchanged() {
let (store, objects, _) = fixture();
let before = objects.clone();
assert_eq!(
store
.validate_backup_maintenance_authority("analytics", &objects)
.unwrap()
.object_count,
0
);
assert_eq!(objects, before);
}
#[test]
fn backup_rejects_current_queued_and_running_even_after_their_timestamps_expired() {
for status in [
TableMetadataMaintenanceJobStatus::Queued,
TableMetadataMaintenanceJobStatus::Running,
] {
let (store, mut objects, entry) = fixture();
let mut active = report(&entry, "active-job", status);
active.job.finished_at = None;
publish(&mut objects, &entry, &active);
let before = objects.clone();
assert_recovery_required(store.validate_backup_maintenance_authority("analytics", &objects));
assert_eq!(objects, before, "lease, worker, attempt or aliases were silently rewritten");
}
}
#[test]
fn stable_authority_keeps_historical_running_jobs_configs_and_quarantine_bytes() {
let (store, mut objects, entry) = fixture();
let stable = report(&entry, "new-job", TableMetadataMaintenanceJobStatus::Successful);
publish(&mut objects, &entry, &stable);
let historical = report(&entry, "old-job", TableMetadataMaintenanceJobStatus::Running);
objects.push(object(
report_paths(&entry, &historical.job.job_id)[0].clone(),
serde_json::to_vec(&historical).unwrap(),
));
let paths = TableCatalogObjectPaths::default();
let namespace = Namespace::parse(&entry.namespace).unwrap();
let table = IdentifierSegment::parse(&entry.table).unwrap();
let config = TableMaintenanceConfig {
quarantine_enabled: true,
quarantine_retention_seconds: 86_400,
..Default::default()
};
for path in [
paths.table_bucket_maintenance_config_path("analytics"),
paths.table_maintenance_config_path("analytics", &namespace, &table, &entry.table_id),
] {
objects.push(object(path, serde_json::to_vec(&config).unwrap()));
}
// Additive audit/quarantine fields remain raw bytes, including across aliases.
for path in report_paths(&entry, &stable.job.job_id) {
let saved = objects.iter_mut().find(|object| object.object == path).unwrap();
let mut value: serde_json::Value = serde_json::from_slice(&saved.data).unwrap();
value["future-quarantine-audit"] = serde_json::json!({"original-worker": "retained-worker", "receipt": [1, 2, 3]});
*saved = object(path, serde_json::to_vec(&value).unwrap());
}
let before = objects.clone();
assert_eq!(
store
.validate_backup_maintenance_authority("analytics", &objects)
.unwrap()
.object_count,
6
);
assert_eq!(objects, before);
}
#[test]
fn backup_requires_every_current_report_view() {
for missing in 0..3 {
let (store, mut objects, entry) = fixture();
let stable = report(&entry, "stable-job", TableMetadataMaintenanceJobStatus::Successful);
publish(&mut objects, &entry, &stable);
let missing = report_paths(&entry, &stable.job.job_id)[missing].clone();
objects.retain(|object| object.object != missing);
assert_recovery_required(store.validate_backup_maintenance_authority("analytics", &objects));
}
}
#[test]
fn backup_rejects_different_attempt_or_lease_even_with_the_same_job_id() {
for changed_field in ["attempt", "lease"] {
let (store, mut objects, entry) = fixture();
let stable = report(&entry, "stable-job", TableMetadataMaintenanceJobStatus::Successful);
publish(&mut objects, &entry, &stable);
let mut changed = stable.clone();
if changed_field == "attempt" {
changed.job.attempt += 1;
} else {
changed.job.lease_id = "another-lease".to_string();
}
let latest = report_paths(&entry, &stable.job.job_id)[1].clone();
let target = objects
.iter_mut()
.find(|object| object.object == latest)
.expect("latest maintenance receipt");
*target = object(latest, serde_json::to_vec(&changed).unwrap());
assert_recovery_required(store.validate_backup_maintenance_authority("analytics", &objects));
}
}
#[test]
fn backup_does_not_repair_partially_published_terminal_receipts() {
let (store, mut objects, entry) = fixture();
let running = report(&entry, "same-attempt", TableMetadataMaintenanceJobStatus::Running);
publish(&mut objects, &entry, &running);
let terminal = report(&entry, "same-attempt", TableMetadataMaintenanceJobStatus::Successful);
let named = report_paths(&entry, &terminal.job.job_id)[0].clone();
let target = objects
.iter_mut()
.find(|object| object.object == named)
.expect("named maintenance receipt");
*target = object(named, serde_json::to_vec(&terminal).unwrap());
let before = objects.clone();
assert_recovery_required(store.validate_backup_maintenance_authority("analytics", &objects));
assert_eq!(objects, before);
}
#[test]
fn backup_validates_report_owner_and_named_job_path() {
for changed in ["bucket", "owner", "job-path"] {
let (store, mut objects, entry) = fixture();
let stable = report(&entry, "stable-job", TableMetadataMaintenanceJobStatus::Successful);
publish(&mut objects, &entry, &stable);
let named = report_paths(&entry, &stable.job.job_id)[0].clone();
let saved = objects.iter_mut().find(|object| object.object == named).unwrap();
let mut changed_report = stable.clone();
match changed {
"bucket" => changed_report.job.table_bucket = "another-bucket".to_string(),
"owner" => changed_report.job.table_id = "another-table-id".to_string(),
"job-path" => changed_report.job.job_id = "another-job".to_string(),
_ => unreachable!(),
}
*saved = object(named, serde_json::to_vec(&changed_report).unwrap());
assert!(matches!(
store.validate_backup_maintenance_authority("analytics", &objects),
Err(TableCatalogStoreError::Invalid(_))
));
}
}
#[test]
fn backup_rejects_an_unknown_sidecar_under_the_current_owner() {
let (store, mut objects, entry) = fixture();
let config = TableCatalogObjectPaths::default().table_maintenance_config_path(
"analytics",
&Namespace::parse(&entry.namespace).unwrap(),
&IdentifierSegment::parse(&entry.table).unwrap(),
&entry.table_id,
);
let path = format!("{}unknown.json", config.trim_end_matches(MAINTENANCE_CONFIG_FILE));
objects.push(object(
path,
serde_json::to_vec(&report(&entry, "job", TableMetadataMaintenanceJobStatus::Successful)).unwrap(),
));
assert!(matches!(
store.validate_backup_maintenance_authority("analytics", &objects),
Err(TableCatalogStoreError::Invalid(_))
));
}
#[test]
fn superseded_incarnation_sidecars_do_not_gain_authority_on_restore() {
let (store, mut objects, mut old_entry) = fixture();
old_entry.table_id = "superseded-incarnation".to_string();
let old = report(&old_entry, "old-running", TableMetadataMaintenanceJobStatus::Running);
publish(&mut objects, &old_entry, &old);
let before = objects.clone();
assert_eq!(
store
.validate_backup_maintenance_authority("analytics", &objects)
.unwrap()
.object_count,
3
);
assert_eq!(objects, before);
}
#[test]
fn isolated_running_history_for_a_retained_owner_requires_explicit_recovery() {
let (store, mut objects, entry) = fixture();
let running = report(&entry, "unresolved-job", TableMetadataMaintenanceJobStatus::Running);
objects.push(object(
report_paths(&entry, &running.job.job_id)[0].clone(),
serde_json::to_vec(&running).unwrap(),
));
assert_recovery_required(store.validate_backup_maintenance_authority("analytics", &objects));
}
#[test]
fn a_deleted_catalog_owner_still_cannot_restore_a_discoverable_running_lease() {
let (store, mut objects, mut entry) = fixture();
entry.state = TableCatalogEntryState::Deleted;
let path = objects
.iter()
.find(|object| object.object.ends_with(TABLE_ENTRY_FILE))
.unwrap()
.object
.clone();
let target = objects
.iter_mut()
.find(|object| object.object == path)
.expect("catalog owner entry");
*target = object(path, serde_json::to_vec(&entry).unwrap());
publish(
&mut objects,
&entry,
&report(&entry, "retained-running", TableMetadataMaintenanceJobStatus::Running),
);
assert_recovery_required(store.validate_backup_maintenance_authority("analytics", &objects));
}
#[test]
fn hierarchical_namespaces_and_a_table_named_maintenance_keep_canonical_paths() {
let (store, mut objects, mut entry) = fixture();
objects.retain(|object| !object.object.ends_with(TABLE_ENTRY_FILE));
entry.namespace = "sales.history".to_string();
entry.table = "maintenance".to_string();
let path = TableCatalogObjectPaths::default().table_entry_path(
"analytics",
&Namespace::parse(&entry.namespace).unwrap(),
&IdentifierSegment::parse(&entry.table).unwrap(),
);
objects.push(object(path, serde_json::to_vec(&entry).unwrap()));
publish(
&mut objects,
&entry,
&report(&entry, "stable", TableMetadataMaintenanceJobStatus::Successful),
);
assert_eq!(
store
.validate_backup_maintenance_authority("analytics", &objects)
.unwrap()
.object_count,
3
);
}
@@ -1,490 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::super::backup_types::{TableCatalogBackupObjectPreparationReport, TableCatalogBackupObjectPreparationStatus};
use super::inventory::{
backup_reference_metadata_log_locations, validate_backup_reference_table_metadata, validate_backup_reference_view_metadata,
};
use super::*;
const BACKUP_PREPARATION_MAX_HISTORY_SCOPES: usize = 1024;
enum BackupObjectPreparationRestoreGate {
Unblocked { version: Option<String> },
Pending { controls: [(String, String); 2] },
}
fn preparation_pending_phases_match(state: ObjectCatalogRestoreStateKind, intent: ObjectCatalogRestoreStateKind) -> bool {
matches!(
(state, intent),
(ObjectCatalogRestoreStateKind::Prepared, ObjectCatalogRestoreStateKind::Prepared)
| (ObjectCatalogRestoreStateKind::Applying, ObjectCatalogRestoreStateKind::Prepared)
| (ObjectCatalogRestoreStateKind::Applying, ObjectCatalogRestoreStateKind::CatalogApplied)
| (
ObjectCatalogRestoreStateKind::CatalogApplied,
ObjectCatalogRestoreStateKind::CatalogApplied
)
| (ObjectCatalogRestoreStateKind::CatalogApplied, ObjectCatalogRestoreStateKind::Completed)
)
}
fn preparation_graph_scope_matches(
entry: &TableEntry,
metadata: &serde_json::Value,
reference: &ObjectCatalogBackupReference,
) -> TableCatalogStoreResult<bool> {
let prefix = table_warehouse_object_prefix(entry)?;
let expected_kind = match reference.kind {
ObjectCatalogBackupReferenceKind::ManifestList => TableMetadataMaintenanceObjectKind::ManifestList,
ObjectCatalogBackupReferenceKind::ManifestFile => TableMetadataMaintenanceObjectKind::ManifestFile,
ObjectCatalogBackupReferenceKind::DataFile => TableMetadataMaintenanceObjectKind::DataFile,
ObjectCatalogBackupReferenceKind::DeleteFile => TableMetadataMaintenanceObjectKind::DeleteFile,
ObjectCatalogBackupReferenceKind::TableStatistics | ObjectCatalogBackupReferenceKind::PartitionStatistics => {
for statistics in table_statistics_object_references(metadata)? {
let kind_matches = matches!(
(reference.kind, statistics.kind),
(ObjectCatalogBackupReferenceKind::TableStatistics, IcebergStatisticsFileKind::Table)
| (
ObjectCatalogBackupReferenceKind::PartitionStatistics,
IcebergStatisticsFileKind::Partition
)
);
if kind_matches
&& reference.object.starts_with(&prefix)
&& table_catalog_object_key_from_location(&entry.table_bucket, &statistics.location).as_deref()
== Some(reference.object.as_str())
{
return Ok(true);
}
}
return Ok(false);
}
ObjectCatalogBackupReferenceKind::TableMetadata | ObjectCatalogBackupReferenceKind::ViewMetadata => return Ok(false),
};
Ok(table_maintenance_object_kind_for_entry(entry, Some(&prefix), &reference.object)
.is_some_and(|actual| table_maintenance_object_kind_matches_reference(&actual, &expected_kind)))
}
fn validate_preparation_warehouse_owner(
entry: &TableEntry,
tables: &[(TableEntry, serde_json::Value)],
) -> TableCatalogStoreResult<()> {
let prefix = table_warehouse_object_prefix(entry)?;
for (other, _) in tables {
if other.table_id != entry.table_id && warehouse_object_prefixes_overlap(&prefix, &table_warehouse_object_prefix(other)?)
{
return Err(TableCatalogStoreError::Conflict(
"backup object preparation scope overlaps another active table".to_string(),
));
}
}
Ok(())
}
impl<B> ObjectTableCatalogStore<B>
where
B: TableCatalogObjectBackend,
{
async fn backup_object_preparation_restore_gate(
&self,
table_bucket: &str,
backup: &ObjectCatalogBackup,
) -> TableCatalogStoreResult<BackupObjectPreparationRestoreGate> {
self.reject_legacy_catalog_control_state(table_bucket).await?;
let state = self.read_object_catalog_restore_state(table_bucket).await?;
let Some((state, state_etag)) = state else {
self.ensure_no_active_restore(table_bucket).await?;
return Ok(BackupObjectPreparationRestoreGate::Unblocked { version: None });
};
if state.state == ObjectCatalogRestoreStateKind::Completed {
self.ensure_no_active_restore(table_bucket).await?;
return Ok(BackupObjectPreparationRestoreGate::Unblocked {
version: Some(state_etag),
});
}
if state.backup_id != backup.backup_id {
return Err(TableCatalogStoreError::Unavailable(
"backup object preparation cannot cross another backup's pending restore".to_string(),
));
}
let (intent, intent_etag) = self
.read_object_catalog_restore_intent(table_bucket, &state.restore_id)
.await?
.ok_or_else(|| TableCatalogStoreError::Unavailable("pending restore is missing its durable intent".to_string()))?;
if state.restore_epoch != intent.restore_epoch
|| state.restore_id != intent.restore_id
|| state.backup_id != intent.backup_id
|| state.request_fingerprint != intent.request_fingerprint
|| state.completed_intent_etag.is_some()
|| state.backup_etag.is_some()
|| !preparation_pending_phases_match(state.state, intent.state)
{
return Err(TableCatalogStoreError::Unavailable(
"pending restore does not authorize backup object preparation for this request".to_string(),
));
}
self.validate_restore_plan(backup, &intent)?;
Ok(BackupObjectPreparationRestoreGate::Pending {
controls: [
(self.paths.catalog_restore_state_path(table_bucket), state_etag),
(self.paths.catalog_restore_intent_path(table_bucket, &state.restore_id), intent_etag),
],
})
}
async fn finish_backup_object_preparation_gate(
&self,
table_bucket: &str,
gate: &BackupObjectPreparationRestoreGate,
) -> TableCatalogStoreResult<()> {
match gate {
BackupObjectPreparationRestoreGate::Unblocked { version } => {
self.finish_catalog_restore_read(table_bucket, version.as_deref()).await
}
BackupObjectPreparationRestoreGate::Pending { controls } => {
for (object, expected) in controls {
let actual = self.backend.object_metadata(self.catalog_bucket(), object).await?;
if actual.as_ref().and_then(|actual| actual.etag.as_deref()) != Some(expected.as_str()) {
return Err(TableCatalogStoreError::Unavailable(
"pending restore controls changed during backup object preparation".to_string(),
));
}
}
Ok(())
}
}
}
async fn read_stable_preparation_object(
&self,
table_bucket: &str,
object: &str,
) -> TableCatalogStoreResult<Option<TableCatalogObject>> {
let Some(before) = self.backend.object_metadata(table_bucket, object).await? else {
return Ok(None);
};
let max_size = u64::try_from(OBJECT_CATALOG_BACKUP_OBJECT_MAX_SIZE)
.map_err(|_| TableCatalogStoreError::Internal("backup preparation size limit is invalid".to_string()))?;
if before.size > max_size || before.etag.as_deref().is_none_or(str::is_empty) {
return Err(TableCatalogStoreError::Conflict(
"existing preparation object cannot be safely compared within the size and ETag limits".to_string(),
));
}
let stored = self
.backend
.read_object_limited(table_bucket, object, OBJECT_CATALOG_BACKUP_OBJECT_MAX_SIZE)
.await?
.ok_or_else(|| TableCatalogStoreError::Conflict("preparation object disappeared during its read".to_string()))?;
let after = self
.backend
.object_metadata(table_bucket, object)
.await?
.ok_or_else(|| TableCatalogStoreError::Conflict("preparation object disappeared after its read".to_string()))?;
if stored.etag != before.etag
|| after.etag != before.etag
|| after.size != before.size
|| u64::try_from(stored.data.len()).ok() != Some(before.size)
{
return Err(TableCatalogStoreError::Conflict("preparation object changed during its read".to_string()));
}
Ok(Some(stored))
}
async fn validate_backup_object_preparation_scope(
&self,
table_bucket: &str,
backup: &ObjectCatalogBackup,
references: &[ObjectCatalogBackupReference],
reference: &ObjectCatalogBackupReference,
data: &[u8],
fence: &TableCatalogObjectMutationFence,
) -> TableCatalogStoreResult<()> {
let views = self.backup_reference_view_entries(table_bucket, &backup.objects)?;
let tables = self.validate_catalog_backup_metadata_identities(table_bucket, &backup.objects)?;
for (entry, metadata) in &tables {
let current = table_catalog_object_key_from_location(table_bucket, &entry.metadata_location).ok_or_else(|| {
TableCatalogStoreError::Invalid("backup table current metadata location is invalid".to_string())
})?;
validate_backup_reference_table_metadata(entry, &current, metadata, true)?;
validate_preparation_warehouse_owner(entry, &tables)?;
if reference.kind == ObjectCatalogBackupReferenceKind::TableMetadata {
let historical = backup_reference_metadata_log_locations(metadata, entry)?;
if reference.object == current || historical.contains(&reference.object) {
if reference.object == current {
let embedded = backup
.objects
.iter()
.find(|object| object.bucket == table_bucket && object.object == current)
.ok_or_else(|| {
TableCatalogStoreError::Invalid("backup is missing its current metadata payload".to_string())
})?;
if embedded.data.as_slice() != data {
return Err(TableCatalogStoreError::Invalid(
"current metadata preparation differs from the embedded backup payload".to_string(),
));
}
}
let prepared = decode_table_metadata_json(&reference.object, data)?;
validate_backup_reference_table_metadata(entry, &reference.object, &prepared, reference.object == current)?;
let mut prepared_entry = entry.clone();
prepared_entry.warehouse_location = table_metadata_location(&prepared)?.to_string();
validate_preparation_warehouse_owner(&prepared_entry, &tables)?;
return Ok(());
}
} else if preparation_graph_scope_matches(entry, metadata, reference)? {
return Ok(());
}
}
if reference.kind == ObjectCatalogBackupReferenceKind::ViewMetadata {
for entry in views {
if table_catalog_object_key_from_location(table_bucket, &entry.metadata_location).as_deref()
== Some(reference.object.as_str())
{
let metadata = decode_table_metadata_json(&reference.object, data)?;
validate_backup_reference_view_metadata(
&entry,
&parse_namespace_for_store(&entry.namespace)?,
&parse_table_for_store(&entry.view)?,
&reference.object,
&metadata,
)?;
return Ok(());
}
}
}
if matches!(
reference.kind,
ObjectCatalogBackupReferenceKind::TableMetadata | ObjectCatalogBackupReferenceKind::ViewMetadata
) {
return Err(TableCatalogStoreError::Invalid(
"backup metadata preparation is outside an active entry's direct metadata scope".to_string(),
));
}
let mut inspected = 0usize;
let mut read_bytes = 0u64;
let mut missing_history = false;
let max_read_bytes = u64::try_from(OBJECT_CATALOG_BACKUP_MAX_SIZE)
.map_err(|_| TableCatalogStoreError::Internal("backup preparation history budget is invalid".to_string()))?;
for (entry, metadata) in &tables {
let current = table_catalog_object_key_from_location(table_bucket, &entry.metadata_location).ok_or_else(|| {
TableCatalogStoreError::Invalid("backup table current metadata location is invalid".to_string())
})?;
for object in backup_reference_metadata_log_locations(metadata, entry)? {
fence.ensure_held()?;
if object == current {
continue;
}
inspected += 1;
if inspected > BACKUP_PREPARATION_MAX_HISTORY_SCOPES {
return Err(TableCatalogStoreError::Unavailable(
"historical preparation scope lookup exceeds its 1024-object budget".to_string(),
));
}
let saved = references
.binary_search_by(|candidate| {
candidate
.bucket
.as_str()
.cmp(table_bucket)
.then_with(|| candidate.object.cmp(&object))
})
.ok()
.and_then(|index| references.get(index))
.ok_or_else(|| {
TableCatalogStoreError::Invalid(
"direct history metadata is missing from the backup reference inventory".to_string(),
)
})?;
if saved.kind != ObjectCatalogBackupReferenceKind::TableMetadata {
return Err(TableCatalogStoreError::Invalid(
"direct history has a conflicting backup reference kind".to_string(),
));
}
read_bytes = read_bytes
.checked_add(saved.size_bytes)
.filter(|bytes| *bytes <= max_read_bytes)
.ok_or_else(|| {
TableCatalogStoreError::Unavailable(
"historical preparation scope lookup exceeds its byte budget".to_string(),
)
})?;
let Some(stored) = self.read_stable_preparation_object(table_bucket, &object).await? else {
missing_history = true;
continue;
};
if u64::try_from(stored.data.len()).ok() != Some(saved.size_bytes)
|| saved.sha256.as_deref() != Some(backup_sha256(&stored.data).as_str())
{
return Err(TableCatalogStoreError::Conflict(
"prepared historical metadata does not match its backup watermark".to_string(),
));
}
let metadata = decode_table_metadata_json(&object, &stored.data)?;
validate_backup_reference_table_metadata(entry, &object, &metadata, false)?;
let mut historical_entry = entry.clone();
historical_entry.warehouse_location = table_metadata_location(&metadata)?.to_string();
validate_preparation_warehouse_owner(&historical_entry, &tables)?;
if preparation_graph_scope_matches(&historical_entry, &metadata, reference)? {
return Ok(());
}
}
}
if missing_history {
return Err(TableCatalogStoreError::Unavailable(
"prepare the referenced direct history metadata before preparing an object outside current warehouse scope"
.to_string(),
));
}
Err(TableCatalogStoreError::Invalid(
"backup object preparation is outside its active entries and prepared direct history scopes".to_string(),
))
}
/// Prepares one small object referenced by a saved v2 backup without publishing catalog state.
///
/// Only an absent key is written. Exact-byte retries keep the target ETag; different existing bytes are never
/// replaced. An ambiguous put is confirmed by a stable read of the original bytes. Cancellation may leave that
/// immutable object prepared; retrying the same input confirms it. Pending restore controls are never changed.
pub(crate) async fn prepare_object_catalog_backup_object(
&self,
table_bucket: &str,
backup_id: &str,
object: &str,
data: Vec<u8>,
) -> TableCatalogStoreResult<TableCatalogBackupObjectPreparationReport> {
if table_bucket == self.catalog_bucket() || !safe_object_key(object) || data.len() > OBJECT_CATALOG_BACKUP_OBJECT_MAX_SIZE
{
return Err(TableCatalogStoreError::Invalid(
"backup object preparation has an invalid bucket, key, or body size".to_string(),
));
}
let guards = self.acquire_object_backup_fence(table_bucket).await?;
let fence = Self::object_backup_mutation_fence(&guards);
fence.ensure_held()?;
let backup_path = self.paths.catalog_backup_path(table_bucket, backup_id);
let artifact = self
.backend
.read_object_limited(self.catalog_bucket(), &backup_path, OBJECT_CATALOG_BACKUP_MAX_SIZE)
.await?
.ok_or_else(|| TableCatalogStoreError::NotFound(format!("catalog backup {backup_id}")))?;
let artifact_etag = artifact.etag.as_deref().filter(|etag| !etag.is_empty()).ok_or_else(|| {
TableCatalogStoreError::Unavailable("catalog backup has no stable ETag for preparation".to_string())
})?;
let backup = self.decode_object_catalog_backup(table_bucket, &artifact.data)?;
if backup.backup_id != backup_id {
return Err(TableCatalogStoreError::Invalid(
"catalog backup identity does not match its preparation request".to_string(),
));
}
let references = backup
.references
.as_deref()
.filter(|_| backup.version == OBJECT_CATALOG_BACKUP_INVENTORY_VERSION)
.ok_or_else(|| {
TableCatalogStoreError::Unsupported("object preparation requires a v2 catalog backup inventory".to_string())
})?;
validate_object_catalog_backup_reference_shape(table_bucket, references)?;
let index = references
.binary_search_by(|reference| {
reference
.bucket
.as_str()
.cmp(table_bucket)
.then_with(|| reference.object.as_str().cmp(object))
})
.map_err(|_| {
TableCatalogStoreError::Invalid("object is not present in this catalog backup reference inventory".to_string())
})?;
let reference = &references[index];
let sha256 = backup_sha256(&data);
if u64::try_from(data.len()).ok() != Some(reference.size_bytes) || reference.sha256.as_deref() != Some(sha256.as_str()) {
return Err(TableCatalogStoreError::Invalid(
"prepared object body does not match its backup size and SHA-256 watermark".to_string(),
));
}
let gate = self.backup_object_preparation_restore_gate(table_bucket, &backup).await?;
self.validate_backup_object_preparation_scope(table_bucket, &backup, references, reference, &data, &fence)
.await?;
match reference.kind {
ObjectCatalogBackupReferenceKind::ManifestList => {
decode_manifest_list_avro_async(data.clone()).await?;
}
ObjectCatalogBackupReferenceKind::ManifestFile => {
decode_manifest_avro_async(data.clone()).await?;
}
_ => {}
}
fence.ensure_held()?;
self.finish_backup_object_preparation_gate(table_bucket, &gate).await?;
let (status, stored) = if let Some(stored) = self.read_stable_preparation_object(table_bucket, object).await? {
if stored.data != data {
return Err(TableCatalogStoreError::Conflict(
"backup preparation never replaces different existing object bytes".to_string(),
));
}
(TableCatalogBackupObjectPreparationStatus::AlreadyPresent, stored)
} else {
let result = self
.backend
.put_object_fenced(table_bucket, object, data.clone(), TableCatalogPutPrecondition::IfAbsent, &fence)
.await;
fence.ensure_held()?;
let saved = self.read_stable_preparation_object(table_bucket, object).await?;
match saved {
Some(saved) if saved.data == data => (
if result.is_ok() {
TableCatalogBackupObjectPreparationStatus::Prepared
} else {
TableCatalogBackupObjectPreparationStatus::AlreadyPresent
},
saved,
),
Some(_) => {
return Err(TableCatalogStoreError::Conflict(
"backup preparation key is occupied by different object bytes".to_string(),
));
}
None => {
result?;
return Err(TableCatalogStoreError::Unavailable(
"prepared object disappeared after its conditional write".to_string(),
));
}
}
};
fence.ensure_held()?;
self.finish_backup_object_preparation_gate(table_bucket, &gate).await?;
let final_artifact = self.backend.object_metadata(self.catalog_bucket(), &backup_path).await?;
if final_artifact.as_ref().and_then(|metadata| metadata.etag.as_deref()) != Some(artifact_etag) {
return Err(TableCatalogStoreError::Unavailable(
"catalog backup changed during object preparation".to_string(),
));
}
fence.ensure_held()?;
Ok(TableCatalogBackupObjectPreparationReport {
table_bucket: table_bucket.to_string(),
backup_id: backup_id.to_string(),
object: object.to_string(),
status,
size_bytes: reference.size_bytes,
sha256,
etag: stored.etag,
})
}
}
#[cfg(test)]
#[path = "object_backup_prepare_tests.rs"]
mod tests;
@@ -1,631 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::table_catalog::test_support::TestCatalogObjectBackend;
const BUCKET: &str = "analytics";
fn original_fixture() -> ObjectCatalogBackup {
serde_json::from_slice(include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/src/table_catalog/tests/fixtures/object-catalog-backup-v1.json"
)))
.unwrap()
}
fn preparation_reference(object: &str, kind: ObjectCatalogBackupReferenceKind, data: &[u8]) -> ObjectCatalogBackupReference {
ObjectCatalogBackupReference {
bucket: BUCKET.to_string(),
object: object.to_string(),
kind,
size_bytes: u64::try_from(data.len()).unwrap(),
etag: Some("original-source-etag".to_string()),
sha256: Some(backup_sha256(data)),
}
}
fn reseal_preparation_fixture(backup: &mut ObjectCatalogBackup) {
backup.version = 2;
backup
.objects
.sort_by(|left, right| (&left.bucket, &left.object).cmp(&(&right.bucket, &right.object)));
backup.source_catalog_etag = catalog_backup_fingerprint(&backup.objects).unwrap();
let references = backup.references.as_mut().unwrap();
references.sort_by(|left, right| (&left.bucket, &left.object).cmp(&(&right.bucket, &right.object)));
backup.backup_id =
catalog_backup_inventory_identity(&backup.table_bucket, &backup.source_catalog_etag, &backup.objects, references)
.unwrap();
}
fn preparation_fixture() -> ObjectCatalogBackup {
let mut backup = original_fixture();
backup.references = Some(
backup
.objects
.iter()
.filter(|object| object.bucket == BUCKET)
.map(|object| preparation_reference(&object.object, ObjectCatalogBackupReferenceKind::TableMetadata, &object.data))
.collect(),
);
reseal_preparation_fixture(&mut backup);
backup
}
fn current_payload(backup: &ObjectCatalogBackup) -> &ObjectCatalogBackupObject {
backup.objects.iter().find(|object| object.bucket == BUCKET).unwrap()
}
async fn save_preparation_fixture(backend: &TestCatalogObjectBackend, backup: &ObjectCatalogBackup) {
backend
.seed_object(
RUSTFS_META_BUCKET,
&TableCatalogObjectPaths::default().catalog_backup_path(BUCKET, &backup.backup_id),
serde_json::to_vec(backup).unwrap(),
)
.await;
}
async fn object_versions(backend: &TestCatalogObjectBackend) -> BTreeMap<(String, String), (Vec<u8>, String)> {
backend
.state
.lock()
.await
.objects
.iter()
.map(|(key, record)| (key.clone(), (record.data.clone(), record.etag.clone())))
.collect()
}
async fn system_versions(backend: &TestCatalogObjectBackend) -> BTreeMap<(String, String), (Vec<u8>, String)> {
object_versions(backend)
.await
.into_iter()
.filter(|((bucket, _), _)| bucket == RUSTFS_META_BUCKET)
.collect()
}
async fn pending_preparation_controls(
backend: &TestCatalogObjectBackend,
backup: &ObjectCatalogBackup,
barrier_phase: ObjectCatalogRestoreStateKind,
intent_phase: ObjectCatalogRestoreStateKind,
) -> (String, String) {
let paths = TableCatalogObjectPaths::default();
let restore_id = "prepare-recovery";
let epoch = Uuid::new_v4().to_string();
let fingerprint = restore_request_fingerprint(BUCKET, restore_id, &backup.backup_id, None, false).unwrap();
let intent = ObjectCatalogRestoreIntent {
version: OBJECT_CATALOG_RESTORE_STATE_VERSION,
table_bucket: BUCKET.to_string(),
restore_epoch: epoch.clone(),
restore_id: restore_id.to_string(),
backup_id: backup.backup_id.clone(),
request_fingerprint: fingerprint.clone(),
expected_catalog_etag: None,
allow_replace: false,
source_catalog_etag: backup.source_catalog_etag.clone(),
state: intent_phase,
previous_snapshot_etag: None,
restored_snapshot_etag: backup.source_catalog_etag.clone(),
object_count: backup.objects.len(),
maintenance_object_count: 0,
writes: backup
.objects
.iter()
.map(|object| ObjectCatalogRestoreWrite {
bucket: object.bucket.clone(),
object: object.object.clone(),
expected_etag: None,
already_present: false,
})
.collect(),
deletes: Vec::new(),
created_at: "2026-10-31T00:00:00Z".to_string(),
updated_at: "2026-10-31T00:00:00Z".to_string(),
};
let state = ObjectCatalogRestoreState {
version: OBJECT_CATALOG_RESTORE_STATE_VERSION,
table_bucket: BUCKET.to_string(),
restore_epoch: epoch,
restore_id: restore_id.to_string(),
backup_id: backup.backup_id.clone(),
request_fingerprint: fingerprint,
state: barrier_phase,
completed_intent_etag: None,
backup_etag: None,
};
let intent_path = paths.catalog_restore_intent_path(BUCKET, restore_id);
let state_path = paths.catalog_restore_state_path(BUCKET);
backend
.seed_object(RUSTFS_META_BUCKET, &intent_path, serde_json::to_vec(&intent).unwrap())
.await;
backend
.seed_object(RUSTFS_META_BUCKET, &state_path, serde_json::to_vec(&state).unwrap())
.await;
(state_path, intent_path)
}
#[tokio::test]
async fn preparation_only_creates_the_referenced_payload_and_replays_without_rewriting_it() {
let backend = TestCatalogObjectBackend::default();
let backup = preparation_fixture();
save_preparation_fixture(&backend, &backup).await;
let before = object_versions(&backend).await;
let store = ObjectTableCatalogStore::new(backend.clone());
let payload = current_payload(&backup);
let prepared = store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &payload.object, payload.data.clone())
.await
.unwrap();
assert!(matches!(prepared.status, TableCatalogBackupObjectPreparationStatus::Prepared));
assert_eq!(prepared.size_bytes, payload.size_bytes);
assert_eq!(prepared.sha256, payload.sha256);
assert_ne!(prepared.etag, backup.references.as_ref().unwrap()[0].etag);
let after = object_versions(&backend).await;
assert_eq!(after.len(), before.len() + 1);
assert_eq!(system_versions(&backend).await, before);
let retried = store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &payload.object, payload.data.clone())
.await
.unwrap();
assert!(matches!(retried.status, TableCatalogBackupObjectPreparationStatus::AlreadyPresent));
assert_eq!(retried.etag, prepared.etag);
assert_eq!(object_versions(&backend).await, after);
assert_eq!(
backend
.state
.lock()
.await
.put_attempts
.get(&(BUCKET.to_string(), payload.object.clone())),
Some(&1)
);
}
#[tokio::test]
async fn preparation_accepts_identical_existing_bytes_with_a_different_target_etag() {
let backend = TestCatalogObjectBackend::default();
let backup = preparation_fixture();
let payload = current_payload(&backup);
save_preparation_fixture(&backend, &backup).await;
backend.seed_object(BUCKET, &payload.object, payload.data.clone()).await;
let before = object_versions(&backend).await;
let store = ObjectTableCatalogStore::new(backend.clone());
let report = store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &payload.object, payload.data.clone())
.await
.unwrap();
assert!(matches!(report.status, TableCatalogBackupObjectPreparationStatus::AlreadyPresent));
assert_ne!(report.etag, backup.references.as_ref().unwrap()[0].etag);
assert_eq!(object_versions(&backend).await, before);
assert!(backend.state.lock().await.put_attempts.is_empty());
}
#[tokio::test]
async fn preparation_never_overwrites_unknown_existing_bytes() {
let backend = TestCatalogObjectBackend::default();
let backup = preparation_fixture();
let payload = current_payload(&backup);
save_preparation_fixture(&backend, &backup).await;
backend
.seed_object(BUCKET, &payload.object, b"unknown-existing-metadata".to_vec())
.await;
let before = object_versions(&backend).await;
let store = ObjectTableCatalogStore::new(backend.clone());
assert!(matches!(
store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &payload.object, payload.data.clone())
.await,
Err(TableCatalogStoreError::Conflict(_))
));
assert_eq!(object_versions(&backend).await, before);
assert!(backend.state.lock().await.put_attempts.is_empty());
}
#[tokio::test]
async fn preparation_rejects_an_existing_object_without_a_stable_etag() {
let backend = TestCatalogObjectBackend::default();
let backup = preparation_fixture();
let payload = current_payload(&backup);
save_preparation_fixture(&backend, &backup).await;
backend.seed_object(BUCKET, &payload.object, payload.data.clone()).await;
backend.omit_etag_for_object(BUCKET, &payload.object).await;
let before = object_versions(&backend).await;
let store = ObjectTableCatalogStore::new(backend.clone());
assert!(matches!(
store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &payload.object, payload.data.clone())
.await,
Err(TableCatalogStoreError::Conflict(_))
));
assert_eq!(object_versions(&backend).await, before);
assert!(backend.state.lock().await.put_attempts.is_empty());
}
#[tokio::test]
async fn preparation_rejects_an_existing_object_that_changes_during_comparison() {
let backend = TestCatalogObjectBackend::default();
let backup = preparation_fixture();
let payload = current_payload(&backup);
save_preparation_fixture(&backend, &backup).await;
backend.seed_object(BUCKET, &payload.object, payload.data.clone()).await;
let before = system_versions(&backend).await;
let pause = backend.pause_next_read(BUCKET, &payload.object).await;
let task_backend = backend.clone();
let backup_id = backup.backup_id.clone();
let object = payload.object.clone();
let bytes = payload.data.clone();
let task = tokio::spawn(async move {
ObjectTableCatalogStore::new(task_backend)
.prepare_object_catalog_backup_object(BUCKET, &backup_id, &object, bytes)
.await
});
tokio::time::timeout(std::time::Duration::from_secs(5), pause.wait_started())
.await
.unwrap();
backend.seed_object(BUCKET, &payload.object, payload.data.clone()).await;
pause.release();
let result = tokio::time::timeout(std::time::Duration::from_secs(5), task)
.await
.unwrap()
.unwrap();
assert!(matches!(result, Err(TableCatalogStoreError::Conflict(_))));
assert_eq!(system_versions(&backend).await, before);
assert!(backend.state.lock().await.put_attempts.is_empty());
}
#[tokio::test]
async fn preparation_confirms_an_ambiguous_put_without_repeating_the_write() {
let backend = TestCatalogObjectBackend::default();
let backup = preparation_fixture();
let payload = current_payload(&backup);
save_preparation_fixture(&backend, &backup).await;
let before = system_versions(&backend).await;
backend.fail_after_next_put(BUCKET, &payload.object).await;
let store = ObjectTableCatalogStore::new(backend.clone());
let report = store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &payload.object, payload.data.clone())
.await
.unwrap();
assert!(matches!(report.status, TableCatalogBackupObjectPreparationStatus::AlreadyPresent));
let first = object_versions(&backend).await;
let retry = store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &payload.object, payload.data.clone())
.await
.unwrap();
assert_eq!(retry.etag, report.etag);
assert_eq!(system_versions(&backend).await, before);
assert_eq!(object_versions(&backend).await, first);
assert_eq!(
backend
.state
.lock()
.await
.put_attempts
.get(&(BUCKET.to_string(), payload.object.clone())),
Some(&1)
);
}
#[tokio::test]
async fn preparation_failure_before_put_has_no_effects_and_can_be_retried() {
let backend = TestCatalogObjectBackend::default();
let backup = preparation_fixture();
let payload = current_payload(&backup);
save_preparation_fixture(&backend, &backup).await;
let before = object_versions(&backend).await;
backend.fail_next_put(BUCKET, &payload.object).await;
let store = ObjectTableCatalogStore::new(backend.clone());
assert!(
store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &payload.object, payload.data.clone())
.await
.is_err()
);
assert_eq!(object_versions(&backend).await, before);
let retry = store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &payload.object, payload.data.clone())
.await
.unwrap();
assert!(matches!(retry.status, TableCatalogBackupObjectPreparationStatus::Prepared));
}
#[tokio::test]
async fn preparation_rejects_bad_bodies_unlisted_keys_and_system_bucket_without_writes() {
let backup = preparation_fixture();
let payload = current_payload(&backup);
let mut corrupt = payload.data.clone();
corrupt[0] ^= 1;
for (bucket, object, data) in [
(BUCKET, payload.object.as_str(), corrupt),
(BUCKET, payload.object.as_str(), vec![0; OBJECT_CATALOG_BACKUP_OBJECT_MAX_SIZE + 1]),
(BUCKET, "tables/table-id/data/unlisted.parquet", vec![1]),
(BUCKET, "../outside", vec![1]),
(RUSTFS_META_BUCKET, payload.object.as_str(), payload.data.clone()),
] {
let backend = TestCatalogObjectBackend::default();
save_preparation_fixture(&backend, &backup).await;
let before = object_versions(&backend).await;
let store = ObjectTableCatalogStore::new(backend.clone());
assert!(
store
.prepare_object_catalog_backup_object(bucket, &backup.backup_id, object, data)
.await
.is_err()
);
assert_eq!(object_versions(&backend).await, before);
assert!(backend.state.lock().await.put_attempts.is_empty());
}
}
#[tokio::test]
async fn inventory_membership_does_not_authorize_an_unowned_or_wrong_kind_key() {
for (object, kind) in [
("unowned/data/object.parquet", ObjectCatalogBackupReferenceKind::DataFile),
("tables/table-id/data/wrong-role.parquet", ObjectCatalogBackupReferenceKind::ManifestFile),
] {
let backend = TestCatalogObjectBackend::default();
let mut backup = preparation_fixture();
backup
.references
.as_mut()
.unwrap()
.push(preparation_reference(object, kind, b"x"));
reseal_preparation_fixture(&mut backup);
save_preparation_fixture(&backend, &backup).await;
let before = object_versions(&backend).await;
let store = ObjectTableCatalogStore::new(backend.clone());
assert!(
store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, object, b"x".to_vec())
.await
.is_err()
);
assert_eq!(object_versions(&backend).await, before);
assert!(backend.state.lock().await.put_attempts.is_empty());
}
}
#[tokio::test]
async fn same_backup_preparation_preserves_a_pending_restore_barrier_and_plan() {
let backend = TestCatalogObjectBackend::default();
let backup = preparation_fixture();
save_preparation_fixture(&backend, &backup).await;
pending_preparation_controls(
&backend,
&backup,
ObjectCatalogRestoreStateKind::Applying,
ObjectCatalogRestoreStateKind::Prepared,
)
.await;
let before = system_versions(&backend).await;
let store = ObjectTableCatalogStore::new(backend.clone());
let payload = current_payload(&backup);
store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &payload.object, payload.data.clone())
.await
.unwrap();
assert_eq!(system_versions(&backend).await, before);
assert!(matches!(
store.ensure_no_active_restore(BUCKET).await,
Err(TableCatalogStoreError::Unavailable(_))
));
store
.restore_object_catalog_backup_with_id(BUCKET, &backup.backup_id, None, false, Some("prepare-recovery"))
.await
.unwrap();
store.ensure_no_active_restore(BUCKET).await.unwrap();
}
#[tokio::test]
async fn preparation_after_completed_restore_repairs_only_the_missing_payload() {
let backend = TestCatalogObjectBackend::default();
let backup = preparation_fixture();
save_preparation_fixture(&backend, &backup).await;
let payload = current_payload(&backup);
let store = ObjectTableCatalogStore::new(backend.clone());
store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &payload.object, payload.data.clone())
.await
.unwrap();
store
.restore_object_catalog_backup_with_id(BUCKET, &backup.backup_id, None, false, Some("completed-prepare"))
.await
.unwrap();
store.ensure_no_active_restore(BUCKET).await.unwrap();
let before = system_versions(&backend).await;
backend
.state
.lock()
.await
.objects
.remove(&(BUCKET.to_string(), payload.object.clone()));
let report = store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &payload.object, payload.data.clone())
.await
.unwrap();
assert!(matches!(report.status, TableCatalogBackupObjectPreparationStatus::Prepared));
assert_eq!(system_versions(&backend).await, before);
store.ensure_no_active_restore(BUCKET).await.unwrap();
}
#[tokio::test]
async fn view_metadata_preparation_requires_the_active_view_identity() {
for valid_identity in [true, false] {
let backend = TestCatalogObjectBackend::default();
let mut backup = preparation_fixture();
let namespace = Namespace::parse("sales").unwrap();
let view = IdentifierSegment::parse("summary").unwrap();
let path = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json");
let entry = ViewEntry {
version: TABLE_CATALOG_ENTRY_VERSION,
table_bucket: BUCKET.to_string(),
namespace: namespace.public_name(),
view: view.as_str().to_string(),
view_id: "view-id".to_string(),
view_uuid: "view-uuid".to_string(),
format: "ICEBERG_VIEW".to_string(),
format_version: 1,
warehouse_location: "s3://analytics/views/view-id".to_string(),
metadata_location: path.clone(),
version_token: "view-token".to_string(),
generation: 1,
state: TableCatalogEntryState::Active,
properties: BTreeMap::new(),
created_at: None,
updated_at: None,
};
let entry_data = serde_json::to_vec(&entry).unwrap();
backup.objects.push(ObjectCatalogBackupObject {
bucket: RUSTFS_META_BUCKET.to_string(),
object: TableCatalogObjectPaths::default().view_entry_path(BUCKET, &namespace, &view),
size_bytes: u64::try_from(entry_data.len()).unwrap(),
etag: Some("source-view-entry-etag".to_string()),
sha256: backup_sha256(&entry_data),
data: entry_data,
});
let metadata = serde_json::json!({
"format-version": 1, "view-uuid": if valid_identity { "view-uuid" } else { "foreign-view" },
"location": "s3://analytics/views/view-id", "current-schema-id": 0, "current-version-id": 1,
"schemas": [{"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "id", "required": true, "type": "long"}]}],
"versions": [{"version-id": 1, "timestamp-ms": 1, "schema-id": 0, "summary": {},
"representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}], "default-namespace": ["sales"]}],
"version-log": [{"timestamp-ms": 1, "version-id": 1}], "properties": {}
});
let data = serde_json::to_vec(&metadata).unwrap();
backup.references.as_mut().unwrap().push(preparation_reference(
&path,
ObjectCatalogBackupReferenceKind::ViewMetadata,
&data,
));
reseal_preparation_fixture(&mut backup);
save_preparation_fixture(&backend, &backup).await;
let before = object_versions(&backend).await;
let store = ObjectTableCatalogStore::new(backend.clone());
let result = store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &path, data)
.await;
if valid_identity {
assert!(result.is_ok());
assert_eq!(system_versions(&backend).await, before);
} else {
assert!(matches!(result, Err(TableCatalogStoreError::Invalid(_))));
assert_eq!(object_versions(&backend).await, before);
}
}
}
#[tokio::test]
async fn pending_wrong_backup_epoch_request_or_plan_cannot_prepare_an_object() {
for case in 0..4 {
let backend = TestCatalogObjectBackend::default();
let backup = preparation_fixture();
save_preparation_fixture(&backend, &backup).await;
let (state_path, intent_path) = pending_preparation_controls(
&backend,
&backup,
ObjectCatalogRestoreStateKind::Applying,
ObjectCatalogRestoreStateKind::Prepared,
)
.await;
let path = if case == 3 { &intent_path } else { &state_path };
let bytes = backend.read_object(RUSTFS_META_BUCKET, path).await.unwrap().unwrap().data;
let mut value: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
match case {
0 => value["backup_id"] = serde_json::json!("another-backup"),
1 => value["restore_epoch"] = serde_json::json!(Uuid::new_v4().to_string()),
2 => value["request_fingerprint"] = serde_json::json!("0".repeat(64)),
_ => {
value["writes"].as_array_mut().unwrap().pop();
}
}
backend
.seed_object(RUSTFS_META_BUCKET, path, serde_json::to_vec(&value).unwrap())
.await;
let before = object_versions(&backend).await;
let store = ObjectTableCatalogStore::new(backend.clone());
let payload = current_payload(&backup);
assert!(
store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &payload.object, payload.data.clone())
.await
.is_err()
);
assert_eq!(object_versions(&backend).await, before);
assert!(backend.state.lock().await.put_attempts.is_empty());
}
}
#[tokio::test]
async fn preparation_requires_prepared_history_before_using_its_different_warehouse_scope() {
let backend = TestCatalogObjectBackend::default();
let mut backup = preparation_fixture();
let current_index = backup.objects.iter().position(|object| object.bucket == BUCKET).unwrap();
let current = backup.objects[current_index].object.clone();
let history = current.replace("00001.metadata.json", "00000.metadata.json");
let mut metadata: serde_json::Value = serde_json::from_slice(&backup.objects[current_index].data).unwrap();
let mut historical = metadata.clone();
historical["location"] = serde_json::json!("s3://analytics/archive/table-id");
let historical = serde_json::to_vec(&historical).unwrap();
metadata["metadata-log"] = serde_json::json!([{"timestamp-ms": 0, "metadata-file": format!("s3://{BUCKET}/{history}")}]);
let current_data = serde_json::to_vec(&metadata).unwrap();
backup.objects[current_index].data = current_data.clone();
backup.objects[current_index].size_bytes = u64::try_from(current_data.len()).unwrap();
backup.objects[current_index].sha256 = backup_sha256(&current_data);
let old_data = "archive/table-id/data/part.parquet";
backup.references = Some(vec![
preparation_reference(&current, ObjectCatalogBackupReferenceKind::TableMetadata, &current_data),
preparation_reference(&history, ObjectCatalogBackupReferenceKind::TableMetadata, &historical),
preparation_reference(old_data, ObjectCatalogBackupReferenceKind::DataFile, b"x"),
]);
reseal_preparation_fixture(&mut backup);
save_preparation_fixture(&backend, &backup).await;
let store = ObjectTableCatalogStore::new(backend.clone());
let before = object_versions(&backend).await;
assert!(matches!(
store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, old_data, b"x".to_vec())
.await,
Err(TableCatalogStoreError::Unavailable(_))
));
assert_eq!(object_versions(&backend).await, before);
store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &history, historical)
.await
.unwrap();
let controls = system_versions(&backend).await;
store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, old_data, b"x".to_vec())
.await
.unwrap();
assert_eq!(system_versions(&backend).await, controls);
}
#[tokio::test]
async fn v1_artifact_does_not_gain_an_implicit_object_preparation_capability() {
let backend = TestCatalogObjectBackend::default();
let backup = original_fixture();
save_preparation_fixture(&backend, &backup).await;
let before = object_versions(&backend).await;
let payload = current_payload(&backup);
let store = ObjectTableCatalogStore::new(backend.clone());
assert!(matches!(
store
.prepare_object_catalog_backup_object(BUCKET, &backup.backup_id, &payload.object, payload.data.clone())
.await,
Err(TableCatalogStoreError::Unsupported(_))
));
assert_eq!(object_versions(&backend).await, before);
}
@@ -1,129 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
impl<B> ObjectTableCatalogStore<B>
where
B: TableCatalogObjectBackend,
{
/// Returns the saved v1 artifact bytes without reencoding or applying them.
///
/// This does not export external manifest/data/delete files. Pending restore
/// or legacy control state is rejected; cancellation leaves catalog state unchanged.
pub(crate) async fn export_object_catalog_backup(
&self,
table_bucket: &str,
backup_id: &str,
) -> TableCatalogStoreResult<Vec<u8>> {
let guards = self.acquire_object_backup_fence(table_bucket).await?;
let fence = Self::object_backup_mutation_fence(&guards);
fence.ensure_held()?;
self.ensure_no_active_restore(table_bucket).await?;
let path = self.paths.catalog_backup_path(table_bucket, backup_id);
let artifact = self
.backend
.read_object_limited(self.catalog_bucket(), &path, OBJECT_CATALOG_BACKUP_MAX_SIZE)
.await?
.ok_or_else(|| TableCatalogStoreError::NotFound(format!("catalog backup {backup_id}")))?;
let backup = self.decode_object_catalog_backup(table_bucket, &artifact.data)?;
if backup.backup_id != backup_id {
return Err(TableCatalogStoreError::Invalid(
"catalog backup identity does not match its request".to_string(),
));
}
let _validated_tables = self.validate_catalog_backup_metadata_identities(table_bucket, &backup.objects)?;
fence.ensure_held()?;
Ok(artifact.data)
}
/// Persists one validated versioned artifact without publishing catalog or metadata objects.
///
/// Only the artifact key is conditionally created. Exact retries keep its bytes,
/// ETag and original creation timestamp; conflicting content is never replaced.
/// A response lost after storage can be retried to confirm the saved artifact.
/// Referenced objects must be prepared and validated separately before restore.
pub(crate) async fn import_object_catalog_backup(
&self,
table_bucket: &str,
data: Vec<u8>,
) -> TableCatalogStoreResult<TableCatalogBackupReport> {
let backup = self.decode_object_catalog_backup(table_bucket, &data)?;
let _validated_tables = self.validate_catalog_backup_metadata_identities(table_bucket, &backup.objects)?;
let (verified_object_count, verified_bytes) = catalog_backup_verification_totals(&backup)?;
let path = self.paths.catalog_backup_path(table_bucket, &backup.backup_id);
let guards = self.acquire_object_backup_fence(table_bucket).await?;
let fence = Self::object_backup_mutation_fence(&guards);
fence.ensure_held()?;
self.ensure_no_active_restore(table_bucket).await?;
let existing = self
.backend
.read_object_limited(self.catalog_bucket(), &path, OBJECT_CATALOG_BACKUP_MAX_SIZE)
.await?;
let status = if let Some(existing) = existing {
if existing.data != data {
return Err(TableCatalogStoreError::Conflict(
"catalog backup identity is already occupied by different bytes".to_string(),
));
}
TableCatalogBackupStatus::AlreadyPresent
} else {
let result = self
.backend
.put_object_fenced(self.catalog_bucket(), &path, data.clone(), TableCatalogPutPrecondition::IfAbsent, &fence)
.await;
fence.ensure_held()?;
let saved = self
.backend
.read_object_limited(self.catalog_bucket(), &path, OBJECT_CATALOG_BACKUP_MAX_SIZE)
.await?;
match saved {
Some(saved) if saved.data == data => {
if result.is_ok() {
TableCatalogBackupStatus::Created
} else {
// The read proves an ambiguous write or identical concurrent
// import completed; it does not authorize another put.
TableCatalogBackupStatus::AlreadyPresent
}
}
Some(_) => {
return Err(TableCatalogStoreError::Conflict(
"catalog backup identity is already occupied by different bytes".to_string(),
));
}
None => {
result?;
return Err(TableCatalogStoreError::Unavailable("catalog backup disappeared after import".to_string()));
}
}
};
fence.ensure_held()?;
Ok(TableCatalogBackupReport {
table_bucket: table_bucket.to_string(),
backup_id: backup.backup_id,
status,
backup_path: path,
source_snapshot_etag: backup.source_catalog_etag.clone(),
source_snapshot_version: backup.version,
catalog_fingerprint: backup.source_catalog_etag,
object_count: backup.objects.len(),
verified_object_count,
verified_bytes,
maintenance_object_count: self.backup_maintenance_object_count(table_bucket, &backup.objects),
created_at: backup.created_at,
})
}
}
@@ -1,217 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::table_catalog::test_support::TestCatalogObjectBackend;
async fn inventory_backup_fixture() -> ObjectCatalogBackup {
let mut backup: ObjectCatalogBackup =
serde_json::from_slice(include_bytes!("../tests/fixtures/object-catalog-backup-v1.json"))
.expect("captured historical v1 artifact");
let backend = TestCatalogObjectBackend::default();
for object in &backup.objects {
backend.seed_object(&object.bucket, &object.object, object.data.clone()).await;
}
let store = ObjectTableCatalogStore::new(backend);
backup.references = Some(
store
.capture_object_catalog_backup_references(&backup.table_bucket, &backup.objects)
.await
.expect("valid source reference graph"),
);
backup.version = OBJECT_CATALOG_BACKUP_INVENTORY_VERSION;
reseal_inventory_backup(&mut backup);
backup
}
fn reseal_inventory_backup(backup: &mut ObjectCatalogBackup) {
backup.backup_id = catalog_backup_inventory_identity(
&backup.table_bucket,
&backup.source_catalog_etag,
&backup.objects,
backup.references.as_deref().expect("v2 inventory"),
)
.expect("inventory identity");
}
fn fixture_table(backup: &ObjectCatalogBackup) -> TableEntry {
serde_json::from_slice(
&backup
.objects
.iter()
.find(|object| object.object.ends_with(TABLE_ENTRY_FILE))
.expect("fixture table entry")
.data,
)
.expect("fixture table identity")
}
#[tokio::test]
async fn version_two_raw_transfer_and_inline_metadata_restore_preserve_the_iceberg_identity() {
let backup = inventory_backup_fixture().await;
let entry = fixture_table(&backup);
let bytes = serde_json::to_vec_pretty(&backup).expect("versioned artifact bytes");
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let imported = store
.import_object_catalog_backup(&backup.table_bucket, bytes.clone())
.await
.expect("import candidate");
assert_eq!(imported.source_snapshot_version, 2);
assert_eq!(imported.verified_object_count, backup.references.as_ref().expect("references").len());
assert_eq!(
store
.export_object_catalog_backup(&backup.table_bucket, &backup.backup_id)
.await
.expect("raw export"),
bytes
);
assert!(
store
.get_table_bucket(&backup.table_bucket)
.await
.expect("no publication during transfer")
.is_none()
);
let metadata_key =
table_catalog_object_key_from_location(&entry.table_bucket, &entry.metadata_location).expect("metadata key");
assert!(
!backend
.object_exists(&entry.table_bucket, &metadata_key)
.await
.expect("metadata initially absent")
);
store
.restore_object_catalog_backup_with_id(&backup.table_bucket, &backup.backup_id, None, false, Some("inventory-v2"))
.await
.expect("restore can publish its validated inline metadata without a separate prepare request");
assert_eq!(
store
.load_table(&entry.table_bucket, &entry.namespace, &entry.table)
.await
.expect("load restored table"),
Some(entry)
);
let payload = backup
.objects
.iter()
.find(|object| object.bucket == backup.table_bucket && object.object == metadata_key)
.expect("inline metadata payload");
assert_eq!(
backend
.read_object(&payload.bucket, &payload.object)
.await
.expect("read immutable restored metadata")
.expect("restored metadata")
.data,
payload.data
);
}
#[tokio::test]
async fn backup_versions_require_their_exact_inventory_shape_before_any_backend_access() {
let backup = inventory_backup_fixture().await;
for mode in [
"v1-with-inventory",
"v2-without-inventory",
"null-inventory",
"unknown-version",
] {
let mut value = serde_json::to_value(&backup).expect("artifact JSON");
match mode {
"v1-with-inventory" => value["version"] = serde_json::json!(1),
"v2-without-inventory" => {
value.as_object_mut().expect("object").remove("references");
}
"null-inventory" => value["references"] = serde_json::Value::Null,
"unknown-version" => value["version"] = serde_json::json!(99),
_ => unreachable!("fixed test case"),
}
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let error = store
.import_object_catalog_backup(&backup.table_bucket, serde_json::to_vec(&value).expect("test body"))
.await
.expect_err("unsupported shape cannot be persisted");
assert!(matches!(error, TableCatalogStoreError::Invalid(_)), "{mode}: {error:?}");
let state = backend.state.lock().await;
assert!(state.objects.is_empty(), "{mode}");
assert!(state.put_attempts.is_empty(), "{mode}");
assert!(state.delete_attempts.is_empty(), "{mode}");
}
}
#[tokio::test]
async fn a_self_consistent_candidate_with_an_extra_reference_cannot_publish_a_catalog() {
let mut backup = inventory_backup_fixture().await;
let entry = fixture_table(&backup);
let references = backup.references.as_mut().expect("inventory");
references.push(ObjectCatalogBackupReference {
bucket: backup.table_bucket.clone(),
object: format!("{}unreachable.parquet", table_warehouse_object_prefix(&entry).expect("warehouse prefix")),
kind: ObjectCatalogBackupReferenceKind::DataFile,
size_bytes: 1,
etag: Some("source-version".to_string()),
sha256: Some(backup_sha256(&[1])),
});
references.sort_by(|left, right| left.bucket.cmp(&right.bucket).then_with(|| left.object.cmp(&right.object)));
reseal_inventory_backup(&mut backup);
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
store
.import_object_catalog_backup(&backup.table_bucket, serde_json::to_vec(&backup).expect("candidate"))
.await
.expect("raw transfer may preserve a candidate before its external graph is prepared");
let (before_objects, before_puts, before_deletes) = {
let state = backend.state.lock().await;
(state.objects.clone(), state.put_attempts.clone(), state.delete_attempts.clone())
};
let error = store
.restore_object_catalog_backup_with_id(&backup.table_bucket, &backup.backup_id, None, false, Some("extra-reference"))
.await
.expect_err("a forged graph must not be published");
assert!(matches!(error, TableCatalogStoreError::Conflict(message) if message.contains("reference graph")));
let state = backend.state.lock().await;
assert_eq!(state.objects.len(), 1, "only the previously imported artifact may exist");
assert_eq!(state.objects, before_objects, "rejected restore changed persisted objects");
assert_eq!(state.put_attempts, before_puts, "rejected restore attempted a write");
assert_eq!(state.delete_attempts, before_deletes, "rejected restore attempted a deletion");
}
#[tokio::test]
async fn declared_reference_byte_overflow_is_rejected_before_saving_the_artifact() {
let mut backup = inventory_backup_fixture().await;
let entry = fixture_table(&backup);
let references = backup.references.as_mut().expect("inventory");
references.push(ObjectCatalogBackupReference {
bucket: backup.table_bucket.clone(),
object: format!("{}oversized.parquet", table_warehouse_object_prefix(&entry).expect("warehouse prefix")),
kind: ObjectCatalogBackupReferenceKind::DataFile,
size_bytes: u64::MAX,
etag: Some("large-source-version".to_string()),
sha256: None,
});
references.sort_by(|left, right| left.bucket.cmp(&right.bucket).then_with(|| left.object.cmp(&right.object)));
reseal_inventory_backup(&mut backup);
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let error = store
.import_object_catalog_backup(&backup.table_bucket, serde_json::to_vec(&backup).expect("overflow body"))
.await
.expect_err("overflow cannot leave an artifact stored before reporting failure");
assert!(matches!(error, TableCatalogStoreError::Invalid(message) if message.contains("byte total overflows")));
let state = backend.state.lock().await;
assert!(state.objects.is_empty());
assert!(state.put_attempts.is_empty());
}
@@ -72,9 +72,13 @@ async fn publication_deadline_registration_inventory_releases_fence_without_writ
.unwrap();
}
async fn commit_fixture() -> (TestCatalogObjectBackend, Arc<dyn TableCatalogStore>, TableCommitRequest) {
async fn commit_fixture(strong: bool) -> (TestCatalogObjectBackend, Arc<dyn TableCatalogStore>, TableCommitRequest) {
let backend = TestCatalogObjectBackend::default();
let store: Arc<dyn TableCatalogStore> = Arc::new(ObjectTableCatalogStore::new(backend.clone()));
let store: Arc<dyn TableCatalogStore> = if strong {
Arc::new(StrongTableCatalogStore::new(backend.clone()))
} else {
Arc::new(ObjectTableCatalogStore::new(backend.clone()))
};
let namespace = Namespace::parse("sales").unwrap();
let table = IdentifierSegment::parse("orders").unwrap();
let current = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json");
@@ -112,7 +116,7 @@ async fn commit_fixture() -> (TestCatalogObjectBackend, Arc<dyn TableCatalogStor
// so only the object backing is exercised here.
#[tokio::test(start_paused = true)]
async fn publication_deadline_commit_metadata_read_does_not_publish_object_backing() {
let (backend, store, request) = commit_fixture().await;
let (backend, store, request) = commit_fixture(false).await;
let pause = backend
.pause_before_next_read("analytics", &request.new_metadata_location)
.await;
@@ -155,24 +159,30 @@ async fn publication_deadline_commit_metadata_read_does_not_publish_object_backi
}
#[tokio::test(start_paused = true)]
async fn publication_deadline_create_finishes_after_first_write_on_object_backing() {
let (backend, store, _) = commit_fixture().await;
let namespace = Namespace::parse("sales").unwrap();
let candidate = registration_entry(&namespace, "candidate");
let paths = TableCatalogObjectPaths::default();
let first_write = paths.warehouse_index_entry_path("analytics", "tables/id-candidate/");
let pause = backend.pause_next_put(RUSTFS_META_BUCKET, &first_write).await;
let creating = {
let store = store.clone();
tokio::spawn(async move { store.create_table(candidate).await })
};
pause.wait_started().await;
tokio::time::advance(publication_preparation_timeout() + StdDuration::from_millis(1)).await;
tokio::task::yield_now().await;
assert!(!creating.is_finished(), "deadline must not cancel a possible durable write");
pause.release();
creating.await.unwrap().unwrap();
assert!(store.load_table("analytics", "sales", "candidate").await.unwrap().is_some());
async fn publication_deadline_create_finishes_after_first_write_on_either_backing() {
for strong in [false, true] {
let (backend, store, _) = commit_fixture(strong).await;
let namespace = Namespace::parse("sales").unwrap();
let candidate = registration_entry(&namespace, "candidate");
let paths = TableCatalogObjectPaths::default();
let first_write = if strong {
StrongTableCatalogStore::<TestCatalogObjectBackend>::snapshot_object_path()
} else {
paths.warehouse_index_entry_path("analytics", "tables/id-candidate/")
};
let pause = backend.pause_next_put(RUSTFS_META_BUCKET, &first_write).await;
let creating = {
let store = store.clone();
tokio::spawn(async move { store.create_table(candidate).await })
};
pause.wait_started().await;
tokio::time::advance(publication_preparation_timeout() + StdDuration::from_millis(1)).await;
tokio::task::yield_now().await;
assert!(!creating.is_finished(), "deadline must not cancel a possible durable write");
pause.release();
creating.await.unwrap().unwrap();
assert!(store.load_table("analytics", "sales", "candidate").await.unwrap().is_some());
}
}
#[tokio::test(start_paused = true)]
@@ -231,7 +241,7 @@ async fn publication_deadline_completed_recovery_cannot_exempt_a_new_transaction
// so only the object backing is exercised here.
#[tokio::test(start_paused = true)]
async fn publication_deadline_create_queue_wait_cannot_restart_after_admission() {
let (backend, store, _) = commit_fixture().await;
let (backend, store, _) = commit_fixture(false).await;
let key = default_table_bucket_publication_lock_path();
let blocker = backend.acquire_read_lock("analytics", &key).await.unwrap();
let attempts = backend.write_lock_acquisition_count("analytics", &key).await;
File diff suppressed because it is too large Load Diff
+77 -119
View File
@@ -410,8 +410,8 @@ pub(crate) fn manifest_avro_bytes_with_entry_sequences(files: &[ManifestEntryWit
pub(crate) struct TestCatalogObjectBackend {
pub(crate) state: Arc<tokio::sync::Mutex<TestCatalogObjectState>>,
pub(crate) locks: TestCatalogObjectLocks,
pub(crate) strong_runtime: Option<StrongTableCatalogRuntime>,
pub(crate) registration_cache: TableCatalogRegistrationCache,
pub(crate) cleanup_publication: Option<Arc<TestCatalogCleanupPublication>>,
// One-shot, path-keyed injection knobs from the admin handler tests'
// former TestTableCatalogObjectBackend (backlog#1837 PR2): each fires
// once for the named object and clears itself, mirroring the original
@@ -425,7 +425,6 @@ pub(crate) struct TestCatalogObjectBackend {
pub(crate) fail_write_lock_path: Arc<tokio::sync::Mutex<Option<(String, TableCatalogStoreError)>>>,
pub(crate) lock_attempts: Arc<tokio::sync::Mutex<Vec<(String, String)>>>,
pub(crate) reject_reads_while_write_locked: bool,
pub(crate) serialize_object_reads: bool,
/// Content-addressed (sha256) etags instead of the store fake's counter.
/// The admin handler tests observe an object's etag and expect rewriting
/// identical bytes to reproduce it, so their fixtures set this.
@@ -452,6 +451,79 @@ impl TestCatalogObjectPause {
}
}
#[derive(Clone)]
pub(crate) struct BlockingObjectPublication {
backend: TestCatalogObjectBackend,
object: String,
started: Arc<tokio::sync::Notify>,
guard: Arc<parking_lot::Mutex<Option<TableCatalogLockGuard>>>,
}
impl BlockingObjectPublication {
pub(crate) fn new(backend: TestCatalogObjectBackend, object: impl Into<String>) -> Self {
Self {
backend,
object: object.into(),
started: Arc::new(tokio::sync::Notify::new()),
guard: Arc::new(parking_lot::Mutex::new(None)),
}
}
pub(crate) async fn wait_started(&self) {
self.started.notified().await;
}
}
#[derive(Default)]
pub(crate) struct UnserializedTestPublication;
#[async_trait::async_trait]
impl TableCommitPublication for UnserializedTestPublication {
async fn begin_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn prepare(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> {
Ok(())
}
fn holds_table_bucket(&self, _table_bucket: &str) -> bool {
true
}
fn holds_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> bool {
true
}
fn complete(&self) {}
}
#[async_trait::async_trait]
impl TableCommitPublication for BlockingObjectPublication {
async fn begin_table_bucket(&self, _table_bucket: &str) -> TableCatalogStoreResult<()> {
Ok(())
}
async fn prepare(&self, table_bucket: &str, _namespace: &str, _table: &str) -> TableCatalogStoreResult<()> {
self.started.notify_one();
let guard = self.backend.acquire_read_lock(table_bucket, &self.object).await?;
*self.guard.lock() = Some(guard);
Ok(())
}
fn holds_table_bucket(&self, _table_bucket: &str) -> bool {
true
}
fn holds_table(&self, _table_bucket: &str, _namespace: &str, _table: &str) -> bool {
self.guard.lock().is_some()
}
fn complete(&self) {
drop(self.guard.lock().take());
}
}
#[derive(Debug, PartialEq, Eq)]
pub(crate) struct TestCatalogWorkerContext {
pub(crate) access_key: Option<String>,
@@ -469,8 +541,6 @@ pub(crate) struct TestCatalogObjectState {
pub(crate) pause_read_attempts: BTreeMap<(String, String), BTreeMap<usize, TestCatalogObjectPause>>,
pub(crate) read_attempts: BTreeMap<(String, String), usize>,
pub(crate) read_limits: Vec<((String, String), usize)>,
pub(crate) list_page_overrides: BTreeMap<(String, String), std::collections::VecDeque<TableCatalogObjectListPage>>,
pub(crate) list_page_pauses: BTreeMap<(String, String), std::collections::VecDeque<TestCatalogObjectPause>>,
pub(crate) fail_put_attempts: BTreeMap<(String, String), BTreeSet<usize>>,
pub(crate) fail_after_put_attempts: BTreeMap<(String, String), BTreeSet<usize>>,
pub(crate) pause_put_attempts: BTreeMap<(String, String), BTreeMap<usize, TestCatalogObjectPause>>,
@@ -491,7 +561,7 @@ pub(crate) struct TestCatalogObjectState {
pub(crate) next_etag: u64,
}
#[derive(Clone, Debug, PartialEq, Eq)]
#[derive(Clone)]
pub(crate) struct TestCatalogObjectRecord {
pub(crate) body_revision: Option<Uuid>,
pub(crate) data: Vec<u8>,
@@ -499,38 +569,7 @@ pub(crate) struct TestCatalogObjectRecord {
pub(crate) mod_time: Option<OffsetDateTime>,
}
pub(crate) struct TestCatalogCleanupPublication {
table_bucket: String,
guards: Mutex<Vec<TableCatalogLockGuard>>,
}
impl TestCatalogCleanupPublication {
fn is_held(&self) -> bool {
let guards = self.guards.lock().expect("cleanup publication guards");
!guards.is_empty() && guards.iter().all(|guard| !guard.is_lock_lost())
}
}
impl TestCatalogObjectBackend {
pub(crate) async fn with_held_cleanup_publication(&self, table_bucket: &str) -> TableCatalogStoreResult<Self> {
let paths = TableCatalogObjectPaths::default();
let guards = vec![
self.acquire_read_lock(RUSTFS_META_BUCKET, &paths.catalog_backup_global_fence_lock_path())
.await?,
self.acquire_read_lock(RUSTFS_META_BUCKET, &paths.catalog_backup_fence_lock_path(table_bucket))
.await?,
self.acquire_write_lock(table_bucket, &default_table_bucket_publication_lock_path())
.await?,
];
Ok(Self {
cleanup_publication: Some(Arc::new(TestCatalogCleanupPublication {
table_bucket: table_bucket.to_string(),
guards: Mutex::new(guards),
})),
..self.clone()
})
}
pub(crate) async fn seed_object(&self, bucket: &str, object: &str, data: Vec<u8>) {
self.seed_object_with_mod_time(bucket, object, data, Some(OffsetDateTime::UNIX_EPOCH))
.await;
@@ -621,18 +660,6 @@ impl TestCatalogObjectBackend {
state.fail_read_attempts.entry(key).or_default().insert(next_attempt);
}
pub(crate) async fn pause_next_list_page(&self, bucket: &str, prefix: &str) -> TestCatalogObjectPause {
let pause = TestCatalogObjectPause::default();
self.state
.lock()
.await
.list_page_pauses
.entry((bucket.to_string(), prefix.to_string()))
.or_default()
.push_back(pause.clone());
pause
}
pub(crate) async fn pause_next_read(&self, bucket: &str, object: &str) -> TestCatalogObjectPause {
let mut state = self.state.lock().await;
let key = (bucket.to_string(), object.to_string());
@@ -954,37 +981,11 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend {
Some(self.registration_cache.clone())
}
fn table_bucket_commit_publication_is_held(&self, table_bucket: &str) -> bool {
self.cleanup_publication
.as_ref()
.is_some_and(|publication| publication.table_bucket == table_bucket && publication.is_held())
}
fn catalog_migration_read_permit_status(&self) -> Option<bool> {
self.cleanup_publication.as_ref().map(|publication| publication.is_held())
}
fn table_commit_object_mutation_fence(&self) -> Option<TableCatalogObjectMutationFence> {
self.cleanup_publication.as_ref().and_then(|publication| {
let guards = publication.guards.lock().expect("cleanup publication guards");
(!guards.is_empty()).then(|| {
TableCatalogObjectMutationFence::from_signals(guards.iter().map(TableCatalogLockGuard::lock_lost_signal))
})
})
}
fn complete_table_commit_publication(&self) {
if let Some(publication) = &self.cleanup_publication {
publication.guards.lock().expect("cleanup publication guards").clear();
}
fn strong_catalog_runtime(&self) -> Option<StrongTableCatalogRuntime> {
self.strong_runtime.clone()
}
async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<Option<TableCatalogObject>> {
let _read_guard = if self.serialize_object_reads {
Some(self.acquire_read_lock(bucket, object).await?)
} else {
None
};
let mut missing_read_object_path = self.missing_read_object_path.lock().await;
if missing_read_object_path.as_deref() == Some(object) {
missing_read_object_path.take();
@@ -1071,7 +1072,6 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend {
async fn read_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<Option<TableCatalogObject>> {
let mut backend = self.clone();
backend.reject_reads_while_write_locked = false;
backend.serialize_object_reads = false;
backend.read_object(bucket, object).await
}
@@ -1224,48 +1224,6 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend {
.collect())
}
async fn list_objects_page(
&self,
bucket: &str,
prefix: &str,
start_after: Option<&str>,
limit: NonZeroUsize,
) -> TableCatalogStoreResult<TableCatalogObjectListPage> {
let pause = self
.state
.lock()
.await
.list_page_pauses
.get_mut(&(bucket.to_string(), prefix.to_string()))
.and_then(std::collections::VecDeque::pop_front);
if let Some(pause) = pause {
pause.started.notify_one();
pause.release.notified().await;
}
let injected = {
let mut state = self.state.lock().await;
let page = state
.list_page_overrides
.get_mut(&(bucket.to_string(), prefix.to_string()))
.and_then(std::collections::VecDeque::pop_front);
if page.is_some() {
state.list_calls += 1;
}
page
};
if let Some(page) = injected {
return Ok(page);
}
let mut objects = self.list_objects(bucket, prefix).await?;
objects.sort();
if let Some(start_after) = start_after {
objects.retain(|object| object.as_str() > start_after);
}
let is_truncated = objects.len() > limit.get();
objects.truncate(limit.get());
Ok(TableCatalogObjectListPage { objects, is_truncated })
}
async fn list_objects_for_cleanup(
&self,
bucket: &str,
File diff suppressed because it is too large Load Diff
@@ -1,227 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
#[derive(Debug, PartialEq, Eq)]
struct BackupCaptureObjectFingerprint {
size_bytes: usize,
sha256: String,
etag: String,
mod_time: Option<OffsetDateTime>,
}
#[derive(Debug, PartialEq, Eq)]
struct BackupCapturePersistedState {
objects: BTreeMap<(String, String), BackupCaptureObjectFingerprint>,
puts: BTreeMap<(String, String), usize>,
deletes: BTreeMap<(String, String), usize>,
}
async fn backup_capture_persisted_state(backend: &TestCatalogObjectBackend) -> BackupCapturePersistedState {
let state = backend.state.lock().await;
// Retain fingerprints rather than another copy of the bounded large fixtures.
let objects = state
.objects
.iter()
.map(|(key, object)| {
let sha256 =
hex_simd::encode_to_string(<sha2::Sha256 as sha2::Digest>::digest(&object.data), hex_simd::AsciiCase::Lower);
(
key.clone(),
BackupCaptureObjectFingerprint {
size_bytes: object.data.len(),
sha256,
etag: object.etag.clone(),
mod_time: object.mod_time,
},
)
})
.collect();
BackupCapturePersistedState {
objects,
puts: state.put_attempts.clone(),
deletes: state.delete_attempts.clone(),
}
}
#[tokio::test]
async fn backup_capture_rejects_malformed_pages_before_publishing_any_object() {
for scenario in [
"foreign-prefix",
"duplicate-object",
"reverse-order",
"oversized",
"empty-truncated",
"repeated-cursor",
] {
let (backend, _, _) = backup_fixture().await;
let paths = TableCatalogObjectPaths::default();
let bucket_path = paths.table_bucket_entry_path("analytics");
let (parent, _) = bucket_path.rsplit_once('/').expect("catalog bucket entry has a parent");
let prefix = format!("{parent}/");
let before = backup_capture_persisted_state(&backend).await;
let existing = before
.objects
.keys()
.filter(|(bucket, object)| bucket == RUSTFS_META_BUCKET && object.starts_with(&prefix))
.map(|(_, object)| object.clone())
.collect::<Vec<_>>();
assert!(existing.len() >= 2, "the healthy fixture provides two real sorted catalog objects");
let first = existing[0].clone();
let second = existing[1].clone();
let pages = match scenario {
"foreign-prefix" => vec![TableCatalogObjectListPage {
objects: vec![paths.table_bucket_entry_path("unrelated")],
is_truncated: false,
}],
"duplicate-object" => vec![TableCatalogObjectListPage {
objects: vec![first.clone(), first],
is_truncated: false,
}],
"reverse-order" => vec![TableCatalogObjectListPage {
objects: vec![second, first],
is_truncated: false,
}],
"oversized" => vec![TableCatalogObjectListPage {
objects: (0..=TABLE_CATALOG_LIST_MAX_KEYS)
.map(|index| format!("{prefix}oversized/{index:04}.json"))
.collect(),
is_truncated: false,
}],
"empty-truncated" => vec![TableCatalogObjectListPage {
objects: Vec::new(),
is_truncated: true,
}],
"repeated-cursor" => vec![
TableCatalogObjectListPage {
objects: vec![first.clone()],
is_truncated: true,
},
TableCatalogObjectListPage {
objects: vec![first],
is_truncated: false,
},
],
_ => unreachable!(),
};
let page_key = (RUSTFS_META_BUCKET.to_string(), prefix);
backend
.state
.lock()
.await
.list_page_overrides
.entry(page_key.clone())
.or_default()
.extend(pages);
let error = ObjectTableCatalogStore::new(backend.clone())
.create_object_catalog_backup("analytics", None)
.await
.expect_err("malformed backend pages must not become a backup");
let expected = if matches!(scenario, "oversized" | "empty-truncated") {
"invalid catalog backup listing page"
} else {
"catalog backup listing escaped its prefix or did not advance"
};
assert!(
matches!(&error, TableCatalogStoreError::Invalid(message) if message == expected),
"{scenario}: {error}"
);
assert_eq!(
backup_capture_persisted_state(&backend).await,
before,
"{scenario} mutated an object, ETag or PUT/DELETE attempt count"
);
assert!(
backend
.state
.lock()
.await
.list_page_overrides
.get(&page_key)
.expect("injected page queue")
.is_empty(),
"{scenario} did not consume the injected listing boundary"
);
}
}
#[tokio::test]
async fn backup_capture_stops_reading_when_the_encoded_payload_budget_is_exhausted() {
let (backend, _, _) = backup_fixture().await;
let bucket_path = TableCatalogObjectPaths::default().table_bucket_entry_path("analytics");
let (parent, _) = bucket_path.rsplit_once('/').expect("catalog bucket entry has a parent");
let prefix = format!("{parent}/");
let first_path = format!("{prefix}zz-backup-budget-0.bin");
let second_path = format!("{prefix}zz-backup-budget-1.bin");
let tail_path = format!("{prefix}zz-backup-budget-2-tail.bin");
// Byte 120 occupies four JSON bytes per array element. Each payload is just
// below 8 MiB to keep its individual encoding below 32 MiB; the existing
// valid catalog plus both payloads exceeds the shared 64 MiB budget.
let payload_bytes = 8 * 1024 * 1024 - 256;
backend
.seed_object(RUSTFS_META_BUCKET, &first_path, vec![b'x'; payload_bytes])
.await;
backend
.seed_object(RUSTFS_META_BUCKET, &second_path, vec![b'x'; payload_bytes])
.await;
backend
.seed_object(RUSTFS_META_BUCKET, &tail_path, b"unreachable tail".to_vec())
.await;
backend.fail_next_read(RUSTFS_META_BUCKET, &tail_path).await;
let before = backup_capture_persisted_state(&backend).await;
let before_reads = backend.state.lock().await.read_attempts.clone();
let error = ObjectTableCatalogStore::new(backend.clone())
.create_object_catalog_backup("analytics", None)
.await
.expect_err("incremental payload accounting must stop before the unread tail");
assert!(
matches!(
&error,
TableCatalogStoreError::Invalid(message)
if message == "catalog backup payload exceeds its encoded byte budget"
),
"expected the incremental byte budget, not the injected read or metadata validation error: {error}"
);
assert_eq!(
backup_capture_persisted_state(&backend).await,
before,
"budget rejection changed persistent object bytes, ETags or write attempts"
);
let state = backend.state.lock().await;
for object in [&first_path, &second_path] {
let key = (RUSTFS_META_BUCKET.to_string(), object.clone());
assert_eq!(
state.read_attempts.get(&key).copied().unwrap_or_default(),
before_reads.get(&key).copied().unwrap_or_default() + 1,
"the collector did not measure the expected bounded payload: {object}"
);
}
let tail_key = (RUSTFS_META_BUCKET.to_string(), tail_path);
let tail_reads = before_reads.get(&tail_key).copied().unwrap_or_default();
assert_eq!(
state.read_attempts.get(&tail_key).copied().unwrap_or_default(),
tail_reads,
"the collector read beyond the exhausted byte budget"
);
assert!(
state
.fail_read_attempts
.get(&tail_key)
.is_some_and(|attempts| attempts.contains(&(tail_reads + 1))),
"the unread tail fault was unexpectedly consumed"
);
}
@@ -1,713 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use sha2::Digest;
async fn backup_recovery_commit_request(
backend: &TestCatalogObjectBackend,
store: &ObjectTableCatalogStore<TestCatalogObjectBackend>,
commit_id: &str,
metadata_file: &str,
) -> TableCommitRequest {
let current = store.load_table("analytics", "sales", "orders").await.unwrap().unwrap();
let namespace = Namespace::parse("sales").unwrap();
let table = IdentifierSegment::parse("orders").unwrap();
let metadata_path = default_table_metadata_file_path(&namespace, &table, metadata_file);
let mut metadata = table_metadata_json_for_backup("analytics", &current.table_id, &current.table_uuid);
metadata["last-updated-ms"] = serde_json::json!(current.generation + 2);
metadata["properties"]["recovery-fixture-commit"] = serde_json::json!(commit_id);
validate_supported_table_metadata(&metadata).expect("commit fixture must contain valid Iceberg metadata");
backend
.seed_object("analytics", &metadata_path, serde_json::to_vec(&metadata).unwrap())
.await;
TableCommitRequest {
table_bucket: "analytics".to_string(),
namespace: "sales".to_string(),
table: "orders".to_string(),
commit_id: commit_id.to_string(),
idempotency_key: Some(format!("request-{commit_id}")),
operation: "append".to_string(),
expected_version_token: current.version_token,
expected_metadata_location: current.metadata_location,
new_metadata_location: metadata_path,
requirements: Vec::new(),
writer: Some("backup-recovery-fixture".to_string()),
}
}
fn assert_backup_recovery_conflict(error: &TableCatalogStoreError, expected_state: &str) {
assert!(
matches!(error, TableCatalogStoreError::Conflict(message)
if message.contains("commit recovery required") && message.contains(expected_state)),
"{error}"
);
}
async fn assert_unhealthy_backup_has_no_writes(
backend: &TestCatalogObjectBackend,
store: &ObjectTableCatalogStore<TestCatalogObjectBackend>,
expected_state: &str,
) {
let diagnostics = store.diagnose_table_catalog("analytics", "sales", "orders", 0).await.unwrap();
assert_eq!(
diagnostics.current_metadata_status,
TableMetadataPointerStatus::Valid,
"the regression must reach recovery policy instead of failing metadata validation"
);
let before = maintenance_test_object_versions(backend).await;
let attempts = backend.state.lock().await.put_attempts.clone();
let error = tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, store.create_object_catalog_backup("analytics", None))
.await
.expect("backup rejects unstable recovery without waiting for another lock")
.unwrap_err();
assert_backup_recovery_conflict(&error, expected_state);
assert_eq!(
maintenance_test_object_versions(backend).await,
before,
"rejected backup changed persisted objects"
);
assert_eq!(backend.state.lock().await.put_attempts, attempts, "rejected backup attempted a write");
}
#[tokio::test]
async fn backup_rejects_pre_cas_staging_until_the_same_commit_retry_converges() {
// This public fixture already sealed a healthy backup with real metadata.
let (backend, baseline, _) = backup_fixture().await;
let store = ObjectTableCatalogStore::new(backend.clone());
let original = store.load_table("analytics", "sales", "orders").await.unwrap().unwrap();
let request = backup_recovery_commit_request(&backend, &store, "before-cas", "00002.metadata.json").await;
let table_path = TableCatalogObjectPaths::default().table_entry_path(
"analytics",
&Namespace::parse("sales").unwrap(),
&IdentifierSegment::parse("orders").unwrap(),
);
backend.fail_next_put(RUSTFS_META_BUCKET, &table_path).await;
assert!(matches!(
store.commit_table(request.clone()).await,
Err(TableCatalogStoreError::Internal(_))
));
assert_eq!(store.load_table("analytics", "sales", "orders").await.unwrap(), Some(original.clone()));
let recovery = store
.plan_table_commit_recovery("analytics", "sales", "orders")
.await
.unwrap();
assert_eq!(recovery.staged_before_table_update_count, 1);
assert_eq!(recovery.commits[0].recovery_state, TableCommitRecoveryState::StagedBeforeTableUpdate);
assert_unhealthy_backup_has_no_writes(&backend, &store, "StagedBeforeTableUpdate").await;
let committed = store
.commit_table(request.clone())
.await
.expect("same commit id safely retries its original staging");
assert_eq!(committed.table.table_uuid, original.table_uuid);
assert_eq!(committed.table.warehouse_location, original.warehouse_location);
assert_eq!(committed.table.generation, original.generation + 1);
assert_eq!(committed.table.metadata_location, request.new_metadata_location);
assert_eq!(committed.commit_log.status, CommitLogStatus::Committed);
let replay = store.commit_table(request).await.expect("same-id committed replay");
assert_eq!(replay.table, committed.table, "replay advanced the catalog a second time");
assert_eq!(replay.commit_log, committed.commit_log);
let backup = store
.create_object_catalog_backup("analytics", None)
.await
.expect("healthy retry is sealable");
assert_ne!(backup.catalog_fingerprint, baseline.catalog_fingerprint);
let diagnostics = store.diagnose_table_catalog("analytics", "sales", "orders", 0).await.unwrap();
assert_eq!(diagnostics.recovery_status, TableCatalogRecoveryStatus::Healthy);
}
#[tokio::test]
async fn backup_rejects_post_cas_log_or_index_finalization_until_explicit_recovery() {
for fail_index in [false, true] {
let (backend, _, _) = backup_fixture().await;
let store = ObjectTableCatalogStore::new(backend.clone());
let request = backup_recovery_commit_request(&backend, &store, "after-cas", "00002.metadata.json").await;
let paths = TableCatalogObjectPaths::default();
let failed_path = if fail_index {
paths.commit_idempotency_entry_path("analytics", "table-id", request.idempotency_key.as_deref().unwrap())
} else {
paths.commit_log_entry_path("analytics", "table-id", &request.commit_id)
};
// First PUT is staging; the second is the durable finalization boundary.
backend.fail_put_attempt(RUSTFS_META_BUCKET, &failed_path, 2).await;
let committed = store
.commit_table(request.clone())
.await
.expect("a successful pointer CAS remains a committed response despite finalization failure");
let recovery = store
.plan_table_commit_recovery("analytics", "sales", "orders")
.await
.unwrap();
let expected = if fail_index {
assert_eq!(recovery.idempotency_repair_required_count, 1);
"IdempotencyIndexRepairRequired"
} else {
assert_eq!(recovery.finalization_required_count, 1);
"FinalizationRequired"
};
assert_unhealthy_backup_has_no_writes(&backend, &store, expected).await;
let table_path = paths.table_entry_path(
"analytics",
&Namespace::parse("sales").unwrap(),
&IdentifierSegment::parse("orders").unwrap(),
);
let table_before = backend.read_object(RUSTFS_META_BUCKET, &table_path).await.unwrap().unwrap();
let metadata_before = backend
.read_object("analytics", &request.new_metadata_location)
.await
.unwrap()
.unwrap();
let repaired = store
.recover_table_commits("analytics", "sales", "orders")
.await
.expect("explicit recovery repairs receipts");
assert_eq!(repaired.finalized_count, 1);
assert_eq!(repaired.finalization_required_count, 0);
assert_eq!(repaired.idempotency_repair_required_count, 0);
assert_eq!(repaired.staged_before_table_update_count, 0);
assert_eq!(repaired.manual_review_count, 0);
assert_eq!(store.load_table("analytics", "sales", "orders").await.unwrap(), Some(committed.table));
assert_eq!(
backend.read_object(RUSTFS_META_BUCKET, &table_path).await.unwrap(),
Some(table_before),
"receipt repair rewrote the table pointer, token, generation, bytes or ETag"
);
assert_eq!(
backend
.read_object("analytics", &request.new_metadata_location)
.await
.unwrap(),
Some(metadata_before)
);
store
.create_object_catalog_backup("analytics", None)
.await
.expect("recovered catalog can be sealed");
let diagnostics = store.diagnose_table_catalog("analytics", "sales", "orders", 0).await.unwrap();
assert_eq!(diagnostics.recovery_status, TableCatalogRecoveryStatus::Healthy);
}
}
// The historical v1 object field order is part of its identity encoding.
// This fixture encoder deliberately omits the v2 references field entirely.
#[derive(serde::Serialize, serde::Deserialize)]
struct RecoveryLegacyBackupObject {
bucket: String,
object: String,
size_bytes: u64,
etag: Option<String>,
sha256: String,
data: Vec<u8>,
}
fn backup_recovery_digest(data: &[u8]) -> String {
hex_simd::encode_to_string(sha2::Sha256::digest(data), hex_simd::AsciiCase::Lower)
}
fn reseal_recovery_legacy_artifact(mut artifact: serde_json::Value) -> (String, Vec<u8>) {
artifact.as_object_mut().unwrap().remove("references");
artifact["version"] = serde_json::json!(1);
let mut objects: Vec<RecoveryLegacyBackupObject> = serde_json::from_value(artifact["objects"].clone()).unwrap();
for object in &mut objects {
object.size_bytes = u64::try_from(object.data.len()).unwrap();
object.sha256 = backup_recovery_digest(&object.data);
}
objects.sort_by(|left, right| (&left.bucket, &left.object).cmp(&(&right.bucket, &right.object)));
let fingerprint = backup_recovery_digest(
&serde_json::to_vec(
&objects
.iter()
.map(|object| (&object.bucket, &object.object, &object.sha256))
.collect::<Vec<_>>(),
)
.unwrap(),
);
let id = backup_recovery_digest(&serde_json::to_vec(&(1u16, "analytics", &fingerprint, &objects)).unwrap());
artifact["source_catalog_etag"] = serde_json::json!(fingerprint);
artifact["backup_id"] = serde_json::json!(id);
artifact["objects"] = serde_json::to_value(objects).unwrap();
(id, serde_json::to_vec(&artifact).unwrap())
}
#[tokio::test]
async fn raw_legacy_v1_import_keeps_pending_commit_state_but_restore_rejects_it_before_any_writes() {
let (source, baseline, _) = backup_fixture().await;
let source_store = ObjectTableCatalogStore::new(source.clone());
let baseline_bytes = source_store
.export_object_catalog_backup("analytics", &baseline.backup_id)
.await
.unwrap();
let mut artifact: serde_json::Value = serde_json::from_slice(&baseline_bytes).unwrap();
let request = backup_recovery_commit_request(&source, &source_store, "legacy-pending", "00002.metadata.json").await;
let paths = TableCatalogObjectPaths::default();
let table_path = paths.table_entry_path(
"analytics",
&Namespace::parse("sales").unwrap(),
&IdentifierSegment::parse("orders").unwrap(),
);
source.fail_next_put(RUSTFS_META_BUCKET, &table_path).await;
assert!(source_store.commit_table(request.clone()).await.is_err());
for path in [
paths.commit_log_entry_path("analytics", "table-id", &request.commit_id),
paths.commit_idempotency_entry_path("analytics", "table-id", request.idempotency_key.as_deref().unwrap()),
] {
let stored = source.read_object(RUSTFS_META_BUCKET, &path).await.unwrap().unwrap();
artifact["objects"].as_array_mut().unwrap().push(serde_json::json!({
"bucket": RUSTFS_META_BUCKET,
"object": path,
"size_bytes": stored.data.len(),
"etag": stored.etag,
"sha256": backup_recovery_digest(&stored.data),
"data": stored.data,
}));
}
let (backup_id, bytes) = reseal_recovery_legacy_artifact(artifact);
let target = TestCatalogObjectBackend::default();
let target_store = ObjectTableCatalogStore::new(target.clone());
let imported = target_store
.import_object_catalog_backup("analytics", bytes.clone())
.await
.expect("raw import stores a valid legacy candidate without publishing its unsafe recovery state");
assert_eq!(imported.backup_id, backup_id);
assert_eq!(imported.source_snapshot_version, 1);
assert_eq!(
target_store
.export_object_catalog_backup("analytics", &backup_id)
.await
.unwrap(),
bytes
);
assert_eq!(
maintenance_test_object_versions(&target).await.len(),
1,
"import published more than its artifact"
);
let before = maintenance_test_object_versions(&target).await;
let attempts = target.state.lock().await.put_attempts.clone();
let error = target_store
.restore_object_catalog_backup_with_id("analytics", &backup_id, None, false, Some("legacy-recovery"))
.await
.unwrap_err();
assert_backup_recovery_conflict(&error, "StagedBeforeTableUpdate");
assert_eq!(maintenance_test_object_versions(&target).await, before);
assert_eq!(target.state.lock().await.put_attempts, attempts);
assert!(
!target
.object_exists(RUSTFS_META_BUCKET, &paths.catalog_restore_state_path("analytics"))
.await
.unwrap()
);
assert!(
!target
.object_exists(RUSTFS_META_BUCKET, &paths.catalog_restore_intent_path("analytics", "legacy-recovery"))
.await
.unwrap()
);
assert!(target_store.get_table_bucket("analytics").await.unwrap().is_none());
assert_eq!(
target_store
.export_object_catalog_backup("analytics", &backup_id)
.await
.unwrap(),
bytes,
"failed restore silently repaired or rewrote the historical artifact"
);
}
#[tokio::test]
async fn allow_replace_cannot_overwrite_a_target_with_pending_commit_recovery() {
let (source, _, _) = backup_fixture().await;
let source_store = ObjectTableCatalogStore::new(source.clone());
let source_request = backup_recovery_commit_request(&source, &source_store, "source-stable", "00002.metadata.json").await;
source_store.commit_table(source_request).await.unwrap();
let backup = source_store.create_object_catalog_backup("analytics", None).await.unwrap();
let bytes = source_store
.export_object_catalog_backup("analytics", &backup.backup_id)
.await
.unwrap();
let (target, _, _) = backup_fixture().await;
let target_store = ObjectTableCatalogStore::new(target.clone());
let target_request = backup_recovery_commit_request(&target, &target_store, "target-pending", "00003.metadata.json").await;
let paths = TableCatalogObjectPaths::default();
let table_path = paths.table_entry_path(
"analytics",
&Namespace::parse("sales").unwrap(),
&IdentifierSegment::parse("orders").unwrap(),
);
target.fail_next_put(RUSTFS_META_BUCKET, &table_path).await;
assert!(target_store.commit_table(target_request).await.is_err());
target_store.import_object_catalog_backup("analytics", bytes).await.unwrap();
let before = maintenance_test_object_versions(&target).await;
let attempts = target.state.lock().await.put_attempts.clone();
let error = target_store
.restore_object_catalog_backup_with_id("analytics", &backup.backup_id, None, true, Some("replace-pending-target"))
.await
.unwrap_err();
assert_backup_recovery_conflict(&error, "StagedBeforeTableUpdate");
assert_eq!(maintenance_test_object_versions(&target).await, before);
assert_eq!(target.state.lock().await.put_attempts, attempts);
assert!(
!target
.object_exists(RUSTFS_META_BUCKET, &paths.catalog_restore_state_path("analytics"))
.await
.unwrap()
);
assert!(
!target
.object_exists(
RUSTFS_META_BUCKET,
&paths.catalog_restore_intent_path("analytics", "replace-pending-target")
)
.await
.unwrap()
);
}
#[tokio::test]
async fn restored_receipts_replay_and_completed_restore_never_rolls_back_a_later_commit() {
let (source, _, _) = backup_fixture().await;
let source_store = ObjectTableCatalogStore::new(source.clone());
let request = backup_recovery_commit_request(&source, &source_store, "portable-receipt", "00002.metadata.json").await;
let original = source_store.commit_table(request.clone()).await.unwrap();
let backup = source_store.create_object_catalog_backup("analytics", None).await.unwrap();
let bytes = source_store
.export_object_catalog_backup("analytics", &backup.backup_id)
.await
.unwrap();
let target = TestCatalogObjectBackend::default();
let target_store = ObjectTableCatalogStore::new(target.clone());
target_store.import_object_catalog_backup("analytics", bytes).await.unwrap();
let mut receipt = target_store
.restore_object_catalog_backup_with_id("analytics", &backup.backup_id, None, false, Some("restore-portable-receipts"))
.await
.expect("healthy committed receipts restore into an isolated backend");
assert_eq!(receipt.status, TableCatalogRestoreStatus::Restored);
assert_eq!(
target_store.load_table("analytics", "sales", "orders").await.unwrap(),
Some(original.table.clone())
);
let replay = target_store
.commit_table(request)
.await
.expect("restored same-id request returns its committed result");
assert_eq!(replay.table, original.table);
assert_eq!(replay.commit_log, original.commit_log);
assert_eq!(
target_store
.get_commit_by_idempotency_key("analytics", "table-id", "request-portable-receipt")
.await
.unwrap(),
Some(original.commit_log)
);
let later_request =
backup_recovery_commit_request(&target, &target_store, "later-pending-finalization", "00003.metadata.json").await;
let later_log = TableCatalogObjectPaths::default().commit_log_entry_path("analytics", "table-id", &later_request.commit_id);
target.fail_put_attempt(RUSTFS_META_BUCKET, &later_log, 2).await;
let later = target_store.commit_table(later_request).await.unwrap();
assert_eq!(later.table.generation, replay.table.generation + 1);
assert_eq!(
target_store
.plan_table_commit_recovery("analytics", "sales", "orders")
.await
.unwrap()
.finalization_required_count,
1
);
let before = maintenance_test_object_versions(&target).await;
let attempts = target.state.lock().await.put_attempts.clone();
receipt.status = TableCatalogRestoreStatus::AlreadyRestored;
let repeated = target_store
.restore_object_catalog_backup_with_id("analytics", &backup.backup_id, None, false, Some("restore-portable-receipts"))
.await
.expect("completed receipt wins before evaluating later target recovery state");
assert_eq!(repeated, receipt);
assert_eq!(target_store.load_table("analytics", "sales", "orders").await.unwrap(), Some(later.table));
assert_eq!(
maintenance_test_object_versions(&target).await,
before,
"completed restore rolled back later catalog or receipts"
);
assert_eq!(
target.state.lock().await.put_attempts,
attempts,
"completed restore tried to publish again"
);
assert_eq!(
target_store
.plan_table_commit_recovery("analytics", "sales", "orders")
.await
.unwrap()
.finalization_required_count,
1
);
}
struct SupersededBackupFixture {
backend: TestCatalogObjectBackend,
loser: TableCommitRequest,
winner: TableCommitRequest,
committed: TableCommitResult,
}
async fn superseded_backup_fixture() -> SupersededBackupFixture {
let (backend, _, _) = backup_fixture().await;
let store = ObjectTableCatalogStore::new(backend.clone());
let loser = backup_recovery_commit_request(&backend, &store, "losing-backup-cas", "00002.metadata.json").await;
let table_path = TableCatalogObjectPaths::default().table_entry_path(
"analytics",
&Namespace::parse("sales").unwrap(),
&IdentifierSegment::parse("orders").unwrap(),
);
backend.fail_next_put(RUSTFS_META_BUCKET, &table_path).await;
assert!(matches!(
store.commit_table(loser.clone()).await,
Err(TableCatalogStoreError::Internal(_))
));
let winner = backup_recovery_commit_request(&backend, &store, "winning-backup-cas", "00003.metadata.json").await;
let committed = store.commit_table(winner.clone()).await.unwrap();
let report = store.recover_table_commits("analytics", "sales", "orders").await.unwrap();
assert_eq!(report.manual_review_count, 0);
assert_eq!(report.finalized_count, 0);
assert_eq!(
report
.commits
.iter()
.find(|entry| entry.commit_id == loser.commit_id)
.unwrap()
.recovery_state,
TableCommitRecoveryState::Superseded
);
SupersededBackupFixture {
backend,
loser,
winner,
committed,
}
}
#[tokio::test]
async fn backup_seals_proven_losers_without_changing_their_staged_receipts_or_the_winner() {
let SupersededBackupFixture {
backend,
loser,
winner,
committed,
} = superseded_backup_fixture().await;
let store = ObjectTableCatalogStore::new(backend.clone());
let paths = TableCatalogObjectPaths::default();
let protected = [
paths.table_entry_path(
"analytics",
&Namespace::parse("sales").unwrap(),
&IdentifierSegment::parse("orders").unwrap(),
),
paths.commit_log_entry_path("analytics", &committed.table.table_id, &loser.commit_id),
paths.commit_idempotency_entry_path("analytics", &committed.table.table_id, loser.idempotency_key.as_deref().unwrap()),
paths.commit_log_entry_path("analytics", &committed.table.table_id, &winner.commit_id),
paths.commit_idempotency_entry_path("analytics", &committed.table.table_id, winner.idempotency_key.as_deref().unwrap()),
];
let mut before = Vec::new();
for path in &protected {
before.push(backend.read_object(RUSTFS_META_BUCKET, path).await.unwrap().unwrap());
}
let backup = store
.create_object_catalog_backup("analytics", None)
.await
.expect("proven losers are sealable");
for (path, original) in protected.iter().zip(&before) {
assert_eq!(
backend.read_object(RUSTFS_META_BUCKET, path).await.unwrap().as_ref(),
Some(original),
"backup rewrote {path}"
);
}
let bytes = store
.export_object_catalog_backup("analytics", &backup.backup_id)
.await
.unwrap();
let artifact: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
let archived = artifact["objects"]
.as_array()
.unwrap()
.iter()
.find(|object| object["object"].as_str() == Some(protected[1].as_str()))
.unwrap();
let staged_bytes: Vec<u8> = serde_json::from_value(archived["data"].clone()).unwrap();
assert_eq!(staged_bytes, before[1].data);
let staged: CommitLogEntry = serde_json::from_slice(&staged_bytes).unwrap();
assert_eq!(staged.status, CommitLogStatus::Staged);
assert_eq!(staged.idempotency_key, loser.idempotency_key);
assert!(matches!(store.commit_table(loser).await, Err(TableCatalogStoreError::Conflict(_))));
assert_eq!(store.load_table("analytics", "sales", "orders").await.unwrap(), Some(committed.table));
}
#[tokio::test]
async fn isolated_restore_retains_loser_proof_retry_conflicts_and_later_commits() {
let SupersededBackupFixture {
backend: source,
loser,
winner,
committed,
} = superseded_backup_fixture().await;
let source_store = ObjectTableCatalogStore::new(source.clone());
let backup = source_store.create_object_catalog_backup("analytics", None).await.unwrap();
let bytes = source_store
.export_object_catalog_backup("analytics", &backup.backup_id)
.await
.unwrap();
let source_before = maintenance_test_object_versions(&source).await;
let loser_path =
TableCatalogObjectPaths::default().commit_log_entry_path("analytics", &committed.table.table_id, &loser.commit_id);
let original_loser = source.read_object(RUSTFS_META_BUCKET, &loser_path).await.unwrap().unwrap();
let target = TestCatalogObjectBackend::default();
let target_store = ObjectTableCatalogStore::new(target.clone());
target_store
.import_object_catalog_backup("analytics", bytes.clone())
.await
.unwrap();
let mut receipt = target_store
.restore_object_catalog_backup_with_id("analytics", &backup.backup_id, None, false, Some("restore-superseded-proof"))
.await
.unwrap();
assert_eq!(receipt.status, TableCatalogRestoreStatus::Restored);
assert_eq!(
target_store.load_table("analytics", "sales", "orders").await.unwrap(),
Some(committed.table.clone())
);
let restored_loser = target.read_object(RUSTFS_META_BUCKET, &loser_path).await.unwrap().unwrap();
assert_eq!(restored_loser.data, original_loser.data);
let restarted = ObjectTableCatalogStore::new(target.clone());
let recovery = restarted.recover_table_commits("analytics", "sales", "orders").await.unwrap();
assert_eq!(recovery.manual_review_count, 0);
assert_eq!(recovery.finalized_count, 0);
assert_eq!(
recovery
.commits
.iter()
.find(|entry| entry.commit_id == loser.commit_id)
.unwrap()
.recovery_state,
TableCommitRecoveryState::Superseded
);
assert!(matches!(
restarted.commit_table(loser.clone()).await,
Err(TableCatalogStoreError::Conflict(_))
));
let replay = restarted.commit_table(winner).await.unwrap();
assert_eq!(replay.table, committed.table);
assert_eq!(replay.commit_log, committed.commit_log);
assert_eq!(
target.read_object(RUSTFS_META_BUCKET, &loser_path).await.unwrap(),
Some(restored_loser.clone())
);
let indexed = restarted
.get_commit_by_idempotency_key("analytics", &committed.table.table_id, loser.idempotency_key.as_deref().unwrap())
.await
.unwrap()
.unwrap();
assert_eq!(indexed.status, CommitLogStatus::Staged);
assert_eq!(indexed.commit_id, loser.commit_id);
let next_request = backup_recovery_commit_request(&target, &restarted, "reloaded-after-restore", "00004.metadata.json").await;
let next = restarted.commit_table(next_request.clone()).await.unwrap();
assert_eq!(next.table.generation, committed.table.generation + 1);
assert_eq!(restarted.commit_table(next_request).await.unwrap().table, next.table);
let versions = maintenance_test_object_versions(&target).await;
receipt.status = TableCatalogRestoreStatus::AlreadyRestored;
assert_eq!(
restarted
.restore_object_catalog_backup_with_id("analytics", &backup.backup_id, None, false, Some("restore-superseded-proof"))
.await
.unwrap(),
receipt
);
assert_eq!(
maintenance_test_object_versions(&target).await,
versions,
"Completed replay republished old state"
);
let restarted = ObjectTableCatalogStore::new(target.clone());
assert_eq!(restarted.load_table("analytics", "sales", "orders").await.unwrap(), Some(next.table));
assert_eq!(
restarted
.diagnose_table_catalog("analytics", "sales", "orders", 0)
.await
.unwrap()
.recovery_status,
TableCatalogRecoveryStatus::Healthy
);
assert_eq!(target.read_object(RUSTFS_META_BUCKET, &loser_path).await.unwrap(), Some(restored_loser));
restarted
.create_object_catalog_backup("analytics", None)
.await
.expect("historical winner proof remains sealable after a later commit");
assert_eq!(
restarted
.export_object_catalog_backup("analytics", &backup.backup_id)
.await
.unwrap(),
bytes
);
assert_eq!(
maintenance_test_object_versions(&source).await,
source_before,
"isolated target changed the source"
);
}
#[tokio::test]
async fn raw_v1_restore_rejects_a_loser_whose_winner_proof_is_missing_without_writes() {
let SupersededBackupFixture {
backend: source,
winner,
committed,
..
} = superseded_backup_fixture().await;
let source_store = ObjectTableCatalogStore::new(source);
let backup = source_store.create_object_catalog_backup("analytics", None).await.unwrap();
let bytes = source_store
.export_object_catalog_backup("analytics", &backup.backup_id)
.await
.unwrap();
let mut artifact: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
let paths = TableCatalogObjectPaths::default();
let winner_log = paths.commit_log_entry_path("analytics", &committed.table.table_id, &winner.commit_id);
let winner_index =
paths.commit_idempotency_entry_path("analytics", &committed.table.table_id, winner.idempotency_key.as_deref().unwrap());
artifact["objects"].as_array_mut().unwrap().retain(|object| {
object["object"].as_str() != Some(winner_log.as_str()) && object["object"].as_str() != Some(winner_index.as_str())
});
let (backup_id, malformed) = reseal_recovery_legacy_artifact(artifact);
let target = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(target.clone());
store
.import_object_catalog_backup("analytics", malformed.clone())
.await
.unwrap();
let before = maintenance_test_object_versions(&target).await;
let attempts = target.state.lock().await.put_attempts.clone();
let error = store
.restore_object_catalog_backup_with_id("analytics", &backup_id, None, false, Some("missing-loser-proof"))
.await
.unwrap_err();
assert_backup_recovery_conflict(&error, "ManualReview");
assert_eq!(maintenance_test_object_versions(&target).await, before);
assert_eq!(target.state.lock().await.put_attempts, attempts);
assert!(store.get_table_bucket("analytics").await.unwrap().is_none());
assert_eq!(store.export_object_catalog_backup("analytics", &backup_id).await.unwrap(), malformed);
}
@@ -1,169 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
#[tokio::test]
async fn backup_waiting_for_publication_does_not_block_authorization_lookup() {
for scenario in ["new-bucket", "cold-index", "missing-entry", "warm-index"] {
let bucket = "analytics";
let backend = if scenario == "new-bucket" {
let backend = TestCatalogObjectBackend::default();
ObjectTableCatalogStore::new(backend.clone())
.put_table_bucket(test_bucket_entry(bucket))
.await
.expect("enable a new empty table bucket");
backend
} else {
backup_fixture().await.0
};
let store = ObjectTableCatalogStore::new(backend.clone());
let paths = TableCatalogObjectPaths::default();
let index_path = paths.warehouse_index_entry_path(bucket, "tables/table-id/");
if matches!(scenario, "cold-index" | "missing-entry") {
backend
.delete_object(RUSTFS_META_BUCKET, &index_path)
.await
.expect("remove index entry");
}
if scenario == "cold-index" {
backend
.delete_object(RUSTFS_META_BUCKET, &paths.warehouse_index_state_path(bucket))
.await
.expect("remove ready marker");
}
let publication_path = default_table_bucket_publication_lock_path();
let reader_guard = backend
.acquire_read_lock(bucket, &publication_path)
.await
.expect("retain S3 publication read guard");
let publication_attempts = backend.write_lock_acquisition_count(bucket, &publication_path).await;
let backup_store = store.clone();
let backup = tokio::spawn(async move { backup_store.create_object_catalog_backup(bucket, None).await });
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async {
while backend.write_lock_acquisition_count(bucket, &publication_path).await == publication_attempts {
tokio::task::yield_now().await;
}
})
.await
.expect("backup owns its catalog fences and is waiting for publication");
assert!(!backup.is_finished(), "backup must wait for the S3 request");
let objects_before = backend.state.lock().await.objects.clone();
let catalog_lock_attempts = backend.lock_attempts.lock().await.clone();
let resource = tokio::time::timeout(
TABLE_CATALOG_TEST_TIMEOUT,
store.resolve_table_data_plane_resource_for_authorization(bucket, "tables/table-id/data/part.parquet"),
)
.await
.expect("S3 lookup must not wait for the backup fence")
.expect("authorization lookup succeeds");
if scenario == "new-bucket" {
assert!(resource.is_none());
} else {
let resource = resource.expect("the table retains data-plane protection");
assert_eq!(resource.table_id, "table-id");
assert_eq!(resource.namespace, "sales");
assert_eq!(resource.table, "orders");
}
assert_eq!(
backend.state.lock().await.objects,
objects_before,
"authorization must not mutate or repair the catalog"
);
assert_eq!(
*backend.lock_attempts.lock().await,
catalog_lock_attempts,
"authorization must not acquire catalog locks behind publication"
);
assert!(!backup.is_finished());
drop(reader_guard);
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, backup)
.await
.expect("backup progresses after S3 completion")
.expect("backup task joins")
.expect("backup remains valid");
if scenario != "new-bucket" {
let repaired = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/part.parquet")
.await
.expect("normal catalog lookup can still repair indexes")
.expect("the table still resolves");
assert_eq!(repaired.table_id, "table-id");
assert!(store.warehouse_index_ready(bucket).await.expect("index readiness"));
assert!(
backend
.read_object(RUSTFS_META_BUCKET, &index_path)
.await
.expect("read repaired index")
.is_some()
);
}
}
}
#[tokio::test]
async fn cold_authorization_lookup_keeps_deleted_table_protection_without_repair() {
let (backend, _, _) = backup_fixture().await;
let store = ObjectTableCatalogStore::new(backend.clone());
let bucket = "analytics";
store.drop_table(bucket, "sales", "orders").await.expect("drop the table");
backend
.delete_object(RUSTFS_META_BUCKET, &store.paths.warehouse_index_state_path(bucket))
.await
.expect("make the warehouse index cold");
let before = backend.state.lock().await.objects.clone();
let resource = store
.resolve_table_data_plane_resource_for_authorization(bucket, "tables/table-id/data/retained.parquet")
.await
.expect("read deleted table scope")
.expect("the tombstone must retain data-plane protection");
assert_eq!(resource.table_id, "table-id");
assert_eq!(backend.state.lock().await.objects, before);
let repaired = table_data_plane_resource_for_object(&store, bucket, "tables/table-id/data/retained.parquet")
.await
.expect("normal lookup may repair the cold index");
assert_eq!(repaired, Some(resource));
}
#[tokio::test]
async fn cold_authorization_lookup_rejects_overlapping_tables_behind_partial_index() {
let (backend, _, _) = backup_fixture().await;
let store = ObjectTableCatalogStore::new(backend.clone());
let namespace = Namespace::parse("sales").expect("namespace");
let second = IdentifierSegment::parse("other_orders").expect("second table");
let mut entry = test_table_entry(
"analytics",
&namespace,
&second,
default_table_metadata_file_path(&namespace, &second, "00001.metadata.json"),
);
entry.table_id = "other-id".to_string();
entry.warehouse_location = "s3://analytics/tables/table-id/".to_string();
backend
.seed_object(
RUSTFS_META_BUCKET,
&store.paths.table_entry_path("analytics", &namespace, &second),
serde_json::to_vec(&entry).expect("encode inconsistent catalog entry"),
)
.await;
backend
.delete_object(RUSTFS_META_BUCKET, &store.paths.warehouse_index_state_path("analytics"))
.await
.expect("clear the incomplete index readiness marker");
let before = backend.state.lock().await.objects.clone();
assert_matches!(
store.resolve_table_data_plane_resource_for_authorization("analytics", "tables/table-id/data/part.parquet").await,
Err(TableCatalogStoreError::Invalid(message)) if message.contains("overlapping active table warehouse prefixes")
);
assert_eq!(backend.state.lock().await.objects, before);
}
@@ -1,475 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
#[derive(Debug, PartialEq)]
struct BackupMaintenanceStoreWrites {
objects: BTreeMap<(String, String), TableCatalogObject>,
puts: BTreeMap<(String, String), usize>,
deletes: BTreeMap<(String, String), usize>,
}
async fn backup_maintenance_store_writes(backend: &TestCatalogObjectBackend) -> BackupMaintenanceStoreWrites {
let objects = maintenance_test_object_versions(backend).await;
let state = backend.state.lock().await;
BackupMaintenanceStoreWrites {
objects,
puts: state.put_attempts.clone(),
deletes: state.delete_attempts.clone(),
}
}
fn backup_maintenance_store_report_paths(report: &TableMetadataMaintenanceReport) -> [String; 3] {
let paths = TableCatalogObjectPaths::default();
let namespace = Namespace::parse(&report.job.namespace).expect("report namespace");
let table = IdentifierSegment::parse(&report.job.table).expect("report table");
[
paths.table_maintenance_job_path(&report.job.table_bucket, &namespace, &table, &report.job.table_id, &report.job.job_id),
paths.table_maintenance_latest_job_path(&report.job.table_bucket, &namespace, &table, &report.job.table_id),
paths.table_maintenance_current_job_path(&report.job.table_bucket, &namespace, &table, &report.job.table_id),
]
}
async fn backup_maintenance_store_report(
store: &ObjectTableCatalogStore<TestCatalogObjectBackend>,
job_id: &str,
status: TableMetadataMaintenanceJobStatus,
) -> TableMetadataMaintenanceReport {
let mut report = store
.plan_table_metadata_maintenance("analytics", "sales", "orders", 0)
.await
.expect("plan against valid Iceberg metadata through the store");
report.job.job_id = job_id.to_string();
report.job.status = status;
report.job.worker_id = Some("retained-worker".to_string());
report.job.lease_id = "retained-lease".to_string();
report.job.attempt = 3;
let expired = maintenance_timestamp(OffsetDateTime::UNIX_EPOCH);
report.job.started_at = Some(expired.clone());
report.job.heartbeat_at = Some(expired);
report.job.finished_at = if matches!(
report.job.status,
TableMetadataMaintenanceJobStatus::Queued | TableMetadataMaintenanceJobStatus::Running
) {
None
} else {
Some(maintenance_timestamp(OffsetDateTime::UNIX_EPOCH + Duration::seconds(1)))
};
report
}
fn assert_backup_maintenance_store_recovery<T>(result: TableCatalogStoreResult<T>) {
match result {
Err(TableCatalogStoreError::Conflict(message)) => {
assert!(message.contains("maintenance recovery required"), "{message}");
}
Err(error) => panic!("expected the maintenance recovery boundary, received {error}"),
Ok(_) => panic!("an authoritative maintenance job was accepted by backup or restore"),
}
}
#[tokio::test]
async fn backup_create_fences_expired_authoritative_jobs_without_writing_or_repairing_aliases() {
for status in [
TableMetadataMaintenanceJobStatus::Queued,
TableMetadataMaintenanceJobStatus::Running,
] {
let (backend, _, _) = backup_fixture().await;
let store = ObjectTableCatalogStore::new(backend.clone());
let mut report = backup_maintenance_store_report(&store, "expired-authoritative-job", status).await;
store
.put_table_metadata_maintenance_report(&report)
.await
.expect("publish named, latest and current authority");
let before = backup_maintenance_store_writes(&backend).await;
assert_backup_maintenance_store_recovery(
ObjectTableCatalogStore::new(backend.clone())
.create_object_catalog_backup("analytics", None)
.await,
);
assert_eq!(
backup_maintenance_store_writes(&backend).await,
before,
"a backup attempt must neither expire a lease nor repair or rewrite reports"
);
report.job.status = TableMetadataMaintenanceJobStatus::Successful;
report.job.finished_at = Some(maintenance_timestamp(OffsetDateTime::UNIX_EPOCH + Duration::seconds(1)));
store
.put_table_metadata_maintenance_report(&report)
.await
.expect("publish all terminal aliases through the store");
let backup = ObjectTableCatalogStore::new(backend.clone())
.create_object_catalog_backup("analytics", None)
.await
.expect("the same valid metadata is acceptable after maintenance reaches a stable terminal state");
assert_eq!(backup.maintenance_object_count, backup_maintenance_store_report_paths(&report).len());
}
}
// Match the saved v1 field order, as the existing transfer fixtures do, so a
// negative authority fixture reaches the maintenance gate with valid digests.
#[derive(serde::Deserialize, serde::Serialize)]
struct BackupMaintenanceStoreArtifactObject {
bucket: String,
object: String,
size_bytes: u64,
etag: Option<String>,
sha256: String,
data: Vec<u8>,
}
#[derive(serde::Deserialize, serde::Serialize)]
struct BackupMaintenanceStoreArtifact {
version: u16,
backup_id: String,
table_bucket: String,
created_at: String,
source_catalog_etag: String,
objects: Vec<BackupMaintenanceStoreArtifactObject>,
}
fn backup_maintenance_store_digest(bytes: &[u8]) -> String {
hex_simd::encode_to_string(<sha2::Sha256 as sha2::Digest>::digest(bytes), hex_simd::AsciiCase::Lower)
}
fn reseal_backup_maintenance_store_artifact(artifact: &mut BackupMaintenanceStoreArtifact) -> Vec<u8> {
for object in &mut artifact.objects {
object.size_bytes = u64::try_from(object.data.len()).expect("fixture object length fits u64");
object.sha256 = backup_maintenance_store_digest(&object.data);
}
artifact
.objects
.sort_by(|left, right| (&left.bucket, &left.object).cmp(&(&right.bucket, &right.object)));
artifact.source_catalog_etag = backup_maintenance_store_digest(
&serde_json::to_vec(
&artifact
.objects
.iter()
.map(|object| (&object.bucket, &object.object, &object.sha256))
.collect::<Vec<_>>(),
)
.expect("encode fixture fingerprint"),
);
artifact.backup_id = backup_maintenance_store_digest(
&serde_json::to_vec(&(artifact.version, &artifact.table_bucket, &artifact.source_catalog_etag, &artifact.objects))
.expect("encode fixture identity"),
);
serde_json::to_vec(artifact).expect("encode the saved v1 artifact")
}
#[tokio::test]
async fn unsafe_v1_maintenance_artifact_transfers_raw_but_restore_rejects_it_before_any_write() {
let (source, baseline, _) = backup_fixture().await;
let source_store = ObjectTableCatalogStore::new(source.clone());
let report =
backup_maintenance_store_report(&source_store, "legacy-running-job", TableMetadataMaintenanceJobStatus::Running).await;
source_store
.put_table_metadata_maintenance_report(&report)
.await
.expect("publish the legacy running report through the store");
let saved = source_store
.export_object_catalog_backup("analytics", &baseline.backup_id)
.await
.expect("export the valid metadata baseline");
let mut legacy: serde_json::Value = serde_json::from_slice(&saved).expect("decode healthy v2 baseline");
assert_eq!(legacy["version"], 2);
let references = legacy
.as_object_mut()
.unwrap()
.remove("references")
.expect("the v2 backup carries an inventory");
assert!(references.is_array());
// The fixture's complete current metadata remains embedded in objects.
// Emit the historical v1 shape and recompute its v1 identity explicitly.
legacy["version"] = serde_json::json!(1);
let mut artifact: BackupMaintenanceStoreArtifact = serde_json::from_value(legacy).expect("construct v1 baseline");
let report_paths = backup_maintenance_store_report_paths(&report);
for path in &report_paths {
let object = source.read_object(RUSTFS_META_BUCKET, path).await.unwrap().unwrap();
artifact.objects.push(BackupMaintenanceStoreArtifactObject {
bucket: RUSTFS_META_BUCKET.to_string(),
object: path.clone(),
size_bytes: u64::try_from(object.data.len()).unwrap(),
etag: object.etag,
sha256: backup_maintenance_store_digest(&object.data),
data: object.data,
});
}
let bytes = reseal_backup_maintenance_store_artifact(&mut artifact);
let target = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(target.clone());
let imported = store
.import_object_catalog_backup("analytics", bytes.clone())
.await
.expect("raw import validates the v1 format without granting execution authority");
assert_eq!(imported.maintenance_object_count, report_paths.len());
assert_eq!(imported.source_snapshot_version, 1);
assert_eq!(
store
.export_object_catalog_backup("analytics", &imported.backup_id)
.await
.unwrap(),
bytes,
"export must preserve the exact unsafe historical artifact bytes"
);
let before = backup_maintenance_store_writes(&target).await;
assert_eq!(before.objects.len(), 1, "import must only publish the artifact");
assert_backup_maintenance_store_recovery(
ObjectTableCatalogStore::new(target.clone())
.restore_object_catalog_backup_with_id("analytics", &imported.backup_id, None, false, Some("unsafe-v1-maintenance"))
.await,
);
assert_eq!(
backup_maintenance_store_writes(&target).await,
before,
"rejected restore must not write an intent, barrier, catalog, report or metadata object"
);
}
#[tokio::test]
async fn backup_restores_terminal_maintenance_bytes_and_replays_its_original_count_after_later_jobs() {
let (source, _, _) = backup_fixture().await;
let source_store = ObjectTableCatalogStore::new(source.clone());
let config = TableMaintenanceConfig {
background_enabled: true,
quarantine_enabled: true,
quarantine_retention_seconds: 86_400,
max_retry_attempts: 2,
..Default::default()
};
source_store
.put_table_bucket_maintenance_config("analytics", config.clone())
.await
.expect("persist bucket maintenance config");
source_store
.put_table_maintenance_config("analytics", "sales", "orders", config.clone())
.await
.expect("persist table maintenance config");
let mut terminal =
backup_maintenance_store_report(&source_store, "terminal-quarantined-job", TableMetadataMaintenanceJobStatus::Failed)
.await;
terminal.job.failure_reason = Some("cleanup candidates remain quarantined".to_string());
terminal.job.quarantine_enabled = true;
terminal.job.quarantine_retention_seconds = 86_400;
terminal.job.quarantined_object_count = 2;
terminal.audit_events.push(TableMaintenanceAuditEvent {
timestamp: terminal.job.finished_at.clone().expect("terminal timestamp"),
actor: TableMaintenanceAuditActor::Worker,
action: TableMaintenanceAuditAction::WorkerFailed,
reason: terminal.job.failure_reason.clone(),
before_status: Some(TableMetadataMaintenanceJobStatus::Running),
after_status: Some(TableMetadataMaintenanceJobStatus::Failed),
before_quarantined_object_count: Some(0),
after_quarantined_object_count: Some(2),
recommended_actions: Vec::new(),
});
source_store
.put_table_metadata_maintenance_report(&terminal)
.await
.expect("publish a terminal quarantine report with its audit event");
let expected_report = source_store
.get_table_metadata_maintenance_report("analytics", "sales", "orders", &terminal.job.job_id)
.await
.unwrap()
.unwrap();
let entry = source_store
.load_table("analytics", "sales", "orders")
.await
.unwrap()
.unwrap();
let paths = TableCatalogObjectPaths::default();
let mut sidecar_paths = backup_maintenance_store_report_paths(&terminal).to_vec();
sidecar_paths.push(paths.table_bucket_maintenance_config_path("analytics"));
sidecar_paths.push(paths.table_maintenance_config_path(
"analytics",
&Namespace::parse("sales").unwrap(),
&IdentifierSegment::parse("orders").unwrap(),
&entry.table_id,
));
let mut expected_bytes = BTreeMap::new();
for path in &sidecar_paths {
let object = source.read_object(RUSTFS_META_BUCKET, path).await.unwrap().unwrap();
assert!(expected_bytes.insert(path.clone(), object.data).is_none());
}
assert_eq!(expected_bytes.len(), 5, "two configs and three report aliases are stored");
let backup = source_store.create_object_catalog_backup("analytics", None).await.unwrap();
assert_eq!(backup.maintenance_object_count, expected_bytes.len());
let raw = source_store
.export_object_catalog_backup("analytics", &backup.backup_id)
.await
.unwrap();
let artifact: serde_json::Value = serde_json::from_slice(&raw).unwrap();
assert_eq!(artifact["version"], 2);
assert!(artifact["references"].is_array(), "new backups include a v2 inventory");
let target = TestCatalogObjectBackend::default();
let target_store = ObjectTableCatalogStore::new(target.clone());
let imported = target_store.import_object_catalog_backup("analytics", raw).await.unwrap();
assert_eq!(imported.maintenance_object_count, expected_bytes.len());
let restore_id = "terminal-maintenance-roundtrip";
let mut first = target_store
.restore_object_catalog_backup_with_id("analytics", &backup.backup_id, None, false, Some(restore_id))
.await
.expect("restore stable maintenance through the isolated target store");
assert_eq!(first.maintenance_object_count, expected_bytes.len());
assert_eq!(first.object_count, backup.object_count);
assert_eq!(target_store.load_table("analytics", "sales", "orders").await.unwrap(), Some(entry));
assert_eq!(
target_store
.get_table_maintenance_config("analytics", "sales", "orders")
.await
.unwrap(),
config
);
for (path, bytes) in &expected_bytes {
let object = target.read_object(RUSTFS_META_BUCKET, path).await.unwrap().unwrap();
assert_eq!(&object.data, bytes, "restoring rewrote config, quarantine or audit bytes: {path}");
}
assert_eq!(
target_store
.get_table_metadata_maintenance_report("analytics", "sales", "orders", &terminal.job.job_id)
.await
.unwrap(),
Some(expected_report)
);
let mut later_config = config;
later_config.max_retry_attempts = 4;
target_store
.put_table_maintenance_config("analytics", "sales", "orders", later_config.clone())
.await
.expect("a later normal config update is allowed after restore");
let later =
backup_maintenance_store_report(&target_store, "later-successful-job", TableMetadataMaintenanceJobStatus::Successful)
.await;
target_store
.put_table_metadata_maintenance_report(&later)
.await
.expect("publish a later normal job through the restored store");
let later_report = target_store
.get_table_metadata_maintenance_report("analytics", "sales", "orders", &later.job.job_id)
.await
.unwrap()
.unwrap();
let before_replay = backup_maintenance_store_writes(&target).await;
let replay = ObjectTableCatalogStore::new(target.clone())
.restore_object_catalog_backup_with_id("analytics", &backup.backup_id, None, false, Some(restore_id))
.await
.expect("completed restore returns its original receipt");
first.status = TableCatalogRestoreStatus::AlreadyRestored;
assert_eq!(replay, first, "the receipt count describes its original restore, not later maintenance");
assert_eq!(backup_maintenance_store_writes(&target).await, before_replay);
assert_eq!(
target_store
.get_table_maintenance_config("analytics", "sales", "orders")
.await
.unwrap(),
later_config
);
assert_eq!(
target_store
.get_table_metadata_maintenance_report("analytics", "sales", "orders", &later.job.job_id)
.await
.unwrap(),
Some(later_report)
);
}
#[tokio::test]
async fn legacy_v1_restore_controls_replay_an_explicit_zero_count_and_reject_a_missing_required_count() {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let backup = store
.import_object_catalog_backup("analytics", include_bytes!("fixtures/object-catalog-backup-v1.json").to_vec())
.await
.expect("import the captured original v1 artifact");
assert_eq!(backup.source_snapshot_version, 1);
let restore_id = "legacy-v1-maintenance-count";
let mut first = store
.restore_object_catalog_backup_with_id("analytics", &backup.backup_id, None, false, Some(restore_id))
.await
.expect("complete the no-maintenance restore");
assert_eq!(first.maintenance_object_count, 0);
let paths = TableCatalogObjectPaths::default();
let intent_path = paths.catalog_restore_intent_path("analytics", restore_id);
let barrier_path = paths.catalog_restore_state_path("analytics");
let mut intent = restore_control_json(&backend, &intent_path).await;
intent["version"] = serde_json::json!(1);
assert_eq!(
intent["maintenance_object_count"], 0,
"the historical v1 intent requires an explicit zero"
);
let mut barrier = restore_control_json(&backend, &barrier_path).await;
barrier["version"] = serde_json::json!(1);
let fields = barrier.as_object_mut().unwrap();
assert!(fields.remove("completed_intent_etag").is_some());
assert!(fields.remove("backup_etag").is_some());
backend
.seed_object(RUSTFS_META_BUCKET, &intent_path, serde_json::to_vec(&intent).unwrap())
.await;
backend
.seed_object(RUSTFS_META_BUCKET, &barrier_path, serde_json::to_vec(&barrier).unwrap())
.await;
let before = backup_maintenance_store_writes(&backend).await;
assert!(
ObjectTableCatalogStore::new(backend.clone())
.load_table("analytics", "sales", "orders")
.await
.expect("read existing v1 controls without a format migration")
.is_some()
);
let replay = ObjectTableCatalogStore::new(backend.clone())
.restore_object_catalog_backup_with_id("analytics", &backup.backup_id, None, false, Some(restore_id))
.await
.expect("old v1 receipts remain replayable");
first.status = TableCatalogRestoreStatus::AlreadyRestored;
assert_eq!(replay, first);
assert_eq!(
backup_maintenance_store_writes(&backend).await,
before,
"reading v1 controls must not rewrite them"
);
assert_eq!(
intent.as_object_mut().unwrap().remove("maintenance_object_count"),
Some(serde_json::json!(0))
);
backend
.seed_object(RUSTFS_META_BUCKET, &intent_path, serde_json::to_vec(&intent).unwrap())
.await;
let before_missing_count = backup_maintenance_store_writes(&backend).await;
assert!(matches!(
ObjectTableCatalogStore::new(backend.clone())
.load_table("analytics", "sales", "orders")
.await,
Err(TableCatalogStoreError::Invalid(_) | TableCatalogStoreError::Unavailable(_))
));
assert!(matches!(
ObjectTableCatalogStore::new(backend.clone())
.restore_object_catalog_backup_with_id("analytics", &backup.backup_id, None, false, Some(restore_id))
.await,
Err(TableCatalogStoreError::Invalid(_) | TableCatalogStoreError::Unavailable(_))
));
assert_eq!(
backup_maintenance_store_writes(&backend).await,
before_missing_count,
"a missing required v1 field must fail closed without defaulting or repairing it"
);
}
@@ -1,420 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::table_catalog::test_support::{
TestCatalogObjectPause, manifest_avro_bytes as reference_manifest_bytes,
manifest_list_avro_bytes as reference_manifest_list_bytes,
};
const REFERENCE_DATA: &str = "tables/table-id/data/revalidation.parquet";
const REFERENCE_MANIFEST: &str = "tables/table-id/metadata/revalidation.avro";
const REFERENCE_LIST: &str = "tables/table-id/metadata/revalidation-list.avro";
const REFERENCE_STATISTICS: &str = "tables/table-id/metadata/revalidation-statistics.puffin";
const REFERENCE_STATISTICS_BYTES: &[u8] = b"PFA1aaaaPFA1";
const CHANGED_REFERENCE_STATISTICS_BYTES: &[u8] = b"PFA1bbbbPFA1";
struct ReferenceRestoreMutation {
object: String,
original: Vec<u8>,
changed: Vec<u8>,
}
struct ReferenceRestoreFixture {
backend: TestCatalogObjectBackend,
backup: TableCatalogBackupReport,
expected_table: TableEntry,
mutations: Vec<ReferenceRestoreMutation>,
}
#[derive(Debug, PartialEq)]
struct ReferenceRestoreWrites {
objects: BTreeMap<(String, String), TableCatalogObject>,
puts: BTreeMap<(String, String), usize>,
deletes: BTreeMap<(String, String), usize>,
}
async fn reference_restore_writes(backend: &TestCatalogObjectBackend) -> ReferenceRestoreWrites {
let objects = maintenance_test_object_versions(backend).await;
let state = backend.state.lock().await;
ReferenceRestoreWrites {
objects,
puts: state.put_attempts.clone(),
deletes: state.delete_attempts.clone(),
}
}
// Seed a valid source graph, then cross the real raw-import and controlled-prepare boundaries.
async fn reference_restore_fixture() -> ReferenceRestoreFixture {
let (source, _, current_path) = backup_fixture().await;
let store = ObjectTableCatalogStore::new(source.clone());
let namespace = Namespace::parse("sales").expect("reference fixture namespace");
let view = IdentifierSegment::parse("recent_orders").expect("reference fixture view");
let history_path = default_table_metadata_file_path(
&namespace,
&IdentifierSegment::parse("orders").expect("reference fixture table"),
"00000.metadata.json",
);
let mut history = table_metadata_json_for_backup("analytics", "table-id", "table-uuid");
history["properties"]["reference-watermark"] = serde_json::json!("a");
let history_bytes = serde_json::to_vec(&history).expect("serialize valid history metadata");
history["properties"]["reference-watermark"] = serde_json::json!("b");
let changed_history = serde_json::to_vec(&history).expect("serialize same-length valid changed history");
let mut metadata = table_metadata_json_for_backup("analytics", "table-id", "table-uuid");
metadata["last-sequence-number"] = serde_json::json!(7);
metadata["last-updated-ms"] = serde_json::json!(2);
metadata["snapshots"] = serde_json::json!([{
"snapshot-id": 42, "sequence-number": 7, "timestamp-ms": 2,
"manifest-list": format!("s3://analytics/{REFERENCE_LIST}"), "summary": {"operation": "append"}
}]);
metadata["current-snapshot-id"] = serde_json::json!(42);
metadata["snapshot-log"] = serde_json::json!([{"timestamp-ms": 2, "snapshot-id": 42}]);
metadata["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 42}});
metadata["metadata-log"] = serde_json::json!([{
"timestamp-ms": 1, "metadata-file": format!("s3://analytics/{history_path}")
}]);
metadata["statistics"] = serde_json::json!([{
"snapshot-id": 42, "statistics-path": format!("s3://analytics/{REFERENCE_STATISTICS}"),
"file-size-in-bytes": REFERENCE_STATISTICS_BYTES.len(), "file-footer-size-in-bytes": 4, "blob-metadata": []
}]);
validate_supported_table_metadata(&metadata).expect("reference fixture must contain valid nonempty metadata");
let data_location = format!("s3://analytics/{REFERENCE_DATA}");
let manifest = reference_manifest_bytes(&[(&data_location, 0, 1, 42, 7)]);
let manifest_location = format!("s3://analytics/{REFERENCE_MANIFEST}");
source.seed_object("analytics", REFERENCE_DATA, b"x".to_vec()).await;
source
.seed_object("analytics", REFERENCE_STATISTICS, REFERENCE_STATISTICS_BYTES.to_vec())
.await;
source.seed_object("analytics", &history_path, history_bytes.clone()).await;
source
.seed_object(
"analytics",
REFERENCE_LIST,
reference_manifest_list_bytes(&[(&manifest_location, manifest.len())], 7, 42),
)
.await;
source.seed_object("analytics", REFERENCE_MANIFEST, manifest).await;
source
.seed_object(
"analytics",
&current_path,
serde_json::to_vec(&metadata).expect("serialize current metadata"),
)
.await;
let view_path = default_view_metadata_file_path(&namespace, &view, "00001.metadata.json");
let entry = test_view_entry("analytics", &namespace, &view, view_path.clone());
let mut view_metadata = serde_json::json!({
"format-version": 1, "view-uuid": entry.view_uuid, "location": entry.warehouse_location,
"schemas": [{"type": "struct", "schema-id": 0,
"fields": [{"id": 1, "name": "id", "required": true, "type": "long"}]}],
"current-schema-id": 0, "current-version-id": 1,
"versions": [{"version-id": 1, "timestamp-ms": 1, "schema-id": 0, "summary": {},
"representations": [{"type": "sql", "sql": "SELECT 1", "dialect": "spark"}],
"default-namespace": ["sales"]}],
"version-log": [{"timestamp-ms": 1, "version-id": 1}], "properties": {}
});
validate_supported_view_metadata(&view_metadata).expect("reference fixture view metadata is valid");
let view_bytes = serde_json::to_vec(&view_metadata).expect("serialize view metadata");
view_metadata["versions"][0]["representations"][0]["sql"] = serde_json::json!("SELECT 2");
let changed_view = serde_json::to_vec(&view_metadata).expect("serialize same-length valid changed view");
source.seed_object("analytics", &view_path, view_bytes.clone()).await;
store
.create_view(entry)
.await
.expect("create the source view through the store");
let expected_table = store
.load_table("analytics", "sales", "orders")
.await
.expect("source table read")
.expect("source table");
let backup = store
.create_object_catalog_backup("analytics", None)
.await
.expect("capture real v2 nonempty reference inventory");
assert_eq!(backup.source_snapshot_version, 2);
let artifact = store
.export_object_catalog_backup("analytics", &backup.backup_id)
.await
.expect("export original artifact bytes");
let decoded: serde_json::Value = serde_json::from_slice(&artifact).expect("artifact is JSON");
let target = TestCatalogObjectBackend::default();
let target_store = ObjectTableCatalogStore::new(target.clone());
target_store
.import_object_catalog_backup("analytics", artifact.clone())
.await
.expect("import artifact without catalog publication");
let references = decoded["references"].as_array().expect("v2 artifact has references");
assert!(references.iter().any(|reference| reference["kind"] == "DATA_FILE"));
for reference in references {
let object = reference["object"].as_str().expect("reference key");
if object == current_path {
// The immutable current metadata remains a real restore-plan write.
continue;
}
let payload = source
.read_object("analytics", object)
.await
.expect("read source reference")
.expect("source reference exists");
target_store
.prepare_object_catalog_backup_object("analytics", &backup.backup_id, object, payload.data)
.await
.expect("prepare exact referenced bytes without publishing the catalog");
}
assert!(
target_store
.load_table("analytics", "sales", "orders")
.await
.expect("empty target read")
.is_none()
);
let mutations = vec![
ReferenceRestoreMutation {
object: REFERENCE_DATA.to_string(),
original: b"x".to_vec(),
changed: b"y".to_vec(),
},
ReferenceRestoreMutation {
object: history_path,
original: history_bytes,
changed: changed_history,
},
ReferenceRestoreMutation {
object: view_path,
original: view_bytes,
changed: changed_view,
},
ReferenceRestoreMutation {
object: REFERENCE_STATISTICS.to_string(),
original: REFERENCE_STATISTICS_BYTES.to_vec(),
changed: CHANGED_REFERENCE_STATISTICS_BYTES.to_vec(),
},
];
for mutation in &mutations {
assert_eq!(
mutation.original.len(),
mutation.changed.len(),
"content, rather than size, is the regression oracle"
);
assert_ne!(mutation.original, mutation.changed);
}
ReferenceRestoreFixture {
backend: target,
backup,
expected_table,
mutations,
}
}
type ReferenceRestoreTask = tokio::task::JoinHandle<TableCatalogStoreResult<TableCatalogRestoreReport>>;
async fn wait_reference_restore_pause(pause: &TestCatalogObjectPause, worker: &mut ReferenceRestoreTask) {
if tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, pause.wait_started())
.await
.is_err()
{
worker.abort();
pause.release();
let _ = tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, worker).await;
panic!("restore did not reach its bounded table-publication barrier");
}
}
async fn join_reference_restore(mut worker: ReferenceRestoreTask) -> TableCatalogStoreResult<TableCatalogRestoreReport> {
match tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, &mut worker).await {
Ok(joined) => joined.expect("restore worker must not panic"),
Err(_) => {
worker.abort();
let _ = tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, worker).await;
panic!("restore worker did not finish within its bounded operation window");
}
}
}
fn assert_reference_watermark_conflict(result: TableCatalogStoreResult<TableCatalogRestoreReport>) {
assert!(
matches!(&result, Err(TableCatalogStoreError::Conflict(message)) if message.contains("watermark")),
"changed reference bytes must fail the inventory watermark before restore completion: {result:?}"
);
}
async fn reference_restore_assert_published(fixture: &ReferenceRestoreFixture, restore_id: &str) {
let store = ObjectTableCatalogStore::new(fixture.backend.clone());
assert_eq!(
store
.load_table("analytics", "sales", "orders")
.await
.expect("fully published table read"),
Some(fixture.expected_table.clone())
);
assert!(
store
.load_view("analytics", "sales", "recent_orders")
.await
.expect("fully published view read")
.is_some()
);
let paths = TableCatalogObjectPaths::default();
assert_eq!(
restore_control_json(&fixture.backend, &paths.catalog_restore_state_path("analytics")).await["state"],
"COMPLETED"
);
assert_eq!(
restore_control_json(&fixture.backend, &paths.catalog_restore_intent_path("analytics", restore_id)).await["state"],
"COMPLETED"
);
}
#[tokio::test]
async fn interrupted_v2_restore_revalidates_every_reference_before_resuming_the_frozen_plan() {
for changed_reference in 0..4 {
let fixture = reference_restore_fixture().await;
let paths = TableCatalogObjectPaths::default();
let restore_id = "interrupted-reference-watermark";
let table_path = paths.table_entry_path(
"analytics",
&Namespace::parse("sales").expect("namespace"),
&IdentifierSegment::parse("orders").expect("table"),
);
let pause = fixture.backend.pause_next_put(RUSTFS_META_BUCKET, &table_path).await;
let worker_backend = fixture.backend.clone();
let backup_id = fixture.backup.backup_id.clone();
let mut worker = tokio::spawn(async move {
ObjectTableCatalogStore::new(worker_backend)
.restore_object_catalog_backup_with_id("analytics", &backup_id, None, false, Some(restore_id))
.await
});
wait_reference_restore_pause(&pause, &mut worker).await;
worker.abort();
pause.release();
let cancelled = tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, worker)
.await
.expect("cancelled restore releases its publication and backup guards")
.expect_err("restore request was cancelled");
assert!(cancelled.is_cancelled());
let intent_path = paths.catalog_restore_intent_path("analytics", restore_id);
assert_eq!(restore_control_json(&fixture.backend, &intent_path).await["state"], "PREPARED");
assert_eq!(
restore_control_json(&fixture.backend, &paths.catalog_restore_state_path("analytics")).await["state"],
"APPLYING"
);
let mutation = &fixture.mutations[changed_reference];
fixture
.backend
.seed_object("analytics", &mutation.object, mutation.changed.clone())
.await;
let before = reference_restore_writes(&fixture.backend).await;
assert_reference_watermark_conflict(
ObjectTableCatalogStore::new(fixture.backend.clone())
.restore_object_catalog_backup_with_id("analytics", &fixture.backup.backup_id, None, false, Some(restore_id))
.await,
);
assert_eq!(
reference_restore_writes(&fixture.backend).await,
before,
"a failed resume must neither advance controls nor mutate the frozen catalog targets"
);
assert_pending_restore_rejects_catalog_access(&fixture.backend).await;
fixture
.backend
.seed_object("analytics", &mutation.object, mutation.original.clone())
.await;
let restored = ObjectTableCatalogStore::new(fixture.backend.clone())
.restore_object_catalog_backup_with_id("analytics", &fixture.backup.backup_id, None, false, Some(restore_id))
.await
.expect("the original restore-id resumes after exact reference bytes are repaired");
assert_eq!(restored.status, TableCatalogRestoreStatus::Restored);
reference_restore_assert_published(&fixture, restore_id).await;
}
}
#[tokio::test]
async fn v2_restore_final_verification_rejects_reference_changes_after_preflight() {
for changed_reference in 0..4 {
let fixture = reference_restore_fixture().await;
let paths = TableCatalogObjectPaths::default();
let restore_id = "final-reference-watermark";
let table_path = paths.table_entry_path(
"analytics",
&Namespace::parse("sales").expect("namespace"),
&IdentifierSegment::parse("orders").expect("table"),
);
let pause = fixture.backend.pause_next_put(RUSTFS_META_BUCKET, &table_path).await;
let worker_backend = fixture.backend.clone();
let backup_id = fixture.backup.backup_id.clone();
let mut worker = tokio::spawn(async move {
ObjectTableCatalogStore::new(worker_backend)
.restore_object_catalog_backup_with_id("analytics", &backup_id, None, false, Some(restore_id))
.await
});
wait_reference_restore_pause(&pause, &mut worker).await;
let mutation = &fixture.mutations[changed_reference];
fixture
.backend
.seed_object("analytics", &mutation.object, mutation.changed.clone())
.await;
pause.release();
assert_reference_watermark_conflict(join_reference_restore(worker).await);
assert_eq!(
restore_control_json(&fixture.backend, &paths.catalog_restore_state_path("analytics")).await["state"],
"APPLYING"
);
assert_eq!(
restore_control_json(&fixture.backend, &paths.catalog_restore_intent_path("analytics", restore_id)).await["state"],
"PREPARED"
);
assert_pending_restore_rejects_catalog_access(&fixture.backend).await;
fixture
.backend
.seed_object("analytics", &mutation.object, mutation.original.clone())
.await;
let restored = ObjectTableCatalogStore::new(fixture.backend.clone())
.restore_object_catalog_backup_with_id("analytics", &fixture.backup.backup_id, None, false, Some(restore_id))
.await
.expect("the same frozen plan finishes after exact reference repair");
assert_eq!(restored.status, TableCatalogRestoreStatus::Restored);
reference_restore_assert_published(&fixture, restore_id).await;
}
}
#[tokio::test]
async fn completed_v2_restore_receipt_replays_after_later_reference_changes_without_revalidation_or_writes() {
let fixture = reference_restore_fixture().await;
let restore_id = "completed-reference-watermark";
let mut receipt = ObjectTableCatalogStore::new(fixture.backend.clone())
.restore_object_catalog_backup_with_id("analytics", &fixture.backup.backup_id, None, false, Some(restore_id))
.await
.expect("complete a healthy nonempty v2 restore");
reference_restore_assert_published(&fixture, restore_id).await;
for mutation in &fixture.mutations {
fixture
.backend
.seed_object("analytics", &mutation.object, mutation.changed.clone())
.await;
}
let before = reference_restore_writes(&fixture.backend).await;
let replay = ObjectTableCatalogStore::new(fixture.backend.clone())
.restore_object_catalog_backup_with_id("analytics", &fixture.backup.backup_id, None, false, Some(restore_id))
.await
.expect("a completed receipt records the original request rather than authorizing another restore");
receipt.status = TableCatalogRestoreStatus::AlreadyRestored;
assert_eq!(replay, receipt);
assert_eq!(
reference_restore_writes(&fixture.backend).await,
before,
"completed receipt replay must not validate, repair, or overwrite later reference state"
);
reference_restore_assert_published(&fixture, restore_id).await;
}
File diff suppressed because one or more lines are too long
@@ -1,406 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use serde::{Deserialize, Serialize};
use sha2::Digest;
const FIXTURE_BACKUP_ID: &str = "83e3024f278b0abeecdbde378068980c3d5c9a2a5ddaa915495d5b398d1d05c2";
const TRANSFER_LIMIT: usize = 64 * 1024 * 1024;
fn fixture_bytes() -> Vec<u8> {
include_bytes!("fixtures/object-catalog-backup-v1.json").to_vec()
}
fn fixture_path() -> String {
TableCatalogObjectPaths::default().catalog_backup_path("analytics", FIXTURE_BACKUP_ID)
}
// Keep the v1 field order when constructing self-consistent negative fixtures.
// Their digests are valid so the test reaches identity validation, not only hashing.
#[derive(Serialize, Deserialize)]
struct TransferFixtureObject {
bucket: String,
object: String,
size_bytes: u64,
etag: Option<String>,
sha256: String,
data: Vec<u8>,
}
#[derive(Serialize, Deserialize)]
struct TransferFixture {
version: u16,
backup_id: String,
table_bucket: String,
created_at: String,
source_catalog_etag: String,
objects: Vec<TransferFixtureObject>,
}
fn digest(data: &[u8]) -> String {
hex_simd::encode_to_string(sha2::Sha256::digest(data), hex_simd::AsciiCase::Lower)
}
fn reseal_fixture(fixture: &mut TransferFixture) -> Vec<u8> {
for object in &mut fixture.objects {
object.size_bytes = u64::try_from(object.data.len()).unwrap();
object.sha256 = digest(&object.data);
}
fixture
.objects
.sort_by(|left, right| (&left.bucket, &left.object).cmp(&(&right.bucket, &right.object)));
fixture.source_catalog_etag = digest(
&serde_json::to_vec(
&fixture
.objects
.iter()
.map(|object| (&object.bucket, &object.object, &object.sha256))
.collect::<Vec<_>>(),
)
.unwrap(),
);
fixture.backup_id = digest(
&serde_json::to_vec(&(fixture.version, &fixture.table_bucket, &fixture.source_catalog_etag, &fixture.objects)).unwrap(),
);
serde_json::to_vec(fixture).unwrap()
}
#[tokio::test]
async fn export_preserves_the_saved_v1_bytes_and_creation_timestamp() {
let backend = TestCatalogObjectBackend::default();
let mut bytes = b" \n".to_vec();
bytes.extend(fixture_bytes());
bytes.extend_from_slice(b"\n ");
backend.seed_object(RUSTFS_META_BUCKET, &fixture_path(), bytes.clone()).await;
let before = maintenance_test_object_versions(&backend).await;
let exported = ObjectTableCatalogStore::new(backend.clone())
.export_object_catalog_backup("analytics", FIXTURE_BACKUP_ID)
.await
.expect("export original bytes without reencoding");
assert_eq!(exported, bytes);
let exported: TransferFixture = serde_json::from_slice(&exported).unwrap();
assert_eq!(exported.created_at, "2026-10-30T03:24:02.276322305Z");
assert_eq!(maintenance_test_object_versions(&backend).await, before);
}
#[tokio::test]
async fn import_only_creates_an_artifact_and_exact_retries_do_not_rewrite_it() {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let bytes = fixture_bytes();
let mut first = store.import_object_catalog_backup("analytics", bytes.clone()).await.unwrap();
assert_eq!(first.status, TableCatalogBackupStatus::Created);
assert_eq!(first.backup_id, FIXTURE_BACKUP_ID);
assert_eq!(first.source_snapshot_version, 1);
assert_eq!(first.created_at, "2026-10-30T03:24:02.276322305Z");
let before = maintenance_test_object_versions(&backend).await;
assert_eq!(before.len(), 1, "import must not publish any catalog or metadata objects");
assert_eq!(before.get(&(RUSTFS_META_BUCKET.to_string(), fixture_path())).unwrap().data, bytes);
assert!(store.get_table_bucket("analytics").await.unwrap().is_none());
assert!(store.load_table("analytics", "sales", "orders").await.unwrap().is_none());
let repeated = store
.import_object_catalog_backup("analytics", fixture_bytes())
.await
.unwrap();
first.status = TableCatalogBackupStatus::AlreadyPresent;
assert_eq!(repeated, first);
assert_eq!(maintenance_test_object_versions(&backend).await, before);
}
#[tokio::test]
async fn same_identity_with_different_saved_bytes_is_a_conflict() {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
store
.import_object_catalog_backup("analytics", fixture_bytes())
.await
.unwrap();
let before = maintenance_test_object_versions(&backend).await;
let mut different_bytes = fixture_bytes();
different_bytes.push(b'\n');
let error = store
.import_object_catalog_backup("analytics", different_bytes)
.await
.unwrap_err();
assert!(matches!(error, TableCatalogStoreError::Conflict(_)));
assert_eq!(maintenance_test_object_versions(&backend).await, before);
}
#[tokio::test]
async fn invalid_or_foreign_artifacts_do_not_write_any_objects() {
let fixture = fixture_bytes();
let mut corrupt: serde_json::Value = serde_json::from_slice(&fixture).unwrap();
corrupt["objects"][0]["sha256"] = serde_json::json!("corrupt");
let mut unknown: serde_json::Value = serde_json::from_slice(&fixture).unwrap();
unknown["force"] = serde_json::json!(true);
for (bucket, bytes) in [
("analytics", Vec::new()),
("analytics", b"not-json".to_vec()),
("analytics", serde_json::to_vec(&corrupt).unwrap()),
("analytics", serde_json::to_vec(&unknown).unwrap()),
("other-bucket", fixture),
] {
let backend = TestCatalogObjectBackend::default();
let result = ObjectTableCatalogStore::new(backend.clone())
.import_object_catalog_backup(bucket, bytes)
.await;
assert!(result.is_err());
assert!(maintenance_test_object_versions(&backend).await.is_empty());
}
}
#[tokio::test]
async fn pure_metadata_identity_validation_precedes_import_storage() {
for field in ["table-uuid", "location", "format-version", "empty-metadata"] {
let mut artifact: TransferFixture = serde_json::from_slice(&fixture_bytes()).unwrap();
let metadata = artifact
.objects
.iter_mut()
.find(|object| object.bucket == "analytics")
.unwrap();
if field == "empty-metadata" {
metadata.data.clear();
} else {
let mut value: serde_json::Value = serde_json::from_slice(&metadata.data).unwrap();
value[field] = match field {
"table-uuid" => serde_json::json!("another-table-uuid"),
"location" => serde_json::json!("s3://analytics/tables/another-id"),
"format-version" => serde_json::json!(0),
_ => unreachable!(),
};
metadata.data = serde_json::to_vec(&value).unwrap();
}
let bytes = reseal_fixture(&mut artifact);
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let result = store.import_object_catalog_backup("analytics", bytes.clone()).await;
assert!(matches!(result, Err(TableCatalogStoreError::Invalid(_))), "invalid {field} was imported");
assert!(maintenance_test_object_versions(&backend).await.is_empty());
let path = TableCatalogObjectPaths::default().catalog_backup_path("analytics", &artifact.backup_id);
backend.seed_object(RUSTFS_META_BUCKET, &path, bytes).await;
let before = maintenance_test_object_versions(&backend).await;
assert!(matches!(
store.export_object_catalog_backup("analytics", &artifact.backup_id).await,
Err(TableCatalogStoreError::Invalid(_)),
));
assert_eq!(maintenance_test_object_versions(&backend).await, before);
}
}
#[tokio::test]
async fn missing_embedded_current_metadata_is_rejected_before_import() {
let mut artifact: TransferFixture = serde_json::from_slice(&fixture_bytes()).unwrap();
artifact.objects.retain(|object| object.bucket != "analytics");
let bytes = reseal_fixture(&mut artifact);
let backend = TestCatalogObjectBackend::default();
let result = ObjectTableCatalogStore::new(backend.clone())
.import_object_catalog_backup("analytics", bytes)
.await;
assert!(matches!(result, Err(TableCatalogStoreError::NotFound(_))));
assert!(maintenance_test_object_versions(&backend).await.is_empty());
}
#[tokio::test]
async fn ambiguous_import_put_is_confirmed_without_rewriting_the_artifact() {
let backend = TestCatalogObjectBackend::default();
let path = fixture_path();
backend.fail_after_next_put(RUSTFS_META_BUCKET, &path).await;
let store = ObjectTableCatalogStore::new(backend.clone());
let first = store
.import_object_catalog_backup("analytics", fixture_bytes())
.await
.expect("a matching saved artifact proves an ambiguous import completed");
assert_eq!(first.status, TableCatalogBackupStatus::AlreadyPresent);
let before = maintenance_test_object_versions(&backend).await;
assert_eq!(before.len(), 1);
let repeated = ObjectTableCatalogStore::new(backend.clone())
.import_object_catalog_backup("analytics", fixture_bytes())
.await
.unwrap();
assert_eq!(repeated, first);
assert_eq!(maintenance_test_object_versions(&backend).await, before);
assert_eq!(backend.state.lock().await.put_attempts[&(RUSTFS_META_BUCKET.to_string(), path)], 1);
}
#[tokio::test]
async fn import_failure_before_put_leaves_no_artifact_and_can_be_retried() {
let backend = TestCatalogObjectBackend::default();
backend.fail_next_put(RUSTFS_META_BUCKET, &fixture_path()).await;
let result = ObjectTableCatalogStore::new(backend.clone())
.import_object_catalog_backup("analytics", fixture_bytes())
.await;
assert!(matches!(result, Err(TableCatalogStoreError::Internal(_))));
assert!(maintenance_test_object_versions(&backend).await.is_empty());
let retried = ObjectTableCatalogStore::new(backend.clone())
.import_object_catalog_backup("analytics", fixture_bytes())
.await
.unwrap();
assert_eq!(retried.status, TableCatalogBackupStatus::Created);
assert_eq!(maintenance_test_object_versions(&backend).await.len(), 1);
}
#[tokio::test]
async fn concurrent_identical_imports_create_exactly_one_artifact_version() {
let backend = TestCatalogObjectBackend::default();
let first_store = ObjectTableCatalogStore::new(backend.clone());
let second_store = ObjectTableCatalogStore::new(backend.clone());
let (first, second) = tokio::join!(
first_store.import_object_catalog_backup("analytics", fixture_bytes()),
second_store.import_object_catalog_backup("analytics", fixture_bytes()),
);
let first = first.unwrap();
let second = second.unwrap();
assert_eq!(
usize::from(first.status == TableCatalogBackupStatus::Created)
+ usize::from(second.status == TableCatalogBackupStatus::Created),
1,
);
assert_eq!(first.backup_id, second.backup_id);
assert_eq!(first.created_at, second.created_at);
assert_eq!(maintenance_test_object_versions(&backend).await.len(), 1);
assert_eq!(
backend.state.lock().await.put_attempts[&(RUSTFS_META_BUCKET.to_string(), fixture_path())],
1,
);
}
#[tokio::test]
async fn export_rejects_a_saved_artifact_under_the_wrong_backup_identity() {
let backend = TestCatalogObjectBackend::default();
let path = TableCatalogObjectPaths::default().catalog_backup_path("analytics", "wrong-backup");
backend.seed_object(RUSTFS_META_BUCKET, &path, fixture_bytes()).await;
let before = maintenance_test_object_versions(&backend).await;
let result = ObjectTableCatalogStore::new(backend.clone())
.export_object_catalog_backup("analytics", "wrong-backup")
.await;
assert!(matches!(result, Err(TableCatalogStoreError::Invalid(_))));
assert_eq!(maintenance_test_object_versions(&backend).await, before);
}
#[tokio::test]
async fn artifact_import_does_not_require_the_external_snapshot_graph_to_be_prepared() {
let mut artifact: TransferFixture = serde_json::from_slice(&fixture_bytes()).unwrap();
let metadata = artifact
.objects
.iter_mut()
.find(|object| object.bucket == "analytics")
.unwrap();
let mut value: serde_json::Value = serde_json::from_slice(&metadata.data).unwrap();
value["last-sequence-number"] = serde_json::json!(1);
value["snapshots"] = serde_json::json!([{
"snapshot-id": 1,
"sequence-number": 1,
"timestamp-ms": 1,
"manifest-list": "s3://analytics/tables/table-id/metadata/snap-1.avro",
"summary": {"operation": "append"}
}]);
value["current-snapshot-id"] = serde_json::json!(1);
value["snapshot-log"] = serde_json::json!([{"timestamp-ms": 1, "snapshot-id": 1}]);
value["refs"] = serde_json::json!({"main": {"type": "branch", "snapshot-id": 1}});
metadata.data = serde_json::to_vec(&value).unwrap();
let bytes = reseal_fixture(&mut artifact);
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let imported = store
.import_object_catalog_backup("analytics", bytes)
.await
.expect("import stores only the candidate artifact before object preparation");
let before = maintenance_test_object_versions(&backend).await;
assert_eq!(before.len(), 1);
let restore = store
.restore_object_catalog_backup_with_id("analytics", &imported.backup_id, None, false, Some("unprepared-graph"))
.await;
assert!(restore.is_err(), "publishing still requires the external graph");
assert_eq!(maintenance_test_object_versions(&backend).await, before);
}
#[tokio::test]
async fn pending_restore_rejects_artifact_export_and_import_without_mutation() {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
store
.import_object_catalog_backup("analytics", fixture_bytes())
.await
.unwrap();
let table_path = TableCatalogObjectPaths::default().table_entry_path(
"analytics",
&Namespace::parse("sales").unwrap(),
&IdentifierSegment::parse("orders").unwrap(),
);
backend.fail_next_put(RUSTFS_META_BUCKET, &table_path).await;
store
.restore_object_catalog_backup_with_id("analytics", FIXTURE_BACKUP_ID, None, false, Some("transfer-pending"))
.await
.expect_err("prepare interrupted restore");
let before = maintenance_test_object_versions(&backend).await;
assert!(matches!(
store.export_object_catalog_backup("analytics", FIXTURE_BACKUP_ID).await,
Err(TableCatalogStoreError::Unavailable(_)),
));
assert!(matches!(
store.import_object_catalog_backup("analytics", fixture_bytes()).await,
Err(TableCatalogStoreError::Unavailable(_)),
));
assert_eq!(maintenance_test_object_versions(&backend).await, before);
}
#[tokio::test]
async fn legacy_control_state_rejects_both_transfer_operations() {
let paths = TableCatalogObjectPaths::default();
for marker in [
paths.legacy_backing_migration_fence_path("analytics"),
paths.legacy_backing_migration_global_fence_path(),
paths.legacy_catalog_restore_intent_path("analytics"),
paths.legacy_catalog_snapshot_path(),
] {
let backend = TestCatalogObjectBackend::default();
backend
.seed_object(RUSTFS_META_BUCKET, &fixture_path(), fixture_bytes())
.await;
backend
.seed_object(RUSTFS_META_BUCKET, &marker, b"legacy-control".to_vec())
.await;
let before = maintenance_test_object_versions(&backend).await;
let store = ObjectTableCatalogStore::new(backend.clone());
assert!(matches!(
store.export_object_catalog_backup("analytics", FIXTURE_BACKUP_ID).await,
Err(TableCatalogStoreError::Unavailable(_)),
));
assert!(matches!(
store.import_object_catalog_backup("analytics", fixture_bytes()).await,
Err(TableCatalogStoreError::Unavailable(_)),
));
assert_eq!(maintenance_test_object_versions(&backend).await, before);
}
}
#[tokio::test]
async fn transfer_body_limit_accepts_the_exact_boundary_and_rejects_one_more_byte() {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let mut exact = fixture_bytes();
exact.resize(TRANSFER_LIMIT, b' ');
let imported = store
.import_object_catalog_backup("analytics", exact)
.await
.expect("a valid artifact at the exact encoded-body limit is accepted");
assert_eq!(imported.status, TableCatalogBackupStatus::Created);
let before = maintenance_test_object_versions(&backend).await;
let mut oversized = fixture_bytes();
oversized.resize(TRANSFER_LIMIT + 1, b' ');
let result = store.import_object_catalog_backup("analytics", oversized).await;
assert!(matches!(result, Err(TableCatalogStoreError::Invalid(_))));
assert_eq!(maintenance_test_object_versions(&backend).await, before);
}
File diff suppressed because it is too large Load Diff
@@ -1,901 +0,0 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
async fn planning_fixture(
metadata_location: &str,
) -> (TestCatalogObjectBackend, ObjectTableCatalogStore<TestCatalogObjectBackend>, TableEntry) {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let namespace = Namespace::parse("sales").expect("namespace");
let table = IdentifierSegment::parse("orders").expect("table");
seed_table_for_metadata_maintenance(&store, "analytics", &namespace, &table, metadata_location.to_string()).await;
let object = table_catalog_object_key_from_location("analytics", metadata_location).expect("metadata object key");
backend
.seed_object(
"analytics",
&object,
serde_json::to_vec(&table_metadata_json_for_backup("analytics", "table-id", "table-uuid"))
.expect("valid table metadata"),
)
.await;
let entry = store.load_table("analytics", "sales", "orders").await.unwrap().unwrap();
(backend, store, entry)
}
#[tokio::test]
async fn maintenance_planning_after_rename_keeps_the_original_pointer_and_identity() {
let namespace = Namespace::parse("sales").expect("namespace");
let table = IdentifierSegment::parse("orders").expect("table");
let current = default_table_metadata_file_path(&namespace, &table, "00003.metadata.json");
let orphan = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json");
let (backend, store, original) = planning_fixture(&current).await;
backend.seed_object("analytics", &orphan, b"{}".to_vec()).await;
store
.rename_table("analytics", "sales", "orders", "sales", "archived_orders")
.await
.expect("rename through the object catalog");
let renamed = store
.load_table("analytics", "sales", "archived_orders")
.await
.unwrap()
.unwrap();
assert_eq!(renamed.metadata_location, original.metadata_location);
assert_eq!(renamed.table_id, original.table_id);
assert_eq!(renamed.table_uuid, original.table_uuid);
assert_eq!(renamed.version_token, original.version_token);
assert_eq!(renamed.generation, original.generation);
let before = maintenance_test_object_versions(&backend).await;
let report = store
.plan_table_metadata_maintenance("analytics", "sales", "archived_orders", 0)
.await
.expect("planning follows the renamed entry's persisted directory");
assert_eq!(report.job.table, "archived_orders");
assert_eq!(report.job.table_id, original.table_id);
assert_eq!(report.current_metadata_location, current);
assert_eq!(report.cleanup_candidate_locations, vec![orphan]);
assert_eq!(maintenance_test_object_versions(&backend).await, before);
assert_eq!(store.load_table("analytics", "sales", "archived_orders").await.unwrap(), Some(renamed));
}
#[tokio::test]
async fn maintenance_planning_does_not_follow_a_recreated_source_name() {
let current = "tables/table-id/metadata/00003.metadata.json";
let orphan = "tables/table-id/metadata/00001.metadata.json";
let (backend, store, original) = planning_fixture(current).await;
backend.seed_object("analytics", orphan, b"{}".to_vec()).await;
store
.rename_table("analytics", "sales", "orders", "sales", "archived_orders")
.await
.unwrap();
let recreated_current = "tables/recreated-id/metadata/00003.metadata.json";
let recreated_orphan = "tables/recreated-id/metadata/00001.metadata.json";
let namespace = Namespace::parse("sales").unwrap();
let table = IdentifierSegment::parse("orders").unwrap();
let mut recreated = test_table_entry("analytics", &namespace, &table, recreated_current.to_string());
recreated.table_id = "recreated-id".to_string();
recreated.table_uuid = "recreated-uuid".to_string();
recreated.warehouse_location = "s3://analytics/tables/recreated-id".to_string();
backend
.seed_object(
"analytics",
recreated_current,
serde_json::to_vec(&table_metadata_json_for_backup("analytics", "recreated-id", "recreated-uuid")).unwrap(),
)
.await;
backend.seed_object("analytics", recreated_orphan, b"{}".to_vec()).await;
store
.create_table(recreated.clone())
.await
.expect("recreate the old name with a new identity");
let before = maintenance_test_object_versions(&backend).await;
let destination_report = store
.plan_table_metadata_maintenance("analytics", "sales", "archived_orders", 0)
.await
.expect("plan the renamed table");
let source_report = store
.plan_table_metadata_maintenance("analytics", "sales", "orders", 0)
.await
.expect("plan the recreated source");
assert_eq!(destination_report.job.table_id, original.table_id);
assert_eq!(destination_report.cleanup_candidate_locations, vec![orphan.to_string()]);
assert_eq!(source_report.job.table_id, recreated.table_id);
assert_eq!(source_report.cleanup_candidate_locations, vec![recreated_orphan.to_string()]);
assert_eq!(maintenance_test_object_versions(&backend).await, before);
}
#[tokio::test]
async fn custom_metadata_uri_dry_run_never_marks_the_current_object_for_cleanup() {
let current = "tables/table-id/metadata/00003.metadata.json";
let orphan = "tables/table-id/metadata/00001.metadata.json";
let uri = format!("s3://analytics/{current}");
let (backend, store, entry) = planning_fixture(&uri).await;
backend.seed_object("analytics", orphan, b"{}".to_vec()).await;
let before = maintenance_test_object_versions(&backend).await;
let report = store
.plan_table_metadata_maintenance("analytics", "sales", "orders", 0)
.await
.expect("plan the same-bucket metadata URI");
assert_eq!(report.current_metadata_location, uri);
assert_eq!(report.cleanup_candidate_locations, vec![orphan.to_string()]);
assert!(
!report
.deletable_metadata_locations
.iter()
.any(|location| location == current || location == &uri)
);
assert_eq!(store.load_table("analytics", "sales", "orders").await.unwrap(), Some(entry));
assert_eq!(maintenance_test_object_versions(&backend).await, before);
}
#[tokio::test]
async fn maintenance_planning_rejects_a_metadata_directory_shared_by_active_owners() {
let first_current = "tables/shared/metadata/orders.metadata.json";
let second_current = "tables/shared/metadata/invoices.metadata.json";
let (backend, store, _) = planning_fixture(first_current).await;
let namespace = Namespace::parse("sales").unwrap();
let second = IdentifierSegment::parse("invoices").unwrap();
let mut entry = test_table_entry("analytics", &namespace, &second, second_current.to_string());
entry.table_id = "invoice-id".to_string();
entry.table_uuid = "invoice-uuid".to_string();
entry.warehouse_location = "s3://analytics/tables/invoice-id".to_string();
backend
.seed_object(
"analytics",
second_current,
serde_json::to_vec(&table_metadata_json_for_backup("analytics", "invoice-id", "invoice-uuid")).unwrap(),
)
.await;
store
.create_table(entry)
.await
.expect("the catalog can represent distinct warehouses with external metadata");
let before = maintenance_test_object_versions(&backend).await;
let result = store.plan_table_metadata_maintenance("analytics", "sales", "orders", 0).await;
assert!(
matches!(result, Err(TableCatalogStoreError::Conflict(_) | TableCatalogStoreError::Invalid(_))),
"a shared directory must not turn another active table's current metadata into a cleanup candidate"
);
assert_eq!(maintenance_test_object_versions(&backend).await, before);
assert!(backend.object_exists("analytics", second_current).await.unwrap());
}
async fn seed_history_report(fixture: &ConditionalMaintenanceFixture, index: usize) -> String {
let mut report = fixture.queued.clone();
report.job.job_id = format!("history-{index:04}");
report.job.status = TableMetadataMaintenanceJobStatus::Successful;
report.job.finished_at = Some(maintenance_timestamp(fixture.now));
let object = TableCatalogObjectPaths::default().table_maintenance_job_path(
"analytics",
&Namespace::parse("sales").unwrap(),
&IdentifierSegment::parse("orders").unwrap(),
&fixture.entry.table_id,
&report.job.job_id,
);
fixture
.backend
.seed_object(RUSTFS_META_BUCKET, &object, serde_json::to_vec(&report).unwrap())
.await;
object
}
#[tokio::test]
async fn scheduler_history_accepts_4096_objects_and_rejects_the_next_without_a_partial_report() {
let fixture = ConditionalMaintenanceFixture::new(TableCatalogBackingMode::ObjectBacked, 0).await;
for index in 1..4096 {
seed_history_report(&fixture, index).await;
}
let store = ObjectTableCatalogStore::new(fixture.backend.clone());
let before = maintenance_test_object_versions(&fixture.backend).await;
let report = store
.get_table_maintenance_scheduler_report_at("analytics", "sales", "orders", fixture.now)
.await
.expect("the exact history budget remains readable");
assert_eq!(
report.current_job.as_ref().map(|job| job.job_id.as_str()),
Some(fixture.queued.job.job_id.as_str())
);
assert!(!report.audit_timeline.is_empty());
assert_eq!(maintenance_test_object_versions(&fixture.backend).await, before);
seed_history_report(&fixture, 4096).await;
let before = maintenance_test_object_versions(&fixture.backend).await;
let error = store
.get_table_maintenance_scheduler_report_at("analytics", "sales", "orders", fixture.now)
.await
.expect_err("one history object beyond the budget must fail closed");
assert!(matches!(error, TableCatalogStoreError::Unavailable(message) if message.contains("4096")));
assert_eq!(maintenance_test_object_versions(&fixture.backend).await, before);
}
#[tokio::test]
async fn scheduler_history_rejects_a_listed_report_that_disappears_before_read() {
let fixture = ConditionalMaintenanceFixture::new(TableCatalogBackingMode::ObjectBacked, 0).await;
let object = seed_history_report(&fixture, 1).await;
let pause = fixture.backend.pause_before_next_read(RUSTFS_META_BUCKET, &object).await;
let backend = fixture.backend.clone();
let now = fixture.now;
let reader = tokio::spawn(async move {
ObjectTableCatalogStore::new(backend)
.get_table_maintenance_scheduler_report_at("analytics", "sales", "orders", now)
.await
});
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, pause.wait_started())
.await
.expect("history listing reached the chosen report read");
fixture
.backend
.state
.lock()
.await
.objects
.remove(&(RUSTFS_META_BUCKET.to_string(), object));
let before = maintenance_test_object_versions(&fixture.backend).await;
pause.release();
let result = tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, reader)
.await
.expect("scheduler read finishes")
.expect("scheduler task joins");
assert!(matches!(result, Err(TableCatalogStoreError::Unavailable(message)) if message.contains("history changed")));
assert_eq!(maintenance_test_object_versions(&fixture.backend).await, before);
}
#[tokio::test]
async fn scheduler_history_rejects_invalid_listing_pages_without_persistent_changes() {
for scenario in [
"oversized",
"empty-truncated",
"foreign-prefix",
"repeated-cursor",
"duplicate-object",
] {
let fixture = ConditionalMaintenanceFixture::new(TableCatalogBackingMode::ObjectBacked, 0).await;
let paths = TableCatalogObjectPaths::default();
let namespace = Namespace::parse("sales").unwrap();
let table = IdentifierSegment::parse("orders").unwrap();
let jobs_prefix = paths.table_maintenance_jobs_prefix("analytics", &namespace, &table, &fixture.entry.table_id);
let job_path = paths.table_maintenance_job_path(
"analytics",
&namespace,
&table,
&fixture.entry.table_id,
&fixture.queued.job.job_id,
);
let pages = match scenario {
"oversized" => vec![TableCatalogObjectListPage {
objects: vec![job_path; TABLE_CATALOG_LIST_MAX_KEYS + 1],
is_truncated: false,
}],
"empty-truncated" => vec![TableCatalogObjectListPage {
objects: Vec::new(),
is_truncated: true,
}],
"foreign-prefix" => vec![TableCatalogObjectListPage {
objects: vec![format!(
"{}/foreign-owner/jobs/report.json",
paths.table_bucket_entry_path("unrelated")
)],
is_truncated: false,
}],
"repeated-cursor" => vec![
TableCatalogObjectListPage {
objects: vec![job_path.clone()],
is_truncated: true,
},
TableCatalogObjectListPage {
objects: vec![job_path],
is_truncated: false,
},
],
"duplicate-object" => vec![TableCatalogObjectListPage {
objects: vec![job_path.clone(), job_path],
is_truncated: false,
}],
_ => unreachable!(),
};
fixture
.backend
.state
.lock()
.await
.list_page_overrides
.entry((RUSTFS_META_BUCKET.to_string(), jobs_prefix))
.or_default()
.extend(pages);
let before = maintenance_test_object_versions(&fixture.backend).await;
let result = ObjectTableCatalogStore::new(fixture.backend.clone())
.get_table_maintenance_scheduler_report_at("analytics", "sales", "orders", fixture.now)
.await;
assert!(
matches!(result, Err(TableCatalogStoreError::Invalid(_) | TableCatalogStoreError::Unavailable(_))),
"invalid {scenario} history listing must not return a scheduler report"
);
assert_eq!(maintenance_test_object_versions(&fixture.backend).await, before, "{scenario}");
}
}
struct SidecarRenameFixture {
backend: TestCatalogObjectBackend,
store: ObjectTableCatalogStore<TestCatalogObjectBackend>,
source: TableEntry,
report: TableMetadataMaintenanceReport,
now: OffsetDateTime,
}
impl SidecarRenameFixture {
async fn new() -> Self {
let backend = TestCatalogObjectBackend::default();
let store = ObjectTableCatalogStore::new(backend.clone());
let now = OffsetDateTime::now_utc();
let (_, _, mut report) = seed_quarantined_table_maintenance(&store, &backend, "analytics", now, None).await;
let source = store.load_table("analytics", "sales", "orders").await.unwrap().unwrap();
backend
.seed_object(
"analytics",
&source.metadata_location,
serde_json::to_vec(&table_metadata_json_for_backup("analytics", &source.table_id, &source.table_uuid)).unwrap(),
)
.await;
report.snapshot_expiration = Some(
store
.plan_table_snapshot_expiration(
"analytics",
"sales",
"orders",
TableSnapshotExpirationConfig {
min_snapshots_to_keep: 1,
max_snapshot_age_ms: 86_400_000,
},
)
.await
.expect("snapshot expiration report should be produced through the store"),
);
report.compaction = Some(
store
.plan_table_compaction(
"analytics",
"sales",
"orders",
TableCompactionPlanningConfig {
target_file_size_bytes: 1_048_576,
small_file_threshold_bytes: 524_288,
min_input_files: 2,
max_rewrite_bytes_per_job: 16_777_216,
},
)
.await
.expect("compaction report should be produced through the store"),
);
store.put_table_metadata_maintenance_report(&report).await.unwrap();
let report = store
.get_table_metadata_maintenance_report("analytics", "sales", "orders", &report.job.job_id)
.await
.unwrap()
.expect("seeded maintenance report remains readable");
Self {
backend,
store,
source,
report,
now,
}
}
fn destination(&self) -> TableEntry {
let mut destination = self.source.clone();
destination.table = "archived_orders".to_string();
destination
}
fn destination_config_path(&self) -> String {
format!("{}{}", rename_maintenance_root(&self.destination()), MAINTENANCE_CONFIG_FILE)
}
fn source_config_path(&self) -> String {
format!("{}{}", rename_maintenance_root(&self.source), MAINTENANCE_CONFIG_FILE)
}
async fn assert_pending(&self) {
let before = maintenance_test_object_versions(&self.backend).await;
let restarted = ObjectTableCatalogStore::new(self.backend.clone());
for table in ["orders", "archived_orders"] {
assert!(matches!(
restarted.load_table("analytics", "sales", table).await,
Err(TableCatalogStoreError::Unavailable(_))
));
}
assert!(matches!(
restarted.get_table_maintenance_config("analytics", "sales", "orders").await,
Err(TableCatalogStoreError::Unavailable(_))
));
assert!(matches!(
restarted
.put_table_maintenance_config("analytics", "sales", "orders", TableMaintenanceConfig::default())
.await,
Err(TableCatalogStoreError::Unavailable(_))
));
let bucket_object = self
.backend
.read_object(
RUSTFS_META_BUCKET,
&TableCatalogObjectPaths::default().table_bucket_entry_path("analytics"),
)
.await
.unwrap()
.unwrap();
let bucket: TableBucketEntry = serde_json::from_slice(&bucket_object.data).unwrap();
assert!(
bucket.active_rename_id.is_some(),
"the bucket fence must outlive interrupted sidecar publication"
);
assert_eq!(maintenance_test_object_versions(&self.backend).await, before);
}
async fn retry(&self) -> TableCatalogStoreResult<()> {
ObjectTableCatalogStore::new(self.backend.clone())
.rename_table("analytics", "sales", "orders", "sales", "archived_orders")
.await
}
}
#[tokio::test]
async fn table_rename_preserves_maintenance_configuration_reports_and_quarantine() {
let fixture = SidecarRenameFixture::new().await;
let expected = rename_maintenance_objects(&fixture.backend, &fixture.source).await;
let configuration = fixture
.store
.get_table_maintenance_config("analytics", "sales", "orders")
.await
.unwrap();
fixture
.retry()
.await
.expect("rename should migrate the complete maintenance owner");
let destination = fixture
.store
.load_table("analytics", "sales", "archived_orders")
.await
.unwrap()
.unwrap();
assert_eq!(destination.table_id, fixture.source.table_id);
assert_eq!(destination.table_uuid, fixture.source.table_uuid);
assert_eq!(destination.metadata_location, fixture.source.metadata_location);
assert_eq!(destination.warehouse_location, fixture.source.warehouse_location);
assert_eq!(destination.version_token, fixture.source.version_token);
assert_eq!(destination.generation, fixture.source.generation);
assert_renamed_maintenance_objects(&fixture.backend, &destination, &expected).await;
assert!(rename_maintenance_objects(&fixture.backend, &fixture.source).await.is_empty());
assert_eq!(
fixture
.store
.get_table_maintenance_config("analytics", "sales", "archived_orders")
.await
.unwrap(),
configuration
);
for id in [
MAINTENANCE_JOB_ALIAS_CURRENT,
MAINTENANCE_JOB_ALIAS_LATEST,
fixture.report.job.job_id.as_str(),
] {
let report = fixture
.store
.get_table_metadata_maintenance_report("analytics", "sales", "archived_orders", id)
.await
.unwrap()
.expect("renamed maintenance report should remain readable through every alias");
let mut expected_report = fixture.report.clone();
expected_report.job.table = destination.table.clone();
expected_report.snapshot_expiration.as_mut().unwrap().table = destination.table.clone();
expected_report.compaction.as_mut().unwrap().table = destination.table.clone();
assert_eq!(report, expected_report);
}
let scheduler = fixture
.store
.get_table_maintenance_scheduler_report_at("analytics", "sales", "archived_orders", fixture.now)
.await
.expect("scheduler must retain the renamed quarantine and history");
assert!(scheduler.quarantine.active);
assert_eq!(scheduler.quarantine.quarantined_object_count, fixture.report.job.quarantined_object_count);
assert!(
scheduler
.audit_timeline
.iter()
.any(|entry| entry.job_id == fixture.report.job.job_id)
);
let plan = fixture
.store
.plan_table_metadata_maintenance("analytics", "sales", "archived_orders", 0)
.await
.unwrap();
assert_eq!(plan.current_metadata_location, fixture.source.metadata_location);
assert!(
!fixture
.backend
.object_exists(RUSTFS_META_BUCKET, &rename_maintenance_intent_path("analytics"))
.await
.unwrap()
);
}
#[tokio::test]
async fn interrupted_maintenance_copy_is_fenced_and_resumes_after_store_recreation() {
for post_write in [false, true] {
let fixture = SidecarRenameFixture::new().await;
let expected = rename_maintenance_objects(&fixture.backend, &fixture.source).await;
let destination = fixture.destination_config_path();
if post_write {
fixture.backend.fail_after_next_put(RUSTFS_META_BUCKET, &destination).await;
} else {
fixture.backend.fail_next_put(RUSTFS_META_BUCKET, &destination).await;
}
assert!(fixture.retry().await.is_err(), "the injected sidecar write interruption must be surfaced");
fixture.assert_pending().await;
fixture
.retry()
.await
.expect("retrying the same rename on a fresh store must resume the frozen plan");
let renamed = fixture
.store
.load_table("analytics", "sales", "archived_orders")
.await
.unwrap()
.unwrap();
assert_renamed_maintenance_objects(&fixture.backend, &renamed, &expected).await;
assert!(rename_maintenance_objects(&fixture.backend, &fixture.source).await.is_empty());
}
}
#[tokio::test]
async fn maintenance_rename_recovers_after_source_cleanup_delete_response_loss() {
let fixture = SidecarRenameFixture::new().await;
let expected = rename_maintenance_objects(&fixture.backend, &fixture.source).await;
fixture
.backend
.fail_after_next_delete(RUSTFS_META_BUCKET, &fixture.source_config_path())
.await;
assert!(
fixture.retry().await.is_err(),
"lost source cleanup response must leave recoverable ownership"
);
fixture.assert_pending().await;
fixture
.retry()
.await
.expect("a fresh store should finish the original cleanup");
let renamed = fixture
.store
.load_table("analytics", "sales", "archived_orders")
.await
.unwrap()
.unwrap();
assert_renamed_maintenance_objects(&fixture.backend, &renamed, &expected).await;
assert!(rename_maintenance_objects(&fixture.backend, &fixture.source).await.is_empty());
assert!(
!fixture
.backend
.object_exists(RUSTFS_META_BUCKET, &rename_maintenance_intent_path("analytics"))
.await
.unwrap()
);
}
#[tokio::test]
async fn maintenance_rename_cannot_overwrite_a_changed_destination_after_interruption() {
let fixture = SidecarRenameFixture::new().await;
let expected_source = rename_maintenance_objects(&fixture.backend, &fixture.source).await;
let path = fixture.destination_config_path();
fixture.backend.fail_after_next_put(RUSTFS_META_BUCKET, &path).await;
assert!(fixture.retry().await.is_err());
let mut configuration = fixture
.store
.backend
.read_object(RUSTFS_META_BUCKET, &fixture.source_config_path())
.await
.unwrap()
.map(|object| serde_json::from_slice::<TableMaintenanceConfig>(&object.data).unwrap())
.unwrap();
configuration.background_enabled = false;
let winner = serde_json::to_vec(&configuration).unwrap();
fixture.backend.seed_object(RUSTFS_META_BUCKET, &path, winner.clone()).await;
let before = maintenance_test_object_versions(&fixture.backend).await;
assert!(matches!(fixture.retry().await, Err(TableCatalogStoreError::Conflict(_))));
assert_eq!(
fixture
.backend
.read_object(RUSTFS_META_BUCKET, &path)
.await
.unwrap()
.unwrap()
.data,
winner
);
assert_eq!(rename_maintenance_objects(&fixture.backend, &fixture.source).await, expected_source);
assert_eq!(maintenance_test_object_versions(&fixture.backend).await, before);
fixture.assert_pending().await;
}
#[tokio::test]
async fn maintenance_rename_cannot_delete_a_recreated_source_cleanup_object() {
let fixture = SidecarRenameFixture::new().await;
let path = fixture.source_config_path();
let configuration = fixture
.store
.get_table_maintenance_config("analytics", "sales", "orders")
.await
.unwrap();
fixture.backend.fail_after_next_delete(RUSTFS_META_BUCKET, &path).await;
assert!(fixture.retry().await.is_err());
let winner = serde_json::to_vec(&configuration).unwrap();
fixture.backend.seed_object(RUSTFS_META_BUCKET, &path, winner.clone()).await;
let before = maintenance_test_object_versions(&fixture.backend).await;
assert!(matches!(fixture.retry().await, Err(TableCatalogStoreError::Conflict(_))));
assert_eq!(
fixture
.backend
.read_object(RUSTFS_META_BUCKET, &path)
.await
.unwrap()
.unwrap()
.data,
winner
);
assert_eq!(maintenance_test_object_versions(&fixture.backend).await, before);
fixture.assert_pending().await;
}
#[tokio::test]
async fn renaming_a_table_without_sidecars_persists_and_cleans_its_empty_plan() {
let (backend, store, original) = planning_fixture("tables/table-id/metadata/00003.metadata.json").await;
let intent_path = rename_maintenance_intent_path("analytics");
assert!(!backend.object_exists(RUSTFS_META_BUCKET, &intent_path).await.unwrap());
store
.rename_table("analytics", "sales", "orders", "sales", "archived_orders")
.await
.unwrap();
let renamed = store
.load_table("analytics", "sales", "archived_orders")
.await
.unwrap()
.unwrap();
assert_eq!(renamed.metadata_location, original.metadata_location);
assert!(rename_maintenance_objects(&backend, &renamed).await.is_empty());
assert!(!backend.object_exists(RUSTFS_META_BUCKET, &intent_path).await.unwrap());
assert_eq!(
backend
.state
.lock()
.await
.put_attempts
.get(&(RUSTFS_META_BUCKET.to_string(), intent_path))
.copied()
.unwrap_or_default(),
1,
"even an empty plan must be durable before maintenance preparation is published"
);
}
#[tokio::test]
async fn missing_empty_maintenance_plan_after_preparation_is_not_reconstructed() {
let (backend, store, _) = planning_fixture("tables/table-id/metadata/00003.metadata.json").await;
let namespace = Namespace::parse("sales").unwrap();
let table = IdentifierSegment::parse("orders").unwrap();
let source_path = store.paths.table_entry_path("analytics", &namespace, &table);
backend.fail_next_put(RUSTFS_META_BUCKET, &source_path).await;
store
.rename_table("analytics", "sales", "orders", "sales", "archived_orders")
.await
.expect_err("stop after durable maintenance preparation but before source fencing");
let rename_id = store
.get_table_bucket("analytics")
.await
.unwrap()
.unwrap()
.active_rename_id
.expect("the publication fence remains active");
let rename_path = store.paths.table_rename_intent_path("analytics", &rename_id);
let (intent, _) = store
.read_entry::<TableRenameIntent>(RUSTFS_META_BUCKET, &rename_path)
.await
.unwrap()
.unwrap();
assert_eq!(intent.state, TableRenameIntentState::MaintenancePrepared);
let maintenance_path = rename_maintenance_intent_path("analytics");
assert!(
backend
.state
.lock()
.await
.objects
.remove(&(RUSTFS_META_BUCKET.to_string(), maintenance_path))
.is_some()
);
let before = maintenance_test_object_versions(&backend).await;
let retry = ObjectTableCatalogStore::new(backend.clone())
.rename_table("analytics", "sales", "orders", "sales", "archived_orders")
.await;
assert!(matches!(retry, Err(TableCatalogStoreError::Unavailable(_))));
assert_eq!(maintenance_test_object_versions(&backend).await, before);
assert!(matches!(
store.load_table("analytics", "sales", "orders").await,
Err(TableCatalogStoreError::Unavailable(_))
));
}
struct ConcurrentCleanupFixture {
backend: TestCatalogObjectBackend,
entries: [TableEntry; 2],
reports: [TableMetadataMaintenanceReport; 2],
orphans: [String; 2],
}
async fn concurrent_cleanup_fixture() -> ConcurrentCleanupFixture {
let first_current = "tables/table-id/metadata/00003.metadata.json";
let second_current = "tables/invoice-id/metadata/00003.metadata.json";
let (mut backend, store, first) = planning_fixture(first_current).await;
let namespace = Namespace::parse("sales").unwrap();
let invoice_table = IdentifierSegment::parse("invoices").unwrap();
let mut second = test_table_entry("analytics", &namespace, &invoice_table, second_current.to_string());
second.table_id = "invoice-id".to_string();
second.table_uuid = "invoice-uuid".to_string();
second.warehouse_location = "s3://analytics/tables/invoice-id".to_string();
backend
.seed_object(
"analytics",
second_current,
serde_json::to_vec(&table_metadata_json_for_backup("analytics", &second.table_id, &second.table_uuid)).unwrap(),
)
.await;
store.create_table(second.clone()).await.unwrap();
let orphans = [
"tables/table-id/metadata/00001.metadata.json".to_string(),
"tables/invoice-id/metadata/00001.metadata.json".to_string(),
];
for orphan in &orphans {
backend.seed_object("analytics", orphan, b"{}".to_vec()).await;
}
let reports = [
store
.plan_table_metadata_maintenance("analytics", "sales", "orders", 0)
.await
.unwrap(),
store
.plan_table_metadata_maintenance("analytics", "sales", "invoices", 0)
.await
.unwrap(),
];
for (report, orphan) in reports.iter().zip(&orphans) {
assert_eq!(report.deletable_metadata_locations, vec![orphan.clone()]);
}
// Both new stores share the original lock map, while reads now exercise the
// real object read-lock boundary that can deadlock under nested owner scans.
backend.serialize_object_reads = true;
ConcurrentCleanupFixture {
backend,
entries: [first, second],
reports,
orphans,
}
}
#[tokio::test]
async fn concurrent_metadata_cleanups_do_not_acquire_each_others_entry_read_locks() {
let fixture = concurrent_cleanup_fixture().await;
let paths = TableCatalogObjectPaths::default();
let owner_prefix = paths.namespace_entries_prefix("analytics");
let first_scan = fixture.backend.pause_next_list_page(RUSTFS_META_BUCKET, &owner_prefix).await;
let second_scan = fixture.backend.pause_next_list_page(RUSTFS_META_BUCKET, &owner_prefix).await;
let before = maintenance_test_object_versions(&fixture.backend).await;
let first_backend = fixture.backend.clone();
let first_report = fixture.reports[0].clone();
let first_cleanup = tokio::spawn(async move {
ObjectTableCatalogStore::new(first_backend)
.delete_table_metadata_maintenance_report("analytics", "sales", "orders", first_report)
.await
});
let second_backend = fixture.backend.clone();
let second_report = fixture.reports[1].clone();
let second_cleanup = tokio::spawn(async move {
ObjectTableCatalogStore::new(second_backend)
.delete_table_metadata_maintenance_report("analytics", "sales", "invoices", second_report)
.await
});
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async {
tokio::join!(first_scan.wait_started(), second_scan.wait_started());
})
.await
.expect("both cleanups must reach owner listing while holding their own entry write lock");
for entry in &fixture.entries {
let path = paths.table_entry_path(
"analytics",
&Namespace::parse(&entry.namespace).unwrap(),
&IdentifierSegment::parse(&entry.table).unwrap(),
);
assert!(fixture.backend.write_lock_is_held(RUSTFS_META_BUCKET, &path).await);
}
first_scan.release();
second_scan.release();
let (first_result, second_result) =
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { tokio::join!(first_cleanup, second_cleanup) })
.await
.expect("independent cleanups must finish instead of waiting on each other's entry lock");
for result in [first_result, second_result] {
let report = result.expect("cleanup task joins").expect("independent cleanup succeeds");
assert_eq!(report.job.deleted_metadata_file_count, 1);
}
let mut expected = before;
for orphan in &fixture.orphans {
expected.remove(&("analytics".to_string(), orphan.clone()));
assert!(!fixture.backend.object_exists("analytics", orphan).await.unwrap());
}
assert_eq!(maintenance_test_object_versions(&fixture.backend).await, expected);
let store = ObjectTableCatalogStore::new(fixture.backend.clone());
for entry in fixture.entries {
assert_eq!(store.load_table("analytics", &entry.namespace, &entry.table).await.unwrap(), Some(entry));
}
}
#[tokio::test]
async fn metadata_cleanup_rejects_an_owner_that_disappears_after_publication_fenced_listing() {
let fixture = concurrent_cleanup_fixture().await;
let paths = TableCatalogObjectPaths::default();
let owner = &fixture.entries[1];
let owner_path = paths.table_entry_path(
"analytics",
&Namespace::parse(&owner.namespace).unwrap(),
&IdentifierSegment::parse(&owner.table).unwrap(),
);
let pause = fixture.backend.pause_before_next_read(RUSTFS_META_BUCKET, &owner_path).await;
let backend = fixture.backend.clone();
let report = fixture.reports[0].clone();
let cleanup = tokio::spawn(async move {
ObjectTableCatalogStore::new(backend)
.delete_table_metadata_maintenance_report("analytics", "sales", "orders", report)
.await
});
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, pause.wait_started())
.await
.expect("cleanup lists the second owner and reaches its object read");
assert!(
fixture
.backend
.state
.lock()
.await
.objects
.remove(&(RUSTFS_META_BUCKET.to_string(), owner_path))
.is_some()
);
let before = maintenance_test_object_versions(&fixture.backend).await;
pause.release();
let result = tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, cleanup)
.await
.expect("missing-owner cleanup returns")
.expect("cleanup task joins");
assert!(matches!(result, Err(TableCatalogStoreError::Conflict(message)) if message.contains("disappeared")));
assert_eq!(maintenance_test_object_versions(&fixture.backend).await, before);
for orphan in &fixture.orphans {
assert!(fixture.backend.object_exists("analytics", orphan).await.unwrap());
}
assert_eq!(
ObjectTableCatalogStore::new(fixture.backend.clone())
.load_table("analytics", "sales", "orders")
.await
.unwrap(),
Some(fixture.entries[0].clone())
);
}
@@ -0,0 +1,363 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
fn planning_report_paths(entry: &TableEntry, job_id: &str) -> [String; 3] {
let paths = TableCatalogObjectPaths::default();
let namespace = Namespace::parse(&entry.namespace).unwrap();
let table = IdentifierSegment::parse(&entry.table).unwrap();
[
paths.table_maintenance_job_path(&entry.table_bucket, &namespace, &table, &entry.table_id, job_id),
paths.table_maintenance_current_job_path(&entry.table_bucket, &namespace, &table, &entry.table_id),
paths.table_maintenance_latest_job_path(&entry.table_bucket, &namespace, &table, &entry.table_id),
]
}
async fn assert_only_planning_sidecars_changed(
backend: &TestCatalogObjectBackend,
before: BTreeMap<(String, String), TableCatalogObject>,
entry: &TableEntry,
report: &TableMetadataMaintenanceReport,
) {
let permitted = planning_report_paths(entry, &report.job.job_id)
.map(|path| (RUSTFS_META_BUCKET.to_string(), path))
.into_iter()
.collect::<BTreeSet<_>>();
let after = maintenance_test_object_versions(backend).await;
for key in &permitted {
let persisted = after.get(key).expect("all three report views must be published");
assert_eq!(
serde_json::from_slice::<TableMetadataMaintenanceReport>(&persisted.data).unwrap(),
*report
);
}
let unchanged_before = before
.into_iter()
.filter(|(key, _)| !permitted.contains(key))
.collect::<BTreeMap<_, _>>();
let unchanged_after = after
.into_iter()
.filter(|(key, _)| !permitted.contains(key))
.collect::<BTreeMap<_, _>>();
assert_eq!(
unchanged_after, unchanged_before,
"planning must not change catalog, metadata, manifests or data"
);
}
#[tokio::test]
async fn durable_strong_planning_uses_authoritative_entry_and_persists_only_audited_sidecars() {
for requested_retention in [0, 2] {
let backend = TestCatalogObjectBackend::default();
let store = StrongTableCatalogStore::new(backend.clone());
let entry = seed_strong_maintenance_table(&store, &backend).await;
let namespace = Namespace::parse(&entry.namespace).unwrap();
let table = IdentifierSegment::parse(&entry.table).unwrap();
let orphan = default_table_metadata_file_path(&namespace, &table, "00001.metadata.json");
let recent = default_table_metadata_file_path(&namespace, &table, "00002.metadata.json");
for path in [&orphan, &recent] {
backend
.seed_object(&entry.table_bucket, path, br#"{"metadata-log":[]}"#.to_vec())
.await;
}
let config = TableMaintenanceConfig {
retain_recent_metadata_files: 3,
delete_enabled: true,
max_retry_attempts: 2,
..Default::default()
};
store
.put_table_maintenance_config(&entry.table_bucket, &entry.namespace, &entry.table, config.clone())
.await
.unwrap();
assert!(
ObjectTableCatalogStore::new(backend.clone())
.load_table(&entry.table_bucket, &entry.namespace, &entry.table)
.await
.unwrap()
.is_none()
);
let before = maintenance_test_object_versions(&backend).await;
let report = store
.run_table_metadata_maintenance_planning(
&entry.table_bucket,
&entry.namespace,
&entry.table,
requested_retention,
None,
None,
)
.await
.unwrap();
assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Successful);
assert_eq!(report.job.operation, TableMetadataMaintenanceOperation::DryRun);
assert_eq!(report.job.table_id, entry.table_id);
assert_eq!(report.current_metadata_location, entry.metadata_location);
assert_eq!(report.job.current_generation, entry.generation);
assert_eq!(report.job.config_source, TableMaintenanceConfigSource::TableOverride);
assert_eq!(report.job.retain_recent_metadata_files, if requested_retention == 0 { 3 } else { 2 });
assert_eq!(report.job.max_retry_attempts, 2);
assert_eq!(report.job.worker_id.as_deref(), Some("rustfs-admin"));
assert_eq!(report.job.attempt, 1);
assert!(report.job.started_at.is_some() && report.job.finished_at.is_some());
assert_eq!(report.job.deleted_metadata_file_count, 0);
assert_eq!(report.job.deleted_object_count, 0);
assert_eq!(
report.cleanup_candidate_locations,
if requested_retention == 0 { vec![] } else { vec![orphan] }
);
for action in [
TableMaintenanceAuditAction::Planned,
TableMaintenanceAuditAction::WorkerStarted,
TableMaintenanceAuditAction::WorkerSucceeded,
] {
assert!(report.audit_events.iter().any(|event| event.action == action));
}
assert_only_planning_sidecars_changed(&backend, before, &entry, &report).await;
let reloaded = StrongTableCatalogStore::new(backend.clone());
assert_eq!(
reloaded
.load_table(&entry.table_bucket, &entry.namespace, &entry.table)
.await
.unwrap()
.unwrap(),
entry
);
assert_eq!(
reloaded
.get_table_maintenance_config(&entry.table_bucket, &entry.namespace, &entry.table)
.await
.unwrap(),
config
);
assert_eq!(
reloaded
.get_table_metadata_maintenance_report(&entry.table_bucket, &entry.namespace, &entry.table, &report.job.job_id)
.await
.unwrap()
.unwrap(),
report
);
}
}
#[tokio::test]
async fn durable_strong_planning_rejects_pending_work_quarantine_and_incomplete_report_views() {
let backend = TestCatalogObjectBackend::default();
let store = StrongTableCatalogStore::new(backend.clone());
let entry = seed_strong_maintenance_table(&store, &backend).await;
let previous = store
.run_table_metadata_maintenance_planning("analytics", "sales", "orders", 0, None, None)
.await
.unwrap();
let paths = planning_report_paths(&entry, &previous.job.job_id);
for (status, quarantined) in [
(TableMetadataMaintenanceJobStatus::Queued, 0),
(TableMetadataMaintenanceJobStatus::Running, 0),
(TableMetadataMaintenanceJobStatus::Failed, 0),
(TableMetadataMaintenanceJobStatus::Successful, 1),
] {
let mut pending = previous.clone();
pending.job.status = status;
pending.job.quarantined_object_count = quarantined;
pending.job.quarantine_enabled = quarantined > 0;
pending.job.quarantine_retention_seconds = if quarantined > 0 { 86_400 } else { 0 };
for path in &paths {
backend
.seed_object(RUSTFS_META_BUCKET, path, serde_json::to_vec(&pending).unwrap())
.await;
}
let before = maintenance_test_object_versions(&backend).await;
assert_matches!(
store
.run_table_metadata_maintenance_planning("analytics", "sales", "orders", 0, None, None)
.await,
Err(TableCatalogStoreError::Conflict(_))
);
assert_eq!(maintenance_test_object_versions(&backend).await, before);
}
for mismatched_view in [0, 2] {
for path in &paths {
backend
.seed_object(RUSTFS_META_BUCKET, path, serde_json::to_vec(&previous).unwrap())
.await;
}
let mut inconsistent = previous.clone();
inconsistent.job.attempt += 1;
backend
.seed_object(RUSTFS_META_BUCKET, &paths[mismatched_view], serde_json::to_vec(&inconsistent).unwrap())
.await;
let before = maintenance_test_object_versions(&backend).await;
assert_matches!(
store
.run_table_metadata_maintenance_planning("analytics", "sales", "orders", 0, None, None)
.await,
Err(TableCatalogStoreError::Conflict(_))
);
assert_eq!(maintenance_test_object_versions(&backend).await, before);
}
}
#[tokio::test]
async fn durable_strong_planning_revalidates_config_entry_and_job_after_unlocked_reads() {
for change in ["config", "entry", "job", "completed-job"] {
let backend = TestCatalogObjectBackend::default();
let store = Arc::new(StrongTableCatalogStore::new(backend.clone()));
let entry = seed_strong_maintenance_table(&store, &backend).await;
let pause = backend.pause_next_read("analytics", &entry.metadata_location).await;
let planning_store = Arc::clone(&store);
let planning = tokio::spawn(async move {
planning_store
.run_table_metadata_maintenance_planning("analytics", "sales", "orders", 0, None, None)
.await
});
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, pause.wait_started())
.await
.expect("planning must reach metadata read");
match change {
"config" => {
let mut config = store
.get_table_maintenance_config("analytics", "sales", "orders")
.await
.unwrap();
config.max_retry_attempts += 1;
tokio::time::timeout(
TABLE_CATALOG_TEST_TIMEOUT,
store.put_table_maintenance_config("analytics", "sales", "orders", config),
)
.await
.expect("planning must release config publication lock")
.unwrap();
}
"entry" => {
let next = default_table_metadata_file_path(
&Namespace::parse("sales").unwrap(),
&IdentifierSegment::parse("orders").unwrap(),
"00004.metadata.json",
);
backend.seed_object("analytics", &next, b"{}".to_vec()).await;
tokio::time::timeout(
TABLE_CATALOG_TEST_TIMEOUT,
store.commit_table(TableCommitRequest {
table_bucket: "analytics".to_string(),
namespace: "sales".to_string(),
table: "orders".to_string(),
commit_id: "concurrent-planning-commit".to_string(),
idempotency_key: None,
operation: "append".to_string(),
expected_version_token: entry.version_token.clone(),
expected_metadata_location: entry.metadata_location.clone(),
new_metadata_location: next,
requirements: Vec::new(),
writer: Some("planning-test".to_string()),
}),
)
.await
.expect("planning must release catalog publication lock")
.unwrap();
}
"job" => {
let winning = tokio::time::timeout(
TABLE_CATALOG_TEST_TIMEOUT,
store.run_table_maintenance_scheduler_once(
"analytics",
"sales",
"orders",
"concurrent-scheduler".to_string(),
),
)
.await
.expect("planning must release job publication lock")
.unwrap();
assert_eq!(winning.report.job.status, TableMetadataMaintenanceJobStatus::Queued);
}
"completed-job" => {
let winner = tokio::time::timeout(
TABLE_CATALOG_TEST_TIMEOUT,
store.run_table_metadata_maintenance_planning("analytics", "sales", "orders", 0, None, None),
)
.await
.expect("a second planner must complete while the first reads metadata")
.unwrap();
assert_eq!(winner.job.status, TableMetadataMaintenanceJobStatus::Successful);
}
_ => unreachable!(),
}
let before_publish = maintenance_test_object_versions(&backend).await;
pause.release();
let result = tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, planning)
.await
.unwrap()
.unwrap();
if change == "completed-job" {
assert_matches!(result, Err(TableCatalogStoreError::Conflict(reason))
if reason == "current maintenance job changed during planning");
} else {
assert_matches!(result, Err(TableCatalogStoreError::Conflict(_)));
}
assert_eq!(
maintenance_test_object_versions(&backend).await,
before_publish,
"stale {change} planning must publish nothing"
);
}
}
#[tokio::test]
async fn durable_strong_planning_partial_terminal_publication_preserves_safe_retry_boundary() {
// A named-only completed plan is harmless history. A published latest alias
// without its current alias is observable incomplete publication and conflicts.
for failed_view in [1, 2] {
let backend = TestCatalogObjectBackend::default();
let store = StrongTableCatalogStore::new(backend.clone());
let entry = seed_strong_maintenance_table(&store, &backend).await;
let paths = planning_report_paths(&entry, "unused");
let before = maintenance_test_object_versions(&backend).await;
backend.fail_next_put(RUSTFS_META_BUCKET, &paths[failed_view]).await;
let result = store
.run_table_metadata_maintenance_planning("analytics", "sales", "orders", 0, None, None)
.await;
assert!(result.is_err(), "an interrupted report publication must not return success");
let interrupted = maintenance_test_object_versions(&backend).await;
for (key, value) in &before {
assert_eq!(interrupted.get(key), Some(value));
}
let (named_key, named_value) = interrupted
.iter()
.find(|(key, _)| !before.contains_key(*key) && key.1 != paths[1] && key.1 != paths[2])
.expect("the completed named report must be retained");
let original: TableMetadataMaintenanceReport = serde_json::from_slice(&named_value.data).unwrap();
assert_eq!(original.job.status, TableMetadataMaintenanceJobStatus::Successful);
assert_eq!(original.job.operation, TableMetadataMaintenanceOperation::DryRun);
assert_eq!(original.job.deleted_object_count, 0);
assert_eq!(original.job.deleted_metadata_file_count, 0);
let retry = store
.run_table_metadata_maintenance_planning("analytics", "sales", "orders", 0, None, None)
.await;
if failed_view == 2 {
assert_eq!(interrupted.len(), before.len() + 1);
let retried = retry.expect("named-only completed dry-run history permits safe replanning");
assert_ne!(retried.job.job_id, original.job.job_id);
assert_eq!(retried.job.status, TableMetadataMaintenanceJobStatus::Successful);
assert_eq!(maintenance_test_object_versions(&backend).await.get(named_key), Some(named_value));
assert_only_planning_sidecars_changed(&backend, interrupted.clone(), &entry, &retried).await;
} else {
assert_eq!(interrupted.len(), before.len() + 2);
assert_matches!(retry, Err(TableCatalogStoreError::Conflict(_)));
assert_eq!(maintenance_test_object_versions(&backend).await, interrupted);
}
assert_eq!(store.load_table("analytics", "sales", "orders").await.unwrap().unwrap(), entry);
}
}
@@ -0,0 +1,166 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
#[tokio::test]
async fn durable_strong_planning_renamed_paths_preserve_identity_and_fail_closed() {
for (target_namespace, delete_content, foreign_manifest) in [
("sales", None, false),
("archive", None, false),
("archive", Some(1), false),
("archive", Some(2), false),
("archive", None, true),
] {
let backend = TestCatalogObjectBackend::default();
let store = StrongTableCatalogStore::new(backend.clone());
let entry = seed_strong_maintenance_table(&store, &backend).await;
let bucket = entry.table_bucket.as_str();
let namespace = Namespace::parse("sales").unwrap();
let table = IdentifierSegment::parse("orders").unwrap();
let metadata_dir = default_table_metadata_dir_path(&namespace, &table);
let data_dir = default_table_data_dir_path(&namespace, &table);
let manifest_list = format!("{metadata_dir}/snap-20.avro");
let manifest = if foreign_manifest {
default_table_metadata_file_path(&namespace, &IdentifierSegment::parse("foreign").unwrap(), "manifest.avro")
} else {
format!("{metadata_dir}/manifest-20.avro")
};
let left = format!("{data_dir}/left.parquet");
let right = format!("{data_dir}/right.parquet");
let delete = format!("{}/delete.parquet", default_table_delete_dir_path(&namespace, &table));
let mut files = vec![(left.as_str(), 0), (right.as_str(), 0)];
if let Some(content) = delete_content {
files.push((delete.as_str(), content));
backend.seed_object(bucket, &delete, b"delete fixture".to_vec()).await;
}
let manifest_bytes = manifest_avro_bytes(&files);
backend
.seed_object(bucket, &manifest_list, manifest_list_avro_bytes(&[(&manifest, manifest_bytes.len())]))
.await;
backend.seed_object(bucket, &manifest, manifest_bytes).await;
backend.seed_object(bucket, &left, parquet_i32_bytes(&[1])).await;
backend.seed_object(bucket, &right, parquet_i32_bytes(&[2])).await;
backend
.seed_object(
bucket,
&entry.metadata_location,
serde_json::to_vec(&serde_json::json!({
"format-version": 2,
"table-uuid": entry.table_uuid,
"location": entry.warehouse_location,
"metadata-log": [],
"snapshots": [
{"snapshot-id": 10, "timestamp-ms": 1000, "manifest-list": manifest_list},
{"snapshot-id": 20, "timestamp-ms": 2000, "manifest-list": manifest_list}
],
"current-snapshot-id": 20,
"refs": {"main": {"snapshot-id": 20, "type": "branch"}}
}))
.unwrap(),
)
.await;
if target_namespace != "sales" {
store
.create_namespace(test_namespace_entry(bucket, &Namespace::parse(target_namespace).unwrap()))
.await
.unwrap();
}
store
.rename_table(bucket, "sales", "orders", target_namespace, "renamed")
.await
.unwrap();
let renamed = store.load_table(bucket, target_namespace, "renamed").await.unwrap().unwrap();
let before = maintenance_test_object_versions(&backend).await;
let report = store
.run_table_metadata_maintenance_planning(
bucket,
target_namespace,
"renamed",
0,
Some(TableSnapshotExpirationConfig {
min_snapshots_to_keep: 1,
max_snapshot_age_ms: 1000,
}),
Some(TableCompactionPlanningConfig {
target_file_size_bytes: 1024 * 1024,
small_file_threshold_bytes: 64 * 1024,
min_input_files: 2,
max_rewrite_bytes_per_job: 2 * 1024 * 1024,
}),
)
.await
.unwrap();
assert_eq!(report.job.operation, TableMetadataMaintenanceOperation::DryRun);
assert_eq!(report.job.status, TableMetadataMaintenanceJobStatus::Successful);
assert_eq!(report.job.deleted_metadata_file_count, 0);
assert_eq!(report.job.deleted_object_count, 0);
let expiration = report.snapshot_expiration.as_ref().unwrap();
assert_eq!(expiration.namespace, target_namespace);
assert_eq!(expiration.table, "renamed");
assert_eq!(expiration.table_id, entry.table_id);
assert_eq!(expiration.current_metadata_location, entry.metadata_location);
assert_eq!(expiration.expiration_candidate_count, 1);
assert!(expiration.committed_metadata_location.is_none());
let compaction = report.compaction.as_ref().unwrap();
assert_eq!(compaction.namespace, target_namespace);
assert_eq!(compaction.table, "renamed");
assert_eq!(compaction.table_id, entry.table_id);
assert_eq!(compaction.current_metadata_location, entry.metadata_location);
assert_eq!(compaction.current_snapshot_id, Some(20));
assert!(compaction.committed_metadata_location.is_none());
if delete_content.is_some() || foreign_manifest {
assert_eq!(compaction.status, TableCompactionPlanningStatus::ManualReviewRequired);
assert!(compaction.rewrite_groups.is_empty());
assert!(compaction.manual_review_count > 0);
if delete_content.is_some() {
assert_eq!(compaction.row_level_planning.delete_file_count, 1);
}
} else {
assert_eq!(compaction.status, TableCompactionPlanningStatus::RewriteCandidates);
assert_eq!(compaction.candidate_file_count, 2);
assert_eq!(compaction.rewrite_group_count, 1);
assert_eq!(compaction.rewrite_groups[0].input_file_locations, vec![left, right]);
assert!(compaction.rewrite_groups[0].output_file_location.is_none());
}
let after = maintenance_test_object_versions(&backend).await;
for (key, value) in &before {
assert_eq!(after.get(key), Some(value), "planning changed an existing object: {key:?}");
}
assert_eq!(after.len(), before.len() + 3, "only named/current/latest reports may be added");
let reloaded = StrongTableCatalogStore::new(backend.clone());
assert_eq!(
reloaded
.load_table(bucket, target_namespace, "renamed")
.await
.unwrap()
.unwrap(),
renamed
);
for job in [
report.job.job_id.as_str(),
MAINTENANCE_JOB_ALIAS_CURRENT,
MAINTENANCE_JOB_ALIAS_LATEST,
] {
assert_eq!(
reloaded
.get_table_metadata_maintenance_report(bucket, target_namespace, "renamed", job)
.await
.unwrap()
.unwrap(),
report
);
}
}
}
+6 -11
View File
@@ -734,9 +734,11 @@ def check_ilm_build_budget(root: Path) -> list[str]:
if len(builds) != 1:
return ["ILM integration requires one workspace archive producer"]
budget = builds[0].get("env", {}).get("CARGO_BUILD_JOBS", "")
# Execution reuses this archive and does not invoke Cargo builds.
if str(budget) != "2":
return ["workspace archive must use the measured two-worker Cargo build budget"]
# This remains the measured build budget; execution has no Cargo build.
compact = re.sub(r"\s+", "", budget)
expected = "${{(github.event_name=='push'||github.event_name=='workflow_dispatch')&&'3'||'2'}}"
if compact != expected:
return ["workspace archive must keep the measured 3/2 Cargo build budget"]
except (OSError, KeyError, TypeError, ValueError) as error:
return [f"invalid ILM workflow contract: {error}"]
return []
@@ -5641,12 +5643,7 @@ class SelfTests(unittest.TestCase):
workflow.write_text(valid)
self.assertEqual(check_ilm_build_budget(root), [])
for old, new in (
('CARGO_BUILD_JOBS: "2"', 'CARGO_BUILD_JOBS: "3"'),
(
'CARGO_BUILD_JOBS: "2"',
"CARGO_BUILD_JOBS: ${{ (github.event_name == 'push' "
"|| github.event_name == 'workflow_dispatch') && '3' || '2' }}",
),
("&& '3' || '2' }}", "&& '3' || '3' }}"),
("outputs.mode == 'full'", "outputs.mode == 'docs'"),
("--profile default -j1", "--profile ci -j1"),
("-j1 --run-ignored", "-j2 --run-ignored"),
@@ -5664,8 +5661,6 @@ class SelfTests(unittest.TestCase):
workflow.write_text(valid.replace(old, new))
self.assertTrue(check_ilm_build_budget(root), (old, new))
# Formatting and the consumer job ID are not execution contracts.
workflow.write_text(valid.replace('CARGO_BUILD_JOBS: "2"', 'CARGO_BUILD_JOBS: 2'))
self.assertEqual(check_ilm_build_budget(root), [])
workflow.write_text(valid.replace(" workspace-tests:", " renamed-workspace-tests:"))
self.assertEqual(check_ilm_build_budget(root), [])
+40 -34
View File
@@ -40,7 +40,7 @@ python3 scripts/table-catalog/pyiceberg_smoke.py \
--cleanup \
--rustfs-build rustfs-v1.0.0-beta.8 \
--git-sha "$(git rev-parse HEAD)" \
--catalog-backing object \
--catalog-backing durable-strong \
--live-evidence-output /tmp/rustfs-pyiceberg-live-evidence.json
```
@@ -58,18 +58,19 @@ instead of support-matrix proof.
The smoke reads `rustfs.catalog-backing` from `/v1/config`, applying overrides
after defaults, before creating objects. `--catalog-backing` is an optional
expected value, not a way to relabel the deployment. A mismatch or an unknown
server backing aborts the run. The only supported wire value is `object`; the
server backing aborts the run. Object mode uses the wire value `object`; the
legacy operator label `object-backed` is accepted as an expected-value alias.
A retired backing value fails closed instead of silently selecting another
implementation. `--client-version` likewise checks the installed PyIceberg
version rather than
Durable-strong must already be provisioned through the
[cutover procedure](../../docs/operations/s3-tables-cutover-runbook.md).
`--client-version` likewise checks the installed PyIceberg version rather than
overriding the version recorded in evidence. A mismatch aborts before setup.
New PyIceberg evidence includes `catalog_probes`, `rest_signing_name`, and
`credential_vending_required`. Probe results distinguish `pass` and `skipped`.
The object catalog exercises maintenance configuration, scheduler/worker
dry-run, job inspection, quarantine inspection, diagnostics, export, and the
catalog pointer/token/generation checks. A completed dry-run must retain its
`credential_vending_required`. Probe results distinguish `pass`,
`expected-unsupported`, and `skipped`. Both backings exercise maintenance
configuration, scheduler/worker dry-run, job inspection, quarantine inspection,
diagnostics, and export. Only the direct metadata-maintenance endpoint is
expected to reject durable-strong requests. A completed dry-run must retain its
queued job identity, report zero deletions, and leave the commit state unchanged;
the smoke restores its disabled, paused configuration after execution.
New full runs use `create-append-reload-scan-backing-contracts-v2`; historical
@@ -93,7 +94,9 @@ The smoke test covers:
- round-trip an encoded multi-level namespace and reject double-encoded aliases
- check LoadTable does not vend credentials for missing or unknown delegation
tokens; when vending is required, check exact-token negotiation and scope
- check the object catalog maintenance dry-run and export against the current pointer, token, and generation
- on both backings, check a successful maintenance dry-run and export against
the current pointer, token, and generation; in durable-strong mode, also
verify the direct metadata-maintenance rejection leaves commit state unchanged
- optionally drop the table and namespace
The default profile uses the canonical RustFS catalog URI:
@@ -167,7 +170,8 @@ temporary credentials:
rejected.
The direct REST catalog probes run by default after the PyIceberg append and
scan. The smoke runs maintenance control, diagnostics, and catalog export probes. Backup and restore require a separate recovery rehearsal.
scan. Both backings run maintenance control, diagnostics, and export probes.
Durable-strong checks the direct metadata-maintenance rejection separately.
For deployments that intentionally expose only the core Iceberg REST
Catalog table path, skip those probes explicitly:
@@ -329,22 +333,24 @@ The smoke test also probes catalog-backed advanced Iceberg surfaces:
inspected, released, retried, or abandoned through an operator endpoint
- catalog diagnostics exposes the table recovery and consistency state used by
operators
- catalog export and diagnostics expose the object catalog recovery state,
commit-log and idempotency information, single-active-writer policy, and scale
validation matrix. Metadata scans follow the persisted directory, including
renamed tables and same-bucket S3 locations. Current metadata, metadata-log
entries, and snapshot references remain protected while maintenance plans are
evaluated. Orphan candidates are read-only observations until a controlled
maintenance operation authorizes deletion.
- catalog backup creates object-catalog artifact v2 with catalog/current table metadata and a bounded inventory of current metadata, direct `metadata-log` history, current view metadata, manifest lists/manifests, live data/delete files, and table/partition statistics. Objects up to 16 MiB are verified by size/SHA-256, allowing a different target ETag for identical bytes; larger objects must retain the recorded size/ETag. External object payloads are not packaged, and the existing v1 contract is unchanged.
`POST /iceberg/v1/{warehouse}/catalog/backup/export` accepts strict `{"backup-id":"..."}` JSON and returns the saved artifact bytes. `POST /iceberg/v1/{warehouse}/catalog/backup/import` accepts those raw bytes, validates the strict versioned artifact, and saves a candidate without publishing catalog or metadata objects; its status is `CREATED` or `ALREADY_PRESENT`. Raw artifacts are limited to 64 MiB. Existing object-catalog v1 artifacts retain their original bytes and legacy catalog/current-table-metadata contract; transfer does not add a v2 external-object inventory.
`POST /iceberg/v1/{warehouse}/catalog/backup/prepare` accepts strict `{"backup-id":"...","object":"...","data-base64":"..."}` JSON. Unknown or duplicate fields and null required values are rejected. The payload uses standard padded Base64; encoded length is bounded before decoding, decoded bytes are limited to 16 MiB, and the JSON body is limited to 24 MiB. Import and prepare retain the 30-second request-body timeout. Preparation checks the saved v2 reference, size/SHA-256, kind, and active table/view scope before an `IfAbsent` write. It returns HTTP 200 with `PREPARED` or `ALREADY_PRESENT` and private/no-store headers; exact-byte retries preserve the target ETag, different existing bytes are never overwritten, and catalog publication remains a separate restore operation. An interrupted restore permits preparation only for its matching backup and validated persistent plan, without releasing its barrier.
Provide original external object bytes through an independent object-backup/restore process or the controlled small-object preparation endpoint; no generic external-object export API is implied here. Current table metadata is already embedded and can be installed by restore. For objects outside the current warehouse scope, prepare the corresponding direct history metadata first; scope lookup is bounded to 1,024 history objects and 64 MiB of declared metadata bytes per request. Recovery uses the same table-bucket name, keys, locations, and Iceberg bytes, without file-format or key migration. Arbitrary bucket/key/location remapping is not provided. Larger objects still require the recorded ETag; a multipart transfer is not guaranteed to preserve it.
These extensions require a table-enabled warehouse and `admin:MigrateTableCatalog`, and also accept `/_iceberg/v1`. Keep the target closed to clients during restore and after an interruption until restore and recovery verification finish; stop all writers and maintenance workers for offline disaster recovery or rollback. All serving/background nodes must understand persistent restore controls, or older nodes/workers must remain out of target service until verification completes. `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` is not proof of restore-control version compatibility, and mixed old/new restore isolation is not claimed. Per-object CAS alone is not a bucket-wide atomic publication or crash-isolation guarantee.
- catalog export and diagnostics expose the current catalog backing manifest,
recoverable commit-log WAL state, strong backing migration target, single
active writer HA policy, and scale validation matrix; durable-strong responses
additionally identify the snapshot path, ETag, and format version, and
diagnostics retry a bounded number of times when the strong snapshot changes
during inspection, then fail closed if no stable observation is available.
Metadata scans follow the persisted directory, including renamed tables and
same-bucket S3 locations. Until strong maintenance configuration is persisted,
diagnostics add no count-based retention; current metadata, metadata-log entries,
and snapshot refs remain protected. Orphan candidates are read-only observations,
not deletion authorization. Object-backed reports share one recovery observation
but do not provide a cross-object linearizable snapshot.
diagnostics fail closed if the strong snapshot changes during inspection;
until strong maintenance configuration is persisted, diagnostics use a
conservative zero-file retention default. Durable catalog backup and restore
are controlled durable-strong operations; backup artifacts record catalog and
referenced-object watermarks, while restore verifies them and retains a
recoverable intent when maintenance-state finalization is interrupted.
## Client Matrix
@@ -380,7 +386,7 @@ The current failure matrix covers:
commit
- external catalog sync conflicts leaving pointer, token, and generation
unchanged
- object backup/restore remaining blocked until recovery and referenced-object validation are clean
- backing migration remaining blocked until WAL/recovery replay is clean
Do not promote a failure case from `probe-required` or `load-test-required` to
an automated claim until the live probe or stress harness is repeatable and its
@@ -415,14 +421,14 @@ The generated phases cover:
- baseline capture through catalog export and `loadTable`
- recovery diagnostics and safe idempotency/commit repair
- operator-selected rollback or import of a known metadata location
- object catalog backup/restore blocker and retry checks
- durable backing migration dry-run blocker checks
- post-recovery `loadTable` and table warehouse data-plane policy probes
Record the RustFS build, catalog backing mode, table identifier, metadata
location, and expected response status for each run. Treat recovery blockers,
location, and expected response status for each run. Treat migration blockers,
manual-review diagnostics, stale rollback/import conflicts, and data-plane
policy failures as fail-closed results that require operator investigation
before recovery or release claims.
before cutover or release claims.
## Scale And Fault Rehearsal
@@ -442,7 +448,7 @@ python3 scripts/table-catalog/failure_coverage.py \
--writer-count 8 \
--maintenance-worker-count 2 \
--iteration-count 50 \
--catalog-backing object \
--catalog-backing durable-strong \
--print-scale-fault-rehearsal
```
@@ -458,7 +464,7 @@ The generated phases cover:
conflict group and retryable conflicts for stale writers
- maintenance scheduler queueing, worker claim, lease expiry recovery, and stale
plan fail-closed checks
- object catalog backup/restore, including graph validation and retry evidence
- durable backing migration dry-run and catalog backup evidence before cutover
- safe recovery repair, rollback, and import conflict behavior after stress
- final `loadTable`, table data-plane policy, and operator artifact capture
@@ -469,7 +475,7 @@ failed-closed operations before using the run as release evidence.
## Production Operations Guide
The engine helper can print a machine-readable operations guide that ties client
conformance, object catalog backup/restore, maintenance, recovery, permissions, and
conformance, durable backing cutover, maintenance, recovery, permissions, and
unsupported-claim governance to the exact evidence operators must record:
```bash
+17 -19
View File
@@ -741,8 +741,8 @@ def live_conformance_evidence(
def pyiceberg_catalog_probe_results(backing: str, *, skipped: bool, vended: bool) -> dict[str, str]:
if backing != "object":
raise ValueError("PyIceberg probe evidence requires the object catalog backing")
if backing not in {"object", "durable-strong"}:
raise ValueError("PyIceberg probe evidence requires an observed RustFS backing")
if skipped:
return {"direct-rest": "skipped"}
probes = dict.fromkeys(("metadata-location", "refs", "views-and-namespace-properties", "rename", "load-table-delegation"), "pass")
@@ -751,7 +751,7 @@ def pyiceberg_catalog_probe_results(backing: str, *, skipped: bool, vended: bool
"maintenance-config", "maintenance-scheduler", "maintenance-worker-dry-run",
"maintenance-job", "maintenance-quarantine-inspect", "diagnostics", "export",
), "pass"))
probes["metadata-maintenance"] = "pass"
probes["metadata-maintenance"] = "expected-unsupported" if backing == "durable-strong" else "pass"
return probes
@@ -1029,35 +1029,33 @@ def production_operations_guide(
),
OrderedDict(
[
("name", "catalog-backup-restore"),
("objective", "Verify object catalog backup and restore remain safe under the recorded pointer and recovery state."),
("commands", [
f"POST {warehouse_catalog_path(warehouse=warehouse, rest_path=rest_path, suffix='/catalog/backup')}",
f"POST {warehouse_catalog_path(warehouse=warehouse, rest_path=rest_path, suffix='/catalog/restore')}",
]),
("name", "catalog-backing-cutover"),
("objective", "Verify durable backing readiness before selecting a non-object-backed catalog mode."),
("commands", [f"GET {warehouse_catalog_path(warehouse=warehouse, rest_path=rest_path, suffix='/catalog/migration')}"]),
(
"required_evidence",
[
"catalog backup artifact and referenced-object watermarks",
"commit recovery and idempotency state",
"target pointer and generation before and after restore",
"restore interruption and retry result",
"migration dry-run response",
"commit recovery blockers",
"idempotency index readiness",
"warehouse prefix index readiness",
"rollback configuration",
],
),
(
"pass_criteria",
[
"backup validates the complete Iceberg metadata and manifest/data/delete object graph",
"restore either advances through the expected CAS or returns a retryable conflict",
"a repeated restore with the same backup is idempotent",
"blockers is empty",
"recommended actions are completed",
"object-backed catalog backup exists before cutover",
],
),
(
"fail_closed_signals",
[
"missing or changed referenced objects",
"stale target pointer or generation",
"interrupted finalization requires the same backup to be retried",
"non-empty blockers",
"recoverable commit gaps require repair",
"warehouse prefix index is missing or stale",
],
),
]
+39 -42
View File
@@ -68,10 +68,10 @@ def production_failure_matrix() -> list[dict[str, str]]:
"external catalog sync must not advance pointer/token/generation on UUID mismatch or stale external metadata token",
),
failure_case(
"backup-restore-blocked",
"backing-migration-blocked",
"diagnostics-probe-required",
"blocked-until-recovery-and-restore-are-clean",
"catalog export and diagnostics must expose recovery blockers before object backup or restore",
"blocked-until-recovery-and-replay-are-clean",
"catalog export and diagnostics must expose WAL/recovery blockers before any strong backing cutover",
),
]
@@ -225,16 +225,16 @@ def disaster_recovery_rehearsal_plan(
"mode": "manual-or-ci-optional",
"ci_gate": "RUSTFS_TABLE_CATALOG_DR_REHEARSAL=1",
"preconditions": [
"record the RustFS build, object catalog mode, node count, and object backend before starting",
"record the RustFS build and catalog backing mode before starting",
"run against a disposable table or a backed-up table warehouse",
"capture the current metadata location and version token from loadTable",
"keep the object catalog and referenced Iceberg objects available until restore validation is complete",
"keep object-backed catalog state available until durable backing cutover is accepted",
],
"expected_invariants": [
"current metadata pointer remains recoverable or deliberately rolled back",
"recovery repair does not move the table pointer",
"rollback/import actions require explicit operator-selected metadata locations",
"object backup and restore remain blocked while recovery blockers or changed references are present",
"durable backing cutover remains blocked while migration blockers are present",
"post-recovery loadTable and table data-plane policy checks still succeed",
],
"phases": [
@@ -319,23 +319,22 @@ def disaster_recovery_rehearsal_plan(
],
),
rehearsal_phase(
"backup-restore-preflight",
"Verify object catalog backup and restore remain explainable and fail-closed.",
"migration-preflight",
"Verify durable backing cutover remains explainable and fail-closed.",
[
probe_step(
"object-catalog-backup",
"POST",
warehouse_path(warehouse, "/catalog/backup", rest_path),
"durable-backing-migration-dry-run",
"GET",
warehouse_path(warehouse, "/catalog/migration", rest_path),
"200",
"backup records the current pointer, recovery state, and complete referenced Iceberg object graph",
"migration blockers must be empty and status must be READY_TO_SNAPSHOT before state transfer",
),
probe_step(
"object-catalog-restore",
"durable-backing-migration-materialize",
"POST",
warehouse_path(warehouse, "/catalog/restore", rest_path),
"200-or-409",
"restore validates the target and publishes through object-backed CAS or returns a retryable conflict",
{"backup-id": "recorded-backup-id", "allow-replace": False},
warehouse_path(warehouse, "/catalog/migration", rest_path),
"200",
"source writes are fenced and the durable strong snapshot is materialized before cutover",
),
],
),
@@ -380,7 +379,7 @@ def scale_fault_rehearsal_plan(
writer_count: int = 8,
maintenance_worker_count: int = 2,
iteration_count: int = 50,
catalog_backing: str = "object",
catalog_backing: str = "object-backed-or-durable-strong",
) -> dict[str, Any]:
table_endpoint = table_path(warehouse, namespace, table, rest_path=rest_path)
table_warehouse_location = table_warehouse_location or f"s3://{warehouse}/tables/table-id"
@@ -395,7 +394,7 @@ def scale_fault_rehearsal_plan(
},
"preconditions": [
"run against a disposable table or a table restored from backup",
"record the RustFS build, object catalog mode, node count, and object backend before starting",
"record the RustFS build, catalog backing mode, node count, and object backend before starting",
"capture baseline loadTable metadata location, version token, generation, and warehouse prefix",
"enable only opt-in clients and workers for this rehearsal run",
],
@@ -404,8 +403,8 @@ def scale_fault_rehearsal_plan(
"each conflicting writer cohort has one winner and retryable conflicts for stale writers",
"maintenance jobs never delete reachable table objects",
"scheduler and worker leases recover without duplicate active jobs for the same table",
"object backup and restore stay blocked while recovery or replay blockers exist",
"backup/restore under load either commits a validated metadata location or conflicts without pointer movement",
"durable backing migration stays blocked while recovery or replay blockers exist",
"rollback/import under load either commits a validated metadata location or conflicts without pointer movement",
],
"phases": [
rehearsal_phase(
@@ -487,31 +486,29 @@ def scale_fault_rehearsal_plan(
],
),
rehearsal_phase(
"object-backup-restore-under-load",
"Verify object catalog backup and restore remain explainable under recent write and maintenance churn.",
"durable-backing-cutover-under-load",
"Verify durable backing cutover preflight remains explainable under recent write and maintenance churn.",
[
probe_step(
"capture-object-catalog-backup",
"POST",
warehouse_path(warehouse, "/catalog/backup", rest_path),
"capture-cutover-backup",
"OPERATOR",
table_path(warehouse, namespace, table, "/catalog/export", rest_path),
"catalog-backup-recorded",
"record catalog-backup artifacts, current metadata location, idempotency indexes, and rollback mode before cutover",
),
probe_step(
"migration-dry-run-before-cutover",
"GET",
warehouse_path(warehouse, "/catalog/migration", rest_path),
"200",
"record the backup artifact, current metadata location, commit recovery state, and referenced Iceberg object watermarks",
"migration dry-run reports inventory, replay blockers, idempotency blockers, and recommended actions",
),
probe_step(
"restore-object-catalog-backup",
"materialize-durable-backing-before-cutover",
"POST",
warehouse_path(warehouse, "/catalog/restore", rest_path),
"200-or-409",
"restore validates the target pointer, UUID, generation, and complete manifest/data/delete graph before publication",
{"backup-id": "recorded-backup-id", "allow-replace": False},
),
probe_step(
"retry-object-catalog-restore",
"POST",
warehouse_path(warehouse, "/catalog/restore", rest_path),
"200-or-409",
"repeating the same restore after an interruption returns the committed result or a safe retryable conflict",
{"backup-id": "recorded-backup-id", "allow-replace": False},
warehouse_path(warehouse, "/catalog/migration", rest_path),
"200",
"materialization drains in-flight writes and reports SNAPSHOT_MATERIALIZED before backing restart",
),
],
),
@@ -578,7 +575,7 @@ def scale_fault_rehearsal_plan(
"EVIDENCE",
"operator-recorded-artifacts",
"recorded",
"record RustFS build, object catalog mode, writer count, worker count, iteration count, final metadata location, observed conflicts, recovered leases, and failed-closed operations",
"record RustFS build, catalog backing, writer count, worker count, iteration count, final metadata location, observed conflicts, recovered leases, and failed-closed operations",
),
],
),
@@ -596,7 +593,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser.add_argument("--writer-count", type=positive_int, default=8)
parser.add_argument("--maintenance-worker-count", type=positive_int, default=2)
parser.add_argument("--iteration-count", type=positive_int, default=50)
parser.add_argument("--catalog-backing", default="object")
parser.add_argument("--catalog-backing", default="object-backed-or-durable-strong")
parser.add_argument("--print-failure-matrix", action="store_true")
parser.add_argument("--print-failure-probes", action="store_true")
parser.add_argument("--print-disaster-recovery-rehearsal", action="store_true")
+40 -17
View File
@@ -28,7 +28,7 @@ DEFAULT_PROFILE = "rustfs"
CATALOG_VENDED_PROFILE = "rustfs-vended-credentials"
VENDED_CREDENTIAL_PROFILES = {CATALOG_VENDED_PROFILE}
CATALOG_BACKING_KEY = "rustfs.catalog-backing"
CATALOG_BACKINGS = {"object"}
CATALOG_BACKINGS = {"object", "durable-strong"}
SMOKE_ROWS = [{"id": 1, "payload": "alpha"}, {"id": 2, "payload": "beta"}]
REQUIRED_STORAGE_CREDENTIAL_KEYS = (
"s3.access-key-id",
@@ -245,9 +245,9 @@ UNSUPPORTED_INVENTORY: list[dict[str, str]] = [
PRODUCTION_READINESS_INVENTORY: list[dict[str, str]] = [
{
"capability": "object-catalog-backup-restore",
"status": "controlled-recovery-supported",
"validation": "backup and restore validate the catalog pointer, commit-log, idempotency, maintenance, and referenced Iceberg object graph while preserving CAS, write-fence, and retryable recovery guarantees",
"capability": "strong-catalog-backing",
"status": "state-transfer-supported",
"validation": "preflight exposes recovery blockers and target agreement; migration execution fences object-backed writes, drains in-flight catalog mutations, and idempotently materializes table bucket, namespace, table, view, commit-log, and idempotency state before cutover",
},
{
"capability": "single-active-writer-ha",
@@ -1466,6 +1466,26 @@ def run_load_table_delegation_probe(args: argparse.Namespace, deps: RuntimeDeps)
verify_vended_credential_data_plane_scope(args, deps, credential, warehouse)
def run_strong_backing_boundaries(args: argparse.Namespace, deps: RuntimeDeps) -> None:
pointer_path = table_endpoint_path(args, "/metadata-location")
before = signed_rest_request(args, deps, "GET", pointer_path)
for key in ("metadata-location", "version-token", "generation"):
if key not in before:
raise RuntimeError("metadata-location response is missing the commit state")
path = table_endpoint_path(args, "/maintenance/metadata")
error = signed_rest_request_expect_error(
args, deps, "POST", path,
{"retain-recent-metadata-files": 1, "delete": False}, expected_statuses={400},
)
envelope = json.loads(error.response_body).get("error")
if not isinstance(envelope, dict) or (envelope.get("code"), envelope.get("type"), envelope.get("message")) != (
400, "BadRequestException", "operation is not supported with durable-strong table catalog backing"
):
raise RuntimeError("strong metadata maintenance did not return the expected Iceberg error envelope")
if signed_rest_request(args, deps, "GET", pointer_path) != before:
raise RuntimeError("unsupported strong backing operation changed the table commit state")
def maintenance_job_id(job: object) -> str | None:
if not isinstance(job, dict):
return None
@@ -1489,19 +1509,22 @@ def run_maintenance_probe(args: argparse.Namespace, deps: RuntimeDeps, backing:
if any(config.get(key) != value for key, value in safe_config.items()):
raise RuntimeError("maintenance config endpoint did not persist the smoke config")
if backing != "object":
raise RuntimeError("maintenance probes require the object catalog backing")
report = signed_rest_request(
args, deps, "POST", table_endpoint_path(args, "/maintenance/metadata"),
{"retain-recent-metadata-files": 1, "delete": False},
)
if maintenance_job_id(report.get("job")) is None:
raise RuntimeError("maintenance metadata endpoint did not return a job id")
audit_events = report.get("audit-events")
if not isinstance(audit_events, list) or not any(
isinstance(event, dict) and event.get("action") == "PLANNED" for event in audit_events
):
raise RuntimeError("maintenance metadata endpoint did not report a planning audit event")
if backing == "durable-strong":
run_strong_backing_boundaries(args, deps)
elif backing == "object":
report = signed_rest_request(
args, deps, "POST", table_endpoint_path(args, "/maintenance/metadata"),
{"retain-recent-metadata-files": 1, "delete": False},
)
if maintenance_job_id(report.get("job")) is None:
raise RuntimeError("maintenance metadata endpoint did not return a job id")
audit_events = report.get("audit-events")
if not isinstance(audit_events, list) or not any(
isinstance(event, dict) and event.get("action") == "PLANNED" for event in audit_events
):
raise RuntimeError("maintenance metadata endpoint did not report a planning audit event")
else:
raise RuntimeError("maintenance probes require a known backing")
scheduler = signed_rest_request(args, deps, "GET", scheduler_path)
expected_statuses = {"READY", "QUEUED", "DISABLED", "PAUSED", "BACKPRESSURED", "RETRY_DEFERRED", "QUARANTINED"}

Some files were not shown because too many files have changed in this diff Show More