Compare commits

...

6 Commits

44 changed files with 1869 additions and 210 deletions
+10
View File
@@ -132,6 +132,12 @@ test-group = 'ecstore-serial-flaky'
filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
test-group = 'embedded-test-ports'
# The real object probe includes bucket creation and cleanup in its short
# measurement budget. Keep competing storage fixtures outside that budget.
[[profile.default.overrides]]
filter = 'package(rustfs) & binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)'
threads-required = "num-test-threads"
# Serialize the durable manual-transition checkpoint test across nextest's
# process boundary; it mutates bucket lifecycle metadata and is not quarantined.
[[profile.default.overrides]]
@@ -327,6 +333,10 @@ test-group = 'ecstore-serial-flaky'
filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnostic_capability_e2e))'
test-group = 'embedded-test-ports'
[[profile.ci.overrides]]
filter = 'package(rustfs) & binary(connect_perf_object) & test(=real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put)'
threads-required = "num-test-threads"
# Serialize the durable manual-transition checkpoint test under the ci profile
# too. No retries: failures stay visible.
[[profile.ci.overrides]]
Generated
+44 -1
View File
@@ -3790,6 +3790,23 @@ dependencies = [
"cipher 0.5.2",
]
[[package]]
name = "dial9"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49b11bce55173f77d9003cce8f129b4af889a4639005c53c3498cef933bc895c"
dependencies = [
"bon",
"dial9-core",
"dial9-macro",
"dial9-metrique",
"dial9-perf-self-profile",
"dial9-tokio-telemetry",
"dial9-trace-format",
"tokio",
"tracing",
]
[[package]]
name = "dial9-core"
version = "0.5.0"
@@ -3812,6 +3829,31 @@ dependencies = [
"ulid",
]
[[package]]
name = "dial9-macro"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4177b8f8bf95d8e3769c1aad596f5d6fb231953a45c4c3d2a5946fe8e4ec327b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "dial9-metrique"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d42fe0e3e14e8780c4ecf9ee14c183dbcb9b1ae3612cc9a689e7dc83b13adef1"
dependencies = [
"dial9-core",
"dial9-trace-format",
"metrique",
"metrique-writer",
"tokio",
"tracing",
]
[[package]]
name = "dial9-perf-self-profile"
version = "0.5.0"
@@ -3827,6 +3869,7 @@ dependencies = [
"libc",
"perf-event-data",
"perf-event-open-sys2",
"tokio",
"tracing",
]
@@ -10345,7 +10388,7 @@ dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"dial9-tokio-telemetry",
"dial9",
"flate2",
"futures-util",
"glob",
+1 -1
View File
@@ -351,7 +351,7 @@ zstd = "0.14.0"
# Observability and Metrics
metrics = "0.24.6"
metrics-util = "0.20"
dial9-tokio-telemetry = "0.5.0"
dial9 = { version = "0.5.0", default-features = false }
opentelemetry = { version = "0.32.0" }
opentelemetry-appender-tracing = { version = "0.32.0" }
opentelemetry-otlp = { version = "0.32.0" }
+1 -1
View File
@@ -39,7 +39,7 @@ pub const ENV_RUNTIME_DIAL9_S3_BUCKET: &str = "RUSTFS_RUNTIME_DIAL9_S3_BUCKET";
/// Accepted but not honoured; see [`ENV_RUNTIME_DIAL9_S3_BUCKET`].
pub const ENV_RUNTIME_DIAL9_S3_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_S3_PREFIX";
// Note: there are deliberately no task-dump knobs. dial9 only captures task
// dumps for futures spawned through `dial9_tokio_telemetry::spawn`, and RustFS
// dumps for futures spawned through `dial9::spawn`, and RustFS
// spawns with `tokio::spawn` throughout, so the switch could never do anything.
// Measured: 0 dumps via tokio::spawn vs 14709 via dial9::spawn on an identical
// workload. See rustfs/backlog#1157 (D9-16).
@@ -515,20 +515,26 @@ impl ExpiryStats {
Self::add_nonnegative(&self.missed_tier_journal_tasks, 1);
}
// The pending and active gauges are balanced by design: every increment
// has exactly one matching decrement. They must not saturate at zero on
// update, because a worker can dequeue (and decrement) before the
// enqueuing side has recorded its increment. Clamping that transient -1
// to 0 turns the later +1 into a phantom task that never drains
// (rustfs#7921). Readers clamp negative snapshots instead.
fn increment_pending_tasks(&self) {
Self::add_nonnegative(&self.pending_tasks, 1);
self.pending_tasks.fetch_add(1, Ordering::AcqRel);
}
fn decrement_pending_tasks(&self) {
Self::add_nonnegative(&self.pending_tasks, -1);
self.pending_tasks.fetch_sub(1, Ordering::AcqRel);
}
fn increment_active_tasks(&self) {
Self::add_nonnegative(&self.active_tasks, 1);
self.active_tasks.fetch_add(1, Ordering::AcqRel);
}
fn decrement_active_tasks(&self) {
Self::add_nonnegative(&self.active_tasks, -1);
self.active_tasks.fetch_sub(1, Ordering::AcqRel);
}
fn increment_workers(&self) {
@@ -1074,9 +1080,13 @@ impl ExpiryState {
}
fn send_expiry_task(&self, wrkr: Sender<Option<ExpiryOpType>>, task: ExpiryOpType) -> bool {
// Account for the task before a worker can observe it. The worker
// decrements on dequeue, so incrementing after `try_send` would let a
// fast dequeue run the gauge through zero first.
self.stats.increment_pending_tasks();
let queued = wrkr.try_send(Some(task)).is_ok();
if queued {
self.stats.increment_pending_tasks();
if !queued {
self.stats.decrement_pending_tasks();
}
queued
}
@@ -1444,11 +1454,14 @@ async fn enqueue_recovered_free_version_with_state(state: &Arc<RwLock<ExpiryStat
return false;
};
// Same ordering rule as `ExpiryState::send_expiry_task`: count first, so
// a worker that dequeues immediately cannot decrement before this
// increment lands.
stats.increment_pending_tasks();
let queued = wrkr.try_send(Some(Box::new(task))).is_ok();
if !queued {
stats.decrement_pending_tasks();
stats.increment_missed_freevers_tasks();
} else {
stats.increment_pending_tasks();
}
stats.record_scanner_expiry_state();
queued
@@ -3757,11 +3770,8 @@ pub async fn enqueue_transition_immediate(oi: &ObjectInfo, src: LcEventSrc) {
}
}
pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) {
let Some(api) = runtime_sources::object_store_handle() else {
return;
};
let configs = match metadata_boundary::get_expiry_configs(&api, &oi.bucket).await {
pub(crate) async fn enqueue_immediate_expiry(api: Arc<ECStore>, oi: &ObjectInfo, src: LcEventSrc, opts: &ObjectOptions) {
let configs = match metadata_boundary::get_expiry_configs_for_options(&api, &oi.bucket, opts).await {
Ok(configs) => configs,
Err(err) => {
observe_lifecycle_observability_event(EVENT_LIFECYCLE_EVALUATION_FAILED, "failed", Some("metadata_unavailable"));
@@ -7572,6 +7582,65 @@ mod tests {
assert!(recovery_notify.notified().now_or_never().is_some());
}
#[tokio::test]
async fn expiry_pending_gauge_survives_dequeue_landing_before_enqueue_accounting() {
// A worker may dequeue and decrement before the enqueuing side records
// its increment. The gauge must return to zero afterwards instead of
// clamping the transient -1 away and reporting a phantom pending task
// that no idle check can ever drain (rustfs#7921).
let state = ExpiryState::new();
let stats = Arc::clone(&state.read().await.stats);
stats.decrement_pending_tasks();
stats.increment_pending_tasks();
assert_eq!(stats.pending_tasks(), 0);
assert_eq!(state.read().await.pending_tasks(), 0);
stats.decrement_active_tasks();
stats.increment_active_tasks();
assert_eq!(stats.active_tasks(), 0);
assert_eq!(state.read().await.active_tasks(), 0);
}
#[tokio::test]
async fn free_version_enqueue_rolls_back_pending_when_queue_full() {
// Single-threaded, so this cannot observe the count-before-publish
// ordering itself; it pins the rollback that ordering requires: a
// rejected send must not leave its speculative increment behind.
let state = ExpiryState::new_with_unconsumed_worker_channel(1);
let oi = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
transitioned_object: TransitionedObject {
name: "remote/object".to_string(),
version_id: "remote-version".to_string(),
tier: "WARM".to_string(),
free_version: true,
..Default::default()
},
..Default::default()
};
assert!(state.read().await.enqueue_free_version(oi.clone()));
assert_eq!(state.read().await.stats.pending_tasks(), 1);
// The single-slot queue is full for both enqueue paths.
assert!(!enqueue_recovered_free_version_with_state(&state, oi.clone()).await);
assert_eq!(state.read().await.stats.pending_tasks(), 1);
assert_eq!(state.read().await.stats.missed_free_vers_tasks(), 1);
assert!(!state.read().await.enqueue_free_version(oi));
assert_eq!(state.read().await.stats.pending_tasks(), 1);
assert_eq!(state.read().await.stats.missed_free_vers_tasks(), 2);
// Draining the one real task returns the gauge to zero.
let receiver = state.read().await.tasks_rx[0].clone();
let task = receiver.lock().await.recv().await.expect("queued task");
assert!(task.is_some());
state.read().await.stats.decrement_pending_tasks();
assert_eq!(state.read().await.pending_tasks(), 0);
}
#[tokio::test]
async fn enqueue_recovered_free_version_reports_false_without_worker_channel() {
let state = ExpiryState::new();
@@ -12703,7 +12772,8 @@ mod tests {
.push((event, state, reason));
});
super::enqueue_immediate_expiry(&object_info, LcEventSrc::S3PutObject).await;
super::enqueue_immediate_expiry(Arc::clone(&ecstore), &object_info, LcEventSrc::S3PutObject, &ObjectOptions::default())
.await;
assert!(
observed.lock().expect("observed events should not poison").contains(&(
@@ -55,7 +55,15 @@ pub(crate) async fn lifecycle_expiry_allowed(
}
pub(crate) async fn get_expiry_configs(api: &crate::store::ECStore, bucket: &str) -> Result<LifecycleExpiryConfigs> {
let bucket_incarnation_id = api.bucket_incarnation_id_from_disk(bucket).await?;
get_expiry_configs_for_options(api, bucket, &crate::object_api::ObjectOptions::default()).await
}
pub(crate) async fn get_expiry_configs_for_options(
api: &crate::store::ECStore,
bucket: &str,
opts: &crate::object_api::ObjectOptions,
) -> Result<LifecycleExpiryConfigs> {
let bucket_incarnation_id = metadata_sys::get_bucket_incarnation_id_for_options_in(&api.ctx, bucket, opts).await?;
let metadata = get_authoritative_metadata(api, bucket, bucket_incarnation_id).await?;
let table_bucket_enabled = metadata.table_bucket_enabled();
+41 -1
View File
@@ -24,6 +24,7 @@ use crate::bucket::metadata::{load_bucket_metadata_parse, load_bucket_metadata_p
use crate::bucket::utils::is_meta_bucketname;
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
use crate::object_api::ObjectOptions;
use crate::runtime::sources as runtime_sources;
use crate::storage_api_contracts::heal::HealOperations as _;
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
@@ -490,6 +491,18 @@ pub(crate) async fn get_bucket_incarnation_id_in(ctx: &crate::runtime::instance:
sys.get_bucket_incarnation_id_from_disk(bucket).await
}
/// Validate against disk while retaining an already-held Object Lock fence.
pub(crate) async fn get_bucket_incarnation_id_for_options_in(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
opts: &ObjectOptions,
) -> Result<Uuid> {
let sys = bucket_metadata_sys_of(ctx)?;
let sys = sys.read().await.clone();
let guard = acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
sys.get_bucket_incarnation_id_under_transaction_lock(bucket, &guard).await
}
pub(crate) async fn get_cached_bucket_incarnation_id_in(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
@@ -1055,6 +1068,24 @@ pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_in(
Ok(lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?)
}
/// Readers already holding an Object Lock snapshot must share its guard:
/// a fresh read acquisition can queue behind a writer waiting for that snapshot.
pub(crate) async fn acquire_bucket_metadata_transaction_read_lock_for_options_in(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
opts: &ObjectOptions,
) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
if let Some(snapshot) = opts.object_lock_config_snapshot.as_ref() {
let store = object_store_in(ctx).await?;
return snapshot
.metadata_transaction_guard_for(store.id, bucket, opts.expected_bucket_incarnation_id)
.ok_or_else(|| {
Error::other("Object Lock snapshot does not hold a valid metadata transaction fence for this bucket")
});
}
Ok(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?))
}
async fn acquire_transaction_lock_with_sys(
sys: &Arc<RwLock<BucketMetadataSys>>,
bucket: &str,
@@ -2357,8 +2388,17 @@ impl BucketMetadataSys {
let _transaction_guard = transaction_lock
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
.await?;
self.get_bucket_incarnation_id_under_transaction_lock(bucket, &_transaction_guard)
.await
}
async fn get_bucket_incarnation_id_under_transaction_lock(
&self,
bucket: &str,
transaction_guard: &rustfs_lock::NamespaceLockGuard,
) -> Result<Uuid> {
let incarnation_id = load_bucket_incarnation(self.object_store(), bucket).await?;
if _transaction_guard.is_lock_lost() {
if transaction_guard.is_lock_lost() {
return Err(Error::other(format!("bucket incarnation metadata transaction lock was lost: {bucket}")));
}
match incarnation_id {
+35 -9
View File
@@ -38,6 +38,7 @@ const MAX_ORPHANS_REAPED_PER_WRITE: usize = 64;
const MAX_ORPHAN_PROBES_PER_WRITE: usize = 128;
const ORPHAN_PROBE_CONCURRENCY: usize = 32;
const EVENT_QUOTA_LEDGER_SETTLEMENT: &str = "quota_ledger_settlement";
const EVENT_QUOTA_ADMISSION: &str = "quota_admission";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_QUOTA: &str = "quota";
@@ -46,6 +47,8 @@ static FAIL_NEXT_LEDGER_SAVE: std::sync::atomic::AtomicBool = std::sync::atomic:
// Lock order: caller-held destination object/upload, bucket metadata
// transaction (read), operation reservation, then quota ledger.
// When Object Lock already holds the metadata transaction read lock, reuse
// that guard: reacquiring it behind a waiting metadata writer would deadlock.
#[cfg(not(any(test, feature = "test-util")))]
const ORPHAN_MIN_AGE_SECONDS: i64 = 30;
@@ -210,7 +213,7 @@ pub(crate) struct QuotaContext {
capability_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
snapshot_admission: Option<QuotaAdmission>,
legacy_data_movement: bool,
metadata_guard: Option<NamespaceLockGuard>,
metadata_guard: Option<Arc<NamespaceLockGuard>>,
pool_index: Option<usize>,
set_index: Option<usize>,
}
@@ -295,7 +298,7 @@ impl QuotaContext {
limit: quota_limit,
});
}
if operation_guard.is_lock_lost() || metadata_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost) {
if operation_guard.is_lock_lost() || metadata_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
return Err(StorageError::NamespaceLockQuorumUnavailable {
mode: "quota_reservation",
bucket: ledger_data.bucket.clone(),
@@ -332,7 +335,7 @@ struct LedgerReservationData {
pub(crate) struct QuotaReservation {
ledger: Option<LedgerReservationData>,
operation_guard: Option<NamespaceLockGuard>,
metadata_guard: Option<NamespaceLockGuard>,
metadata_guard: Option<Arc<NamespaceLockGuard>>,
capability_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
state: ReservationState,
}
@@ -346,7 +349,7 @@ enum ReservationState {
}
impl QuotaReservation {
fn unlimited(metadata_guard: Option<NamespaceLockGuard>) -> Self {
fn unlimited(metadata_guard: Option<Arc<NamespaceLockGuard>>) -> Self {
Self {
ledger: None,
operation_guard: None,
@@ -358,7 +361,7 @@ impl QuotaReservation {
pub(crate) fn is_lock_lost(&self) -> bool {
self.operation_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|| self.metadata_guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
|| self.metadata_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
}
pub(crate) fn capability_proof_matches(&self) -> bool {
@@ -438,11 +441,12 @@ pub(crate) async fn begin(
ctx: &crate::runtime::instance::InstanceContext,
bucket: &str,
object: &str,
snapshot_admission: Option<QuotaAdmission>,
data_movement: bool,
opts: &ObjectOptions,
pool_index: usize,
set_index: usize,
) -> Result<QuotaContext> {
let snapshot_admission = opts.quota_admission;
let data_movement = opts.data_movement;
if crate::bucket::utils::is_meta_bucketname(bucket) {
return Ok(QuotaContext {
store: None,
@@ -497,7 +501,7 @@ pub(crate) async fn begin(
});
}
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_in(ctx, bucket).await?;
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
let (quota, bucket_incarnation, quota_revision) =
metadata_sys::get_quota_config_and_incarnation_from_disk_in(ctx, bucket).await?;
if metadata_guard.is_lock_lost() {
@@ -513,6 +517,7 @@ pub(crate) async fn begin(
.as_ref()
.is_some_and(|quota| quota.has_unsupported_reservation_protocol())
{
log_admission_rejected(bucket, object, "unsupported_reservation_protocol");
return Err(StorageError::PartMissingOrCorrupt);
}
let durable_quota = quota.as_ref().filter(|quota| quota.uses_durable_reservations());
@@ -529,7 +534,16 @@ pub(crate) async fn begin(
Some(quota) => match (quota.quota, snapshot_admission) {
(Some(limit), Some(admission)) if admission.quota_limit() == limit => Some(admission),
(Some(_), None) if data_movement => None,
(Some(_), _) => return Err(StorageError::PartMissingOrCorrupt),
(Some(_), _) => {
// A snapshot-protocol quota requires the request handler's
// admission on every write. Missing or mismatched admission
// means a caller rebuilt `ObjectOptions` without carrying it
// over (rustfs/rustfs#7674 lost it on CopyObject); fail closed
// but leave a diagnosable trace, because the storage error is
// the generic `PartMissingOrCorrupt`.
log_admission_rejected(bucket, object, "snapshot_quota_admission_missing");
return Err(StorageError::PartMissingOrCorrupt);
}
(None, _) => None,
},
None => None,
@@ -904,6 +918,18 @@ pub fn fail_next_quota_ledger_save_for_test() {
FAIL_NEXT_LEDGER_SAVE.store(true, std::sync::atomic::Ordering::SeqCst);
}
fn log_admission_rejected(bucket: &str, object: &str, state: &'static str) {
warn!(
event = EVENT_QUOTA_ADMISSION,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_QUOTA,
state,
bucket = %bucket,
object = %object,
"quota admission rejected the write before commit"
);
}
fn log_deferred_settlement(data: &LedgerReservationData, state: &'static str, err: &StorageError) {
warn!(
event = EVENT_QUOTA_LEDGER_SETTLEMENT,
+20 -2
View File
@@ -481,6 +481,9 @@ mod decommission_lock_order_tests {
if let Some(deployment_id) = other_store.ctx.deployment_id() {
ctx.set_deployment_id(deployment_id);
}
for pool in &mut pools {
Arc::make_mut(pool).set_instance_ctx_for_test(Arc::clone(&ctx));
}
let store = Arc::new(crate::store::ECStore {
id: uuid::Uuid::new_v4(),
disk_map: other_store.disk_map.clone(),
@@ -6645,7 +6648,19 @@ mod decommission_lock_order_tests {
let restore_get_barrier = if matches!(mutation, ExternalObjectMutation::Restore) {
let tier_name = format!("ORDERRESTORE{}", &uuid::Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
let backend = register_mock_tier(&store.pools[2].instance_ctx().tier_config_mgr(), &tier_name).await;
let source_tiers = store.tier_config_mgr();
let restore_tiers = other_store.tier_config_mgr();
let backend = register_mock_tier(&source_tiers, &tier_name).await;
// Both peers must resolve the same persisted backend identity, including its prefix.
let tier_config = source_tiers.read().await.tiers[&tier_name].clone();
restore_tiers.write().await.tiers.insert(tier_name.clone(), tier_config);
crate::services::tier::tier::TierConfigMgr::install_test_driver_in(
&restore_tiers,
&tier_name,
Box::new(backend.clone()),
)
.await
.expect("install the same mock tier on the restoring peer");
store.pools[2]
.transition_object(
&bucket,
@@ -6919,7 +6934,10 @@ mod decommission_lock_order_tests {
}
});
if let Some(get_barrier) = restore_get_barrier.as_ref() {
get_barrier.wait_until_paused().await;
tokio::select! {
() = get_barrier.wait_until_paused() => {}
result = &mut ordinary_mutation => panic!("restore finished before the tier GET barrier: {result:?}"),
}
let read_opts = ObjectOptions {
skip_decommissioned: true,
..Default::default()
+10
View File
@@ -324,6 +324,15 @@ impl Sets {
&self.ctx
}
/// Keep simulated peers' metadata ownership separate while sharing disks and lock clients.
#[cfg(test)]
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
for set in &mut self.disk_set {
Arc::make_mut(set).set_instance_ctx_for_test(Arc::clone(&ctx));
}
self.ctx = ctx;
}
async fn monitor_and_connect_endpoints_task(sets: Weak<Sets>, mut rx: Receiver<()>) {
let startup_delay = tokio::time::sleep(Duration::from_secs(5));
tokio::pin!(startup_delay);
@@ -790,6 +799,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
version_id: dst_opts.version_id.clone(),
mod_time: dst_opts.mod_time,
http_preconditions: dst_opts.http_preconditions.clone(),
quota_admission: dst_opts.quota_admission,
..Default::default()
};
+22 -4
View File
@@ -41,7 +41,7 @@ impl Debug for NamespaceLockFence {
}
impl NamespaceLockFence {
fn new() -> Self {
pub(crate) fn new() -> Self {
Self {
signals: Arc::default(),
#[cfg(test)]
@@ -153,7 +153,7 @@ pub struct ObjectLockConfigSnapshot {
state: crate::bucket::metadata_sys::ObjectLockConfigState,
lifecycle_fence: NamespaceLockFence,
_lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
metadata_transaction_guard: Option<rustfs_lock::NamespaceLockGuard>,
metadata_transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
}
impl ObjectLockConfigSnapshot {
@@ -170,6 +170,7 @@ impl ObjectLockConfigSnapshot {
}
}
#[cfg(test)]
pub(crate) fn for_store_bucket(
store_id: Uuid,
bucket: &str,
@@ -210,7 +211,7 @@ impl ObjectLockConfigSnapshot {
state,
lifecycle_fence,
_lifecycle_guard: Some(lifecycle_guard),
metadata_transaction_guard: Some(metadata_transaction_guard),
metadata_transaction_guard: Some(Arc::new(metadata_transaction_guard)),
}
}
@@ -221,7 +222,7 @@ impl ObjectLockConfigSnapshot {
config_revision: OffsetDateTime,
state: crate::bucket::metadata_sys::ObjectLockConfigState,
lifecycle_fence: NamespaceLockFence,
metadata_transaction_guard: rustfs_lock::NamespaceLockGuard,
metadata_transaction_guard: Arc<rustfs_lock::NamespaceLockGuard>,
) -> Self {
Self {
store_id: Some(store_id),
@@ -265,6 +266,23 @@ impl ObjectLockConfigSnapshot {
.is_some_and(|guard| !guard.is_lock_lost())
}
/// Share the held transaction lock without queuing another reader behind
/// a metadata writer that is itself waiting for this snapshot to drop.
pub(crate) fn metadata_transaction_guard_for(
&self,
store_id: Uuid,
bucket: &str,
expected_incarnation_id: Option<Uuid>,
) -> Option<Arc<rustfs_lock::NamespaceLockGuard>> {
let bucket_incarnation_id = self.bucket_incarnation_id?;
if expected_incarnation_id.is_some_and(|expected| expected != bucket_incarnation_id)
|| !self.is_valid_for_destructive_put(store_id, bucket, bucket_incarnation_id)
{
return None;
}
self.metadata_transaction_guard.clone()
}
pub(crate) fn add_lock_fences(&self, opts: &mut ObjectOptions) {
opts.bucket_lifecycle_lock_fence
.get_or_insert_with(NamespaceLockFence::new)
+5 -1
View File
@@ -222,10 +222,14 @@ async fn test_pool_stores_with_contexts(
std::sync::Arc::clone(&ctx)
};
let make_store = |store_ctx: std::sync::Arc<crate::runtime::instance::InstanceContext>| {
let mut store_pools = pools.clone();
for pool in &mut store_pools {
std::sync::Arc::make_mut(pool).set_instance_ctx_for_test(std::sync::Arc::clone(&store_ctx));
}
std::sync::Arc::new(crate::store::ECStore {
id: uuid::Uuid::new_v4(),
disk_map: std::collections::HashMap::new(),
pools: pools.clone(),
pools: store_pools,
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, std::sync::Arc::clone(&store_ctx)),
pool_meta: tokio::sync::RwLock::new(pool_meta.clone()),
rebalance_meta: tokio::sync::RwLock::new(rebalance_meta.clone()),
+6
View File
@@ -4703,6 +4703,12 @@ impl SetDisks {
&self.ctx
}
#[cfg(test)]
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
self.local_lock_manager = ctx.lock_manager();
self.ctx = ctx;
}
/// Read the persisted bucket identity through this set's metadata owner.
/// Missing or non-authoritative legacy identities remain errors.
pub async fn bucket_incarnation_id_from_disk(&self, bucket: &str) -> Result<Uuid> {
+1 -10
View File
@@ -2613,16 +2613,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
fi.parts = Vec::with_capacity(uploaded_parts.len());
let quota_context = reservation::begin(
&self.ctx,
bucket,
object,
opts.quota_admission,
opts.data_movement,
self.pool_index,
self.set_index,
)
.await?;
let quota_context = reservation::begin(&self.ctx, bucket, object, opts, self.pool_index, self.set_index).await?;
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
let preserve_replication_ciphertext = opts.replication_request
&& contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT);
+69 -18
View File
@@ -3427,8 +3427,8 @@ impl SetDisks {
Ok(removed)
}
async fn validate_bucket_incarnation(&self, bucket: &str, expected: Uuid) -> Result<()> {
let current = metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
async fn validate_bucket_incarnation(&self, bucket: &str, expected: Uuid, opts: &ObjectOptions) -> Result<()> {
let current = metadata_sys::get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, opts).await?;
if current != expected {
return Err(StorageError::BucketNotFound(bucket.to_string()));
}
@@ -3636,13 +3636,22 @@ impl SetDisks {
// Transformed streams (unknown stored size) are admitted by their
// plaintext size and then take the streaming encode with inline
// buffer writers, since the single-block fast path needs a known length.
// Protected inline writes retain the single-block bound on proof metadata.
// Every inline candidate keeps the single-block bound on its admission
// size: the inline buffer writer grows without a cap, so an explicit
// INLINE_BLOCK budget above `block_size` must not admit multi-block
// payloads into memory and xl.meta. Protected inline writes also need
// a known stored length for their proof metadata, so they never fall
// back to the plaintext size.
let inline_admission_size = if put_object_size >= 0 || protect_write {
put_object_size
} else {
data.actual_size()
};
let is_inline_buffer = storage_class_config.should_inline(
inline_admission_shard_size(&erasure, put_object_size, data.actual_size()),
erasure.data_shards,
opts.versioned,
) && (!protect_write
|| usize::try_from(put_object_size).is_ok_and(|size| size <= erasure.block_size));
) && usize::try_from(inline_admission_size).is_ok_and(|size| size <= erasure.block_size);
let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
let shard_file_size = shard_file_size_raw;
@@ -4234,16 +4243,7 @@ impl SetDisks {
None
};
let quota_context = reservation::begin(
&self.ctx,
bucket,
object,
opts.quota_admission,
opts.data_movement,
self.pool_index,
self.set_index,
)
.await?;
let quota_context = reservation::begin(&self.ctx, bucket, object, opts, self.pool_index, self.set_index).await?;
let quota_mutation_fence = quota_context.is_enforced() || opts.quota_admission.is_some();
let mut replication_quota_size = None;
@@ -8869,7 +8869,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
return Ok(ObjectInfo::default());
}
if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id {
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
self.validate_bucket_incarnation(bucket, expected_incarnation_id, &opts)
.await?;
}
ensure_delete_commit_locks_held(_lock_guard.as_ref(), bucket, object, &opts)?;
begin_scanner_publication_delete_mutation(scanner_publication_commit_scope.as_ref())?;
@@ -9283,7 +9284,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
check_object_lock_retention_update(bucket, object, &obj_info, opts)?;
if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id {
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
self.validate_bucket_incarnation(bucket, expected_incarnation_id, opts)
.await?;
}
if _lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|| opts
@@ -9863,7 +9865,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
.await?
.acquire_bucket_lifecycle_read_lock(bucket)
.await?;
self.validate_bucket_incarnation(bucket, expected_incarnation_id).await?;
self.validate_bucket_incarnation(bucket, expected_incarnation_id, opts)
.await?;
Some(guard)
} else {
None
@@ -11801,6 +11804,54 @@ mod inline_put_commit_path_tests {
}
}
#[tokio::test]
#[serial]
async fn unprotected_puts_keep_multi_block_inline_candidates_external() {
for transformed in [true, false] {
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
let storage_class =
temp_env::with_var(INLINE_BLOCK_ENV, Some("16MiB"), || lookup_config_for_pools(&KVS::new(), &[4]))
.expect("large inline budget should resolve");
set_disks.set_test_storage_class_config(storage_class);
let bucket = "unprotected-inline-candidates";
let object = "object.bin";
make_bucket(&disk_stores, bucket).await;
// Above the single erasure block: an explicit INLINE_BLOCK budget must
// not pull a multi-block payload into the unbounded inline buffer.
let plaintext = compressible_payload(1024 * 1024 + 17);
let (mut reader, opts) = if transformed {
transformed_put(plaintext.clone())
} else {
(PutObjReader::from_vec(plaintext.clone()), ObjectOptions::default())
};
temp_env::async_with_vars(
[(
crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
Some("false"),
)],
set_disks.put_object(bucket, object, &mut reader, &opts),
)
.await
.expect("multi-block PUT should commit with part files");
for (disk_index, disk) in disk_stores.iter().enumerate() {
let file_info = disk
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.unwrap_or_else(|err| panic!("disk {disk_index} should persist metadata: {err}"));
assert!(
!file_info.inline_data(),
"disk {disk_index}: a payload above block_size must stay external (transformed={transformed})"
);
}
assert!(
count_part_files(&temp_dirs) > 0,
"a multi-block object must keep part files (transformed={transformed})"
);
assert_eq!(read_back(&set_disks, bucket, object).await, plaintext);
}
}
#[tokio::test]
async fn transformed_put_above_inline_budget_keeps_part_files() {
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+11 -1
View File
@@ -2776,7 +2776,16 @@ mod tests {
let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
store.pools[0]
.put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default())
.put_object(
&bucket,
active_object,
&mut active_reader,
&ObjectOptions {
// Drain the rename tail before removing a physical shard.
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("active owner object should be written");
let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
@@ -2812,6 +2821,7 @@ mod tests {
&mut duplicate_reader,
&ObjectOptions {
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
},
)
+171 -12
View File
@@ -2905,8 +2905,17 @@ mod tests {
if !version.is_empty() {
insert_str(&mut metadata, SUFFIX_TRANSITIONED_VERSION_ID, version.to_string());
}
// The commit path acknowledges at write quorum and drains
// the remaining rename fan-out in the background. This
// fixture "crashes" the store right after the overwrite,
// so it must wait for that tail: a disk left with the old
// live transitioned source makes exact cleanup fail closed
// (a minority live owner still references the remote
// tuple) until heal repairs it, which this fixture never
// runs (rustfs#7921).
let options = ObjectOptions {
version_suspended: suspended,
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
};
store
@@ -2978,6 +2987,7 @@ mod tests {
assert_eq!(free.len(), 1, "{state:?}, suspended={suspended}, copy={self_copy}");
assert_eq!(free[0].transitioned_objname, remote);
assert_eq!(free[0].transition_version_state, state);
assert_every_disk_holds_only_the_cleanup_owner(&set, &bucket, object, &remote).await;
assert!(backend.contains(&remote).await, "commit must not delete remote bytes before cleanup");
let removed_before = backend.remove_count().await;
@@ -3149,6 +3159,120 @@ mod tests {
shutdown.cancel();
}
/// rustfs/rustfs#7674: a bucket carrying a legacy snapshot-protocol quota
/// (written before durable reservations existed, so `quota.json` has no
/// `reservation_protocol`) must accept a cross-key CopyObject whose
/// destination options carry the handler's quota admission. The storage
/// layer fails closed with `PartMissingOrCorrupt` when a quota-enforced
/// bucket sees a write without admission, so dropping the admission while
/// rebuilding the destination options turns every copy into a
/// deterministic "part missing or corrupt" failure even though the source
/// object is perfectly readable.
#[cfg(feature = "test-util")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial(storage_class_env)]
async fn copy_object_forwards_quota_admission_on_legacy_snapshot_quota_bucket() {
let temp_dir = tempfile::tempdir().expect("create legacy quota copy store dir");
let (ctx, store, shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-quota-copy", &[1])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = format!("legacy-quota-copy-{}", Uuid::new_v4());
let source_object = "docker/registry/v2/repositories/example/_uploads/upload-id/data";
let target_object = "docker/registry/v2/blobs/sha256/a5/digest/data";
let payload = vec![0x5A; 8178];
let quota_limit = 1u64 << 30;
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create bucket for legacy quota copy");
// Legacy quota shape: no `reservation_protocol`, so the storage layer
// requires the handler-supplied snapshot admission on every write.
let legacy_quota = format!(r#"{{"quota":{quota_limit},"quota_type":"Hard"}}"#);
crate::bucket::metadata_sys::update_in(
&ctx,
&bucket,
crate::bucket::metadata::BUCKET_QUOTA_CONFIG_FILE,
legacy_quota.into_bytes(),
)
.await
.expect("persist legacy snapshot quota");
let (quota, _, _) = crate::bucket::metadata_sys::get_quota_config_and_incarnation_from_disk_in(&ctx, &bucket)
.await
.expect("legacy quota should load");
let quota = quota.expect("legacy quota must be persisted");
assert_eq!(quota.quota, Some(quota_limit));
assert!(!quota.uses_durable_reservations(), "fixture must stay on the snapshot protocol");
let mut write_opts = ObjectOptions::default();
assert!(write_opts.set_quota_admission(0, quota_limit));
let upload = store
.new_multipart_upload(&bucket, source_object, &write_opts)
.await
.expect("create source multipart upload");
let mut part_reader = PutObjReader::from_vec(payload.clone());
let part = store
.put_object_part(&bucket, source_object, &upload.upload_id, 1, &mut part_reader, &write_opts)
.await
.expect("stage multipart source part");
store
.clone()
.complete_multipart_upload(
&bucket,
source_object,
&upload.upload_id,
vec![crate::storage_api_contracts::multipart::CompletePart {
part_num: part.part_num,
etag: part.etag,
..Default::default()
}],
&write_opts,
)
.await
.expect("complete the multipart source under the legacy quota");
let source_reader = store
.get_object_reader(&bucket, source_object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("completed multipart source should be readable");
let mut copy_info = source_reader.object_info.clone();
let actual_size = copy_info.get_actual_size().expect("copy source logical size should resolve");
assert_eq!(actual_size, payload.len() as i64);
let copy_reader = rustfs_rio::HashReader::from_stream(source_reader.stream, actual_size, actual_size, None, None, false)
.expect("copy source hash reader should build");
copy_info.put_object_reader = Some(PutObjReader::new(copy_reader));
let mut dst_opts = ObjectOptions::default();
assert!(dst_opts.set_quota_admission(payload.len() as u64, quota_limit));
store
.copy_object(
&bucket,
source_object,
&bucket,
target_object,
&mut copy_info,
&ObjectOptions::default(),
&dst_opts,
)
.await
.expect("CopyObject must forward the handler quota admission to the destination write");
let mut target_reader = store
.get_object_reader(&bucket, target_object, None, HeaderMap::new(), &ObjectOptions::default())
.await
.expect("copied target should be readable");
let mut target_body = Vec::new();
target_reader
.stream
.read_to_end(&mut target_body)
.await
.expect("target body should stream");
assert_eq!(target_body, payload);
shutdown.cancel();
}
#[cfg(feature = "test-util")]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial_test::serial(storage_class_env)]
@@ -9712,14 +9836,7 @@ mod tests {
.await
.expect("manual task receipt path should resolve");
let target_task_set = store.pools[1].get_disks_by_key(&manual_task_receipt_path);
let original_target_task_disks = {
let mut disks = target_task_set.disks.write().await;
let original = disks.clone();
for disk in disks.iter_mut().take(2) {
*disk = None;
}
original
};
let offline_target_task_disks = force_set_disk_range_offline_for_test(&target_task_set, 0..2).await;
let receipt_quorum_error = store
.verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
0,
@@ -9728,7 +9845,7 @@ mod tests {
)
.await
.expect_err("target read quorum without receipt write quorum must retain the source");
*target_task_set.disks.write().await = original_target_task_disks;
drop(offline_target_task_disks);
let receipt_quorum_error = receipt_quorum_error.to_string();
assert!(receipt_quorum_error.contains("receipt"));
assert!(receipt_quorum_error.contains(&manual_task_path));
@@ -12648,10 +12765,45 @@ mod tests {
}
}
/// Every physical copy must carry the replacement plus its tier
/// free-version owner, and none may still hold the pre-overwrite live
/// transitioned source. A stale minority copy is exactly what a lost
/// early-ACK rename tail leaves behind, and exact cleanup refuses to
/// delete remote bytes while such a live reference exists.
#[cfg(feature = "test-util")]
async fn assert_every_disk_holds_only_the_cleanup_owner(
set: &crate::set_disk::SetDisks,
bucket: &str,
object: &str,
remote: &str,
) {
use crate::disk::DiskAPI as _;
let disk_object = rustfs_utils::path::encode_dir_object(object);
for (index, disk) in set.disk_inventory().await.into_iter().enumerate() {
let disk = disk.unwrap_or_else(|| panic!("disk{index} should be online"));
let raw = disk
.read_xl(bucket, &disk_object, false)
.await
.unwrap_or_else(|err| panic!("disk{index} xl.meta should be readable after the overwrite: {err:?}"));
let versions = rustfs_filemeta::FileMeta::load(&raw.buf)
.and_then(|meta| meta.get_all_file_info_versions(bucket, object, true))
.unwrap_or_else(|err| panic!("disk{index} xl.meta should decode: {err:?}"));
let all: Vec<_> = versions.versions.iter().chain(versions.free_versions.iter()).collect();
let owners = all.iter().filter(|fi| fi.tier_free_version()).count();
let live_sources = all
.iter()
.filter(|fi| !fi.tier_free_version() && fi.transitioned_objname == remote)
.count();
assert_eq!(owners, 1, "disk{index} must hold exactly one cleanup owner after the drained overwrite");
assert_eq!(live_sources, 0, "disk{index} must not retain the pre-overwrite live transitioned source");
}
}
#[cfg(feature = "test-util")]
async fn wait_for_expiry_workers_idle(store: &crate::store::ECStore) {
let expiry_state = store.ctx.expiry_state();
tokio::time::timeout(Duration::from_secs(30), async {
let idle = tokio::time::timeout(Duration::from_secs(30), async {
loop {
let idle = {
let state = expiry_state.read().await;
@@ -12671,8 +12823,15 @@ mod tests {
}
}
})
.await
.expect("lifecycle expiry workers should become idle");
.await;
if idle.is_err() {
let state = expiry_state.read().await;
panic!(
"lifecycle expiry workers should become idle: pending={} active={}",
state.pending_tasks(),
state.active_tasks()
);
}
}
#[cfg(feature = "test-util")]
+18 -5
View File
@@ -395,12 +395,19 @@ async fn scan_metadata_less_residue_with_budget(
Ok(scan)
}
async fn enqueue_transition_after_write(result: Result<ObjectInfo>, src: LcEventSrc) -> Result<ObjectInfo> {
async fn enqueue_transition_after_write(
store: &ECStore,
result: Result<ObjectInfo>,
src: LcEventSrc,
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
match result {
Ok(oi) => {
if should_enqueue_transition_immediately(&oi) {
enqueue_transition_immediate(&oi, src.clone()).await;
enqueue_immediate_expiry(&oi, src).await;
if let Ok(api) = metadata_sys::object_store_in(&store.ctx).await {
enqueue_immediate_expiry(api, &oi, src, opts).await;
}
}
Ok(oi)
}
@@ -1281,9 +1288,11 @@ impl ECStore {
opts: &ObjectOptions,
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
let result = match self.handle_put_object(bucket, object, data, opts).await {
Ok((object_info, old_current_size)) => enqueue_transition_after_write(Ok(object_info), LcEventSrc::S3PutObject)
.await
.map(|object_info| (object_info, old_current_size)),
Ok((object_info, old_current_size)) => {
enqueue_transition_after_write(self, Ok(object_info), LcEventSrc::S3PutObject, opts)
.await
.map(|object_info| (object_info, old_current_size))
}
Err(err) => Err(err),
};
if result.is_ok() {
@@ -1356,9 +1365,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for ECStore {
dst_opts: &ObjectOptions,
) -> Result<ObjectInfo> {
let result = enqueue_transition_after_write(
self,
self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
.await,
LcEventSrc::S3CopyObject,
dst_opts,
)
.await;
if result.is_ok() {
@@ -1625,10 +1636,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for ECStore {
opts: &ObjectOptions,
) -> Result<ObjectInfo> {
let result = enqueue_transition_after_write(
self.as_ref(),
self.clone()
.handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
.await,
LcEventSrc::S3CompleteMultipartUpload,
opts,
)
.await;
if result.is_ok() {
+4 -2
View File
@@ -355,7 +355,7 @@ impl ECStore {
if let Some(guard) = guard.as_ref() {
opts.add_bucket_lifecycle_lock_guard(guard);
}
let current = crate::bucket::metadata_sys::get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
let current = crate::bucket::metadata_sys::get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
if opts.expected_bucket_incarnation_id != Some(current) {
return Err(StorageError::BucketNotFound(bucket.to_string()));
}
@@ -1047,6 +1047,7 @@ impl ECStore {
opts: &ObjectOptions,
publication_fence: Option<RemoteTuplePublicationFence>,
) -> Result<ObjectInfo> {
let request_opts = opts;
let (target_pool_idx, mutation_fence) = target;
check_complete_multipart_args(bucket, object, upload_id)?;
if !opts.data_movement {
@@ -1132,7 +1133,8 @@ impl ECStore {
)
.await;
drop(publication_guard);
let result = enqueue_transition_after_write(result, LcEventSrc::S3CompleteMultipartUpload).await;
let result =
enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, request_opts).await;
if result.is_ok() {
list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
}
+30 -16
View File
@@ -29,8 +29,8 @@ use crate::bucket::lifecycle::{
},
};
use crate::bucket::metadata_sys::{
acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_in, get_cached_bucket_incarnation_id_in,
get_object_lock_config_and_incarnation_from_disk_in,
acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_for_options_in,
get_cached_bucket_incarnation_id_in, get_object_lock_config_and_incarnation_from_disk_in,
};
use crate::bucket::object_lock::objectlock_sys::{
check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state,
@@ -2837,7 +2837,7 @@ impl ECStore {
config_revision,
state,
lifecycle_fence.clone(),
metadata_guard,
Arc::new(metadata_guard),
)))
}
@@ -4296,6 +4296,7 @@ impl ECStore {
if !opts.data_movement {
return Err(Error::other("data movement PUT requires data_movement options"));
}
let request_opts = opts;
let (object, mut opts) = self.prepare_put_object(bucket, object, opts).await?;
ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts);
let idx = self
@@ -4338,7 +4339,7 @@ impl ECStore {
},
)
.await;
let result = enqueue_transition_after_write(result, LcEventSrc::S3PutObject).await;
let result = enqueue_transition_after_write(self, result, LcEventSrc::S3PutObject, request_opts).await;
if result.is_ok() {
list_objects::observe_list_objects_mutation(self, bucket).await;
}
@@ -4445,7 +4446,7 @@ impl ECStore {
};
let current_bucket_incarnation_id = if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
dst_opts.add_bucket_lifecycle_lock_guard(guard);
let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, dst_bucket).await?;
let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, dst_bucket, &dst_opts).await?;
if dst_opts
.expected_bucket_incarnation_id
.is_some_and(|expected| expected != current_incarnation_id)
@@ -4564,6 +4565,7 @@ impl ECStore {
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
quota_admission: dst_opts.quota_admission,
..Default::default()
};
if !self.single_pool() {
@@ -4606,6 +4608,7 @@ impl ECStore {
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
quota_admission: dst_opts.quota_admission,
..Default::default()
};
if !self.single_pool() {
@@ -4658,6 +4661,7 @@ impl ECStore {
namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
quota_admission: dst_opts.quota_admission,
..Default::default()
};
if !self.single_pool() {
@@ -4823,22 +4827,26 @@ impl ECStore {
get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await?;
}
let _object_lock_metadata_guard = if !is_meta_bucketname(bucket) {
Some(acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?)
Some(Arc::new(acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?))
} else {
None
};
if let Some(guard) = _object_lock_metadata_guard.as_ref() {
opts.add_namespace_lock_guard(guard);
}
let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() {
let current_bucket_incarnation_id = if let Some(metadata_guard) = _object_lock_metadata_guard.as_ref() {
let (state, incarnation_id, config_revision) =
get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
self.id,
bucket,
incarnation_id,
config_revision,
state,
opts.bucket_lifecycle_lock_fence
.clone()
.unwrap_or_else(NamespaceLockFence::new),
Arc::clone(metadata_guard),
)));
Some(incarnation_id)
} else {
@@ -5257,26 +5265,32 @@ impl ECStore {
let _object_lock_metadata_guard = if is_meta_bucketname(bucket) {
None
} else {
Some(match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
Ok(guard) => guard,
Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
})
Some(Arc::new(
match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
Ok(guard) => guard,
Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
},
))
};
if let Some(guard) = _object_lock_metadata_guard.as_ref() {
opts.add_namespace_lock_guard(guard);
}
let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() {
let current_bucket_incarnation_id = if let Some(metadata_guard) = _object_lock_metadata_guard.as_ref() {
let (state, incarnation_id, config_revision) =
match get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await {
Ok(snapshot) => snapshot,
Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
};
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
self.id,
bucket,
incarnation_id,
config_revision,
state,
opts.bucket_lifecycle_lock_fence
.clone()
.unwrap_or_else(NamespaceLockFence::new),
Arc::clone(metadata_guard),
)));
Some(incarnation_id)
} else {
@@ -5617,7 +5631,7 @@ impl ECStore {
opts.add_bucket_lifecycle_lock_guard(guard);
}
if !is_meta_bucketname(bucket) {
let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
if opts.expected_bucket_incarnation_id != Some(current_incarnation_id) {
return Err(StorageError::BucketNotFound(bucket.to_string()));
}
@@ -5690,7 +5704,7 @@ impl ECStore {
None
} else {
let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
let current_incarnation_id = get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
if opts
.expected_bucket_incarnation_id
.is_some_and(|expected| expected != current_incarnation_id)
@@ -0,0 +1,288 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![cfg(feature = "test-util")]
mod storage_api;
use std::sync::Arc;
use std::time::Duration;
use storage_api::metadata_lock::{
BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys,
isolated_store_over_temp_disks,
};
use tokio::io::AsyncReadExt;
use tokio::time::timeout;
use uuid::Uuid;
#[tokio::test]
async fn replica_put_completes_while_metadata_writer_waits_for_its_snapshot() {
replica_write_with_waiting_metadata_writer(false).await;
}
#[tokio::test]
async fn replica_multipart_completes_while_metadata_writer_waits_for_its_snapshot() {
replica_write_with_waiting_metadata_writer(true).await;
}
async fn replica_write_with_waiting_metadata_writer(multipart: bool) {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "replica-metadata-read-reuse";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
store
.update_bucket_metadata_config(
bucket,
"lifecycle.xml",
br#"<LifecycleConfiguration><Rule><ID>expire-old-versions</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>"#.to_vec(),
)
.await
.expect("configure post-write lifecycle evaluation");
let opts = ObjectOptions {
versioned: true,
version_id: Some(Uuid::new_v4().to_string()),
replication_request: true,
expected_bucket_incarnation_id: Some(store.bucket_incarnation_id(bucket).await.expect("load incarnation")),
object_lock_config_snapshot: Some(
store
.object_lock_config_snapshot(bucket)
.await
.expect("capture Object Lock snapshot"),
),
..Default::default()
};
let upload = if multipart {
let upload = store
.new_multipart_upload(bucket, "object", &opts)
.await
.expect("stage upload");
let mut data = PutObjReader::from_vec(b"replicated version".to_vec());
let part = store
.put_object_part(bucket, "object", &upload.upload_id, 1, &mut data, &opts)
.await
.expect("stage part");
Some((
upload.upload_id,
vec![CompletePart {
part_num: 1,
etag: part.etag,
..Default::default()
}],
))
} else {
None
};
let lock = store
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
.await
.expect("create metadata writer");
let writer = lock.get_write_lock(Duration::from_secs(5));
tokio::pin!(writer);
// Leave the writer registered while the replica continues. A second read
// acquisition would queue behind this writer, which needs our first read.
assert!(futures::poll!(&mut writer).is_pending());
timeout(Duration::from_secs(2), async {
if let Some((upload_id, parts)) = upload {
store
.clone()
.complete_multipart_upload(bucket, "object", &upload_id, parts, &opts)
.await
} else {
let mut data = PutObjReader::from_vec(b"replicated version".to_vec());
store.put_object(bucket, "object", &mut data, &opts).await
}
})
.await
.expect("replica must reuse its metadata read lock instead of waiting behind the writer")
.expect("replica PUT should commit");
assert!(futures::poll!(&mut writer).is_pending(), "the snapshot must still fence metadata updates");
let read_opts = ObjectOptions {
version_id: opts.version_id.clone(),
..Default::default()
};
drop(opts);
timeout(Duration::from_secs(2), writer)
.await
.expect("metadata writer must proceed after the snapshot drops")
.expect("acquire metadata writer");
let mut reader = store
.get_object_reader(bucket, "object", None, Default::default(), &read_opts)
.await
.expect("read committed replica version");
let mut bytes = Vec::new();
reader.stream.read_to_end(&mut bytes).await.expect("read replica body");
assert_eq!(bytes, b"replicated version");
}
#[tokio::test]
async fn reused_metadata_lock_still_enforces_quota() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "metadata-read-reuse-quota";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
store
.update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
.await
.expect("set quota");
let mut opts = ObjectOptions {
versioned: true,
version_id: Some(Uuid::new_v4().to_string()),
expected_bucket_incarnation_id: Some(store.bucket_incarnation_id(bucket).await.expect("load incarnation")),
object_lock_config_snapshot: Some(store.object_lock_config_snapshot(bucket).await.expect("capture snapshot")),
..Default::default()
};
assert!(opts.set_quota_admission(0, 1));
let lock = store
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
.await
.expect("create writer");
let writer = lock.get_write_lock(Duration::from_secs(5));
tokio::pin!(writer);
assert!(futures::poll!(&mut writer).is_pending());
let mut data = PutObjReader::from_vec(b"too large".to_vec());
let error = timeout(Duration::from_secs(2), store.put_object(bucket, "object", &mut data, &opts))
.await
.expect("quota admission must not reacquire the lock")
.expect_err("quota must reject growth");
assert!(matches!(error, Error::QuotaExceeded { limit: 1, .. }), "{error:?}");
drop(opts);
timeout(Duration::from_secs(2), writer)
.await
.expect("release metadata reader after denial")
.expect("acquire writer");
}
#[tokio::test]
async fn cancelled_put_releases_shared_metadata_guard() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "metadata-read-reuse-cancel";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let opts = ObjectOptions {
versioned: true,
version_id: Some(Uuid::new_v4().to_string()),
object_lock_config_snapshot: Some(store.object_lock_config_snapshot(bucket).await.expect("capture snapshot")),
..Default::default()
};
let barrier = PutObjectCommitBarrier::install(bucket, "object", PutObjectCommitPause::AfterQuotaReservation);
let put_store = Arc::clone(&store);
let put = tokio::spawn(async move {
let mut data = PutObjReader::from_vec(b"cancel before commit".to_vec());
put_store.put_object(bucket, "object", &mut data, &opts).await
});
barrier.wait_until_paused().await;
let lock = store
.new_ns_lock(".rustfs.sys", &format!("bucket-targets/{bucket}/transaction.lock"))
.await
.expect("create writer");
let writer = lock.get_write_lock(Duration::from_secs(5));
tokio::pin!(writer);
assert!(futures::poll!(&mut writer).is_pending());
put.abort();
assert!(put.await.expect_err("PUT should be cancelled").is_cancelled());
timeout(Duration::from_secs(2), writer)
.await
.expect("all metadata reader owners must drop after cancellation")
.expect("acquire writer");
}
#[tokio::test]
async fn multipart_rejects_metadata_snapshots_from_another_scope() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let (_other_dirs, other_store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&other_store), Vec::new()).await;
let bucket = "metadata-read-reuse-scope";
for target in [&store, &other_store] {
target
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
}
store
.make_bucket("other-bucket", &MakeBucketOptions::default())
.await
.expect("create other bucket");
let other_store_incarnation = other_store
.bucket_incarnation_id(bucket)
.await
.expect("load other store incarnation");
let other_bucket_incarnation = store
.bucket_incarnation_id("other-bucket")
.await
.expect("load other bucket incarnation");
for (snapshot_store, snapshot_bucket, expected) in [
(&other_store, bucket, other_store_incarnation),
(&store, "other-bucket", other_bucket_incarnation),
(&store, bucket, Uuid::new_v4()),
] {
let opts = ObjectOptions {
expected_bucket_incarnation_id: Some(expected),
object_lock_config_snapshot: Some(
snapshot_store
.object_lock_config_snapshot(snapshot_bucket)
.await
.expect("capture snapshot"),
),
..Default::default()
};
let error = store
.new_multipart_upload(bucket, "object", &opts)
.await
.expect_err("foreign snapshot must be rejected");
assert!(error.to_string().contains("valid metadata transaction fence"), "{error:?}");
}
}
#[tokio::test]
async fn delete_prefix_reuses_its_metadata_snapshot_for_generation_validation() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
let bucket = "metadata-read-reuse-delete";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let mut data = PutObjReader::from_vec(b"remove prefix".to_vec());
store
.put_object(bucket, "prefix/object", &mut data, &ObjectOptions::default())
.await
.expect("seed object");
store
.delete_object(
bucket,
"prefix/",
ObjectOptions {
delete_prefix: true,
..Default::default()
},
)
.await
.expect("delete prefix with an internally captured metadata snapshot");
let error = store
.get_object_info(bucket, "prefix/object", &ObjectOptions::default())
.await
.expect_err("prefix object must be deleted");
assert!(matches!(error, Error::ObjectNotFound(..)), "{error:?}");
}
+11
View File
@@ -6,6 +6,17 @@ pub(crate) use rustfs_ecstore::api::erasure::Erasure;
pub(crate) use rustfs_ecstore::api::object::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
pub(crate) use rustfs_ecstore::api::{error::Error, set_disk::SetDisks, storage::ECStore};
use rustfs_storage_api as storage_contracts;
#[cfg(feature = "test-util")]
pub(crate) mod metadata_lock {
pub(crate) use super::storage_contracts::{
BucketOperations, CompletePart, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
};
pub(crate) use super::{Error, ObjectOptions, PutObjReader};
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
};
pub(crate) use rustfs_ecstore::api::set_disk::test_util::{PutObjectCommitBarrier, PutObjectCommitPause};
}
pub(crate) mod contract_compat {
pub(crate) use super::storage_contracts::{
+36 -6
View File
@@ -105,13 +105,17 @@ pub struct HealObjectReceipt {
impl HealObjectReceipt {
pub(crate) fn verified_for(&self, expected: &HealObjectIdentity) -> bool {
matches!(
self.disposition,
let disposition_verifies = match self.disposition {
HealObjectDisposition::Repaired
| HealObjectDisposition::VerifiedHealthy
| HealObjectDisposition::MetadataHealthy
| HealObjectDisposition::AuthoritativelyAbsent
) && self.identity.kind == expected.kind
| HealObjectDisposition::VerifiedHealthy
| HealObjectDisposition::AuthoritativelyAbsent => true,
// A metadata/presence proof never certifies payload bytes, so it
// cannot discharge a request that exists to decode the payload.
HealObjectDisposition::MetadataHealthy => expected.kind != HealObjectKind::Decode,
_ => false,
};
disposition_verifies
&& self.identity.kind == expected.kind
&& self.identity.bucket == expected.bucket
&& self.identity.object == expected.object
&& self.identity.version_id == expected.version_id
@@ -530,6 +534,32 @@ mod canonical_outcome_tests {
assert!(serde_json::from_value::<HealTaskOutcome>(value).is_err());
}
#[test]
fn metadata_health_receipt_never_discharges_a_payload_decode_request() {
let incarnation = Uuid::new_v4();
let mut expected = HealObjectIdentity {
bucket_incarnation_id: Some(incarnation),
..item(HealObjectDisposition::Unknown).identity
};
let mut receipt = HealObjectReceipt {
identity: expected.clone(),
disposition: HealObjectDisposition::MetadataHealthy,
};
assert!(
receipt.verified_for(&expected),
"a presence proof still settles a metadata-level object request"
);
expected.kind = HealObjectKind::Decode;
receipt.identity.kind = HealObjectKind::Decode;
assert!(
!receipt.verified_for(&expected),
"a presence-only proof must not clear a payload decode responsibility"
);
receipt.disposition = HealObjectDisposition::VerifiedHealthy;
assert!(receipt.verified_for(&expected));
}
#[test]
fn canonical_outcome_categories_have_one_terminal_count() {
let mut outcome = HealTaskOutcome::default();
+3 -3
View File
@@ -74,11 +74,11 @@ hotpath-cpu = [
# Tokio runtime-level telemetry. Requires a `--cfg tokio_unstable` build; the
# build script fails the compile when that flag is missing. Off by default so
# ordinary builds neither pay for nor depend on Tokio's unstable API.
dial9 = ["dep:dial9-tokio-telemetry", "dial9-tokio-telemetry/process-resource"]
dial9 = ["dep:dial9"]
#
# NOTE: there is deliberately no `dial9-taskdump` feature. dial9 only captures a
# task dump for futures it wrapped itself, i.e. those spawned via
# `dial9_tokio_telemetry::spawn`. RustFS spawns with `tokio::spawn` throughout,
# `dial9::spawn`. RustFS spawns with `tokio::spawn` throughout,
# so enabling `tokio/taskdump` would cost a Linux-only build constraint and
# record nothing. Measured on an identical workload: 0 dumps via `tokio::spawn`,
# 14709 via `dial9::spawn`. Re-adding this feature only makes sense together
@@ -149,7 +149,7 @@ tracing-subscriber = { workspace = true, features = ["fmt", "env-filter", "traci
tokio = { workspace = true, features = ["sync", "rt-multi-thread", "time", "macros"] }
tokio-util = { workspace = true, features = ["io", "compat"] }
url.workspace = true
dial9-tokio-telemetry = { workspace = true, optional = true }
dial9 = { workspace = true, optional = true, features = ["tokio", "process-resource"] }
thiserror = { workspace = true }
zstd = { workspace = true, features = ["zstdmt"] }
sysinfo = { workspace = true }
+1 -1
View File
@@ -14,7 +14,7 @@
//! Enforces that the `dial9` feature is paired with `--cfg tokio_unstable`.
//!
//! `dial9-tokio-telemetry` hooks Tokio's unstable runtime instrumentation. The
//! `dial9` hooks Tokio's unstable runtime instrumentation. The
//! flag lives in `RUSTFLAGS`, which a caller can silently clobber by exporting
//! their own value — that used to turn telemetry off with no diagnostic. Fail
//! the build instead.
+4 -4
View File
@@ -14,10 +14,10 @@
//! Configuration for dial9 Tokio runtime telemetry.
//!
//! This module carries no dependency on the `dial9-tokio-telemetry` crate, so
//! it compiles identically whether or not the `dial9` feature is enabled. That
//! lets callers read the configured state (and export metrics about it) from a
//! binary that was built without telemetry support.
//! This module carries no dependency on the optional `dial9` crate, so it
//! compiles identically whether or not the `dial9` feature is enabled. That lets
//! callers read the configured state (and export metrics about it) from a binary
//! that was built without telemetry support.
use super::state::dial9_runtime_state;
use rustfs_config::{
+1 -1
View File
@@ -20,7 +20,7 @@
use crate::TelemetryError;
/// Placeholder for `dial9_tokio_telemetry::telemetry::TelemetryGuard`.
/// Placeholder for `dial9::TelemetryGuard`.
///
/// Never constructed: [`build_traced_runtime`] always fails without the
/// `dial9` feature.
+4 -5
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! `dial9-tokio-telemetry` integration, compiled when the `dial9` feature is on.
//! `dial9` integration, compiled when the `dial9` feature is on.
//!
//! Captures Tokio runtime-level events (poll start/end, worker park/unpark,
//! task spawn/terminate) into rotating binary trace segments. This is an
@@ -23,9 +23,8 @@ use super::config::Dial9Config;
use super::state::{dial9_runtime_state, measure_disk_usage_bytes};
use super::{EVENT_DIAL9_STATE, LOG_COMPONENT_OBS, LOG_SUBSYSTEM_DIAL9};
use crate::TelemetryError;
use dial9_tokio_telemetry::telemetry::{
Dial9Handle, Dial9HandleTokioExt, DiskBuffer, ProcessResourceUsageConfig, RecorderPerfExt, TokioAttachOptions, recorder,
};
use dial9::process::ProcessResourceUsageConfig;
use dial9::{Dial9Handle, Dial9HandleTokioExt, DiskBuffer, RecorderPerfExt, TokioAttachOptions, recorder};
use std::time::Duration;
use tracing::{info, warn};
@@ -134,7 +133,7 @@ pub fn build_traced_runtime(
.map(|runtime| (runtime, guard, shutdown));
// No task dumps here. dial9 captures a task dump only for futures it
// wrapped itself, i.e. those spawned via `dial9_tokio_telemetry::spawn`;
// wrapped itself, i.e. those spawned via `dial9::spawn`;
// `tokio::spawn` gets no wrapper. RustFS spawns with `tokio::spawn`
// throughout, so enabling task dumps records nothing. Measured on an
// identical workload: 0 dumps via `tokio::spawn`, 14709 via `dial9::spawn`.
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Tokio runtime-level telemetry via `dial9-tokio-telemetry`.
//! Tokio runtime-level telemetry via `dial9`.
//!
//! This is an on-demand profiler for executor-level faults — long polls that
//! stall a worker, park/unpark storms, tasks that never yield — which are
@@ -39,9 +39,9 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
| Client or engine | Status | Claim |
|---|---|---|
| PyIceberg | Automated | Namespace and table create, append, reload, scan, metadata-location, refs, views, maintenance, diagnostics, optional vended credentials with an exact-prefix data-plane scope check. |
| Spark Iceberg REST catalog | Manual/live harness | Pinned package inputs, catalog properties, SQL, expected `row_count=2`, and a CI opt-in gate for create/append/refresh/count/cleanup. Live execution and commit-conflict probing remain manual unless enabled in the runner. |
| Spark Iceberg REST catalog | Manual/live harness | Pinned package inputs, catalog properties, atomic CTAS SQL through staged create, expected `row_count=2`, and a CI opt-in gate for create/refresh/count/cleanup. Live execution and commit-conflict probing remain manual unless enabled in the runner. |
| Trino Iceberg REST catalog | Manual/live harness | Catalog properties and a read-only `SELECT COUNT(*)` against a PyIceberg- or Spark-created table. Write compatibility not claimed. |
| DuckDB Iceberg 1.5.5 | Automated | `duckdb_smoke.py` covers metadata-location read, single-table create/insert/update/delete/merge, schema evolution, snapshots, concurrent writers, drop, PyIceberg cross-read, and both signing profiles. Staged create, purge-on-drop, and format v3 are verified fail-closed. Two-table mode runs without claiming cross-table atomicity. AWS `ENDPOINT_TYPE S3_TABLES` and vended-credential integration not claimed. |
| DuckDB Iceberg 1.5.5 | Automated | `duckdb_smoke.py` covers metadata-location read, single-table create/insert/update/delete/merge, schema evolution, snapshots, concurrent writers, drop, PyIceberg cross-read, and both signing profiles. Purge-on-drop and format v3 are verified fail-closed. Two-table mode runs without claiming cross-table atomicity. AWS `ENDPOINT_TYPE S3_TABLES`, staged-create client integration, and vended-credential integration are not claimed. |
| StarRocks Iceberg REST catalog | Documented, not automated | External catalog read-path reference only. |
| Databend | Manual/live harness | S3 stage read probe for table data files only; Iceberg REST integration not claimed. |
| Snowflake Open Catalog / Iceberg integrations | Generated harness | Operator-adapted external volume/catalog SQL template only; live interop not claimed. |
@@ -62,7 +62,7 @@ RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on t
| Catalog config | Supported | `GET /v1/config` advertises defaults and only the supported OpenAPI REST paths in `endpoints`; RustFS extensions (administration, maintenance, migration, diagnostics, refs, metadata-location) are not presented as standard endpoints. |
| Table bucket discovery | Supported | `PUT` / `GET /v1/buckets/{warehouse}` enable and inspect table bucket state. |
| Namespaces | Supported | Create, list, load, exists, drop on both prefixes; `pageSize`/`pageToken` pagination with context-bound tokens; identifiers limited to 512 ASCII characters. |
| Tables | Supported | Create, register, list, load, exists, rename, commit, metadata-location get/update, drop on both prefixes. Rename uses a bucket-scoped persistent fence, recoverable intent, and conditional publication; the source name is reusable only via an ETag-conditional tombstone replacement. Commit identifiers must match the URL; unknown requirements/updates fail as bad requests; staged create, register overwrite, purge-on-drop, and v3-only encryption-key updates return an explicit unsupported-operation response. |
| Tables | Supported | Direct and staged create, register, list, load, exists, rename, commit, metadata-location get/update, and drop on both prefixes. Staged create is stateless until an `assert-create` commit atomically publishes the initial metadata and table entry. Rename uses a bucket-scoped persistent fence, recoverable intent, and conditional publication; the source name is reusable only via an ETag-conditional tombstone replacement. Commit identifiers must match the URL; unknown requirements/updates fail as bad requests; register overwrite, purge-on-drop, and v3-only encryption-key updates return an explicit unsupported-operation response. |
| Commit CAS | Supported | Single-table commits validate base metadata, version token, referenced object existence, warehouse scope, and Iceberg requirements before advancing the pointer; external metadata transitions preserve monotonic assignment watermarks and immutable retained definitions. `idempotency-key-lifetime` is not advertised; mutation-wide `Idempotency-Key` semantics are unsupported. |
| Commit recovery | Supported | Commit log, idempotency lookup, diagnostics, and recovery routes expose and repair finalization gaps without moving the pointer. |
| Snapshot refs | Supported | List, create/replace, delete via commits; `main` is protected; refs with explicit retention need forced delete. |
+3 -4
View File
@@ -3,9 +3,8 @@
**Use this when:** you need Tokio runtime-level evidence (which task held a worker, long polls, park/unpark behaviour) that Prometheus metrics and `tracing` spans cannot provide, or you are building or running the opt-in `dial9` profiling binary.
**Source of truth:** `crates/obs/src/telemetry/dial9/mod.rs` (session setup), `crates/obs/src/metrics/collectors/dial9.rs` (metrics), `crates/config/src/constants/runtime.rs` (`RUSTFS_RUNTIME_DIAL9_*` and defaults), `.config/make/build.mak` (`build-profiling`), `crates/obs/build.rs` (feature/cfg pairing check).
`dial9-tokio-telemetry` records Tokio runtime-level events — poll start/end,
worker park/unpark, task spawn/terminate, and optionally async backtraces of
stalled tasks — into binary trace segments.
`dial9` records Tokio runtime-level events — poll start/end, worker park/unpark,
task spawn/terminate — into binary trace segments.
It answers questions that Prometheus metrics and `tracing` spans cannot:
@@ -29,7 +28,7 @@ For drive stalls use the `rustfs_io_*` metrics and the drive-stall budget. dial9
answers a different question: which task held a worker, and for how long.
**It cannot tell you where a task was stuck.** That would need a task dump, and
dial9 only captures those for futures spawned through `dial9_tokio_telemetry::spawn`.
dial9 only captures those for futures spawned through `dial9::spawn`.
RustFS spawns with `tokio::spawn` throughout, so no task dump is ever recorded and
no configuration exposes one. Tracked as D9-16 in rustfs/backlog#1157.
+302 -25
View File
@@ -910,6 +910,16 @@ struct RestLoadTableResponse {
storage_credentials: Vec<RestStorageCredential>,
}
#[derive(Debug, Serialize)]
struct RestCreateTableResponse {
#[serde(rename = "metadata-location")]
metadata_location: Option<String>,
metadata: serde_json::Value,
config: BTreeMap<String, String>,
#[serde(rename = "storage-credentials")]
storage_credentials: Vec<RestStorageCredential>,
}
#[derive(Debug, Serialize)]
struct RestLoadCredentialsResponse {
config: BTreeMap<String, String>,
@@ -2706,6 +2716,25 @@ fn load_table_response_from_entry(entry: crate::table_catalog::TableEntry, metad
}
}
fn create_table_response_from_entry(
entry: crate::table_catalog::TableEntry,
metadata: serde_json::Value,
staged: bool,
) -> RestCreateTableResponse {
let RestLoadTableResponse {
metadata_location,
metadata,
config,
storage_credentials,
} = load_table_response_from_entry(entry, metadata);
RestCreateTableResponse {
metadata_location: (!staged).then_some(metadata_location),
metadata,
config,
storage_credentials,
}
}
fn load_view_response_from_entry(entry: crate::table_catalog::ViewEntry, metadata: serde_json::Value) -> RestLoadViewResponse {
let mut config = BTreeMap::new();
let warehouse_location = entry.warehouse_location.clone();
@@ -3104,16 +3133,9 @@ fn table_entry_from_create_table_request(
mut schema,
mut partition_spec,
mut write_order,
stage_create,
stage_create: _,
mut properties,
} = request;
if stage_create {
return Err(iceberg_rest_error(
ICEBERG_ERROR_UNSUPPORTED_OPERATION,
StatusCode::NOT_ACCEPTABLE,
"stage-create is not supported",
));
}
let table = crate::table_catalog::IdentifierSegment::parse(name)
.map_err(|err| s3_error!(InvalidRequest, "invalid table name: {}", err))?;
@@ -3490,6 +3512,26 @@ fn validate_table_commit_requirements(metadata: &serde_json::Value, requirements
Ok(())
}
fn request_has_assert_create_requirement(request: &RestCommitTableRequest) -> bool {
request
.requirements
.iter()
.any(|requirement| requirement.get("type").and_then(serde_json::Value::as_str) == Some("assert-create"))
}
fn validate_create_table_commit_requirements(requirements: &[serde_json::Value]) -> S3Result<()> {
if requirements.is_empty()
|| requirements
.iter()
.any(|requirement| requirement.get("type").and_then(serde_json::Value::as_str) != Some("assert-create"))
{
return Err(S3Error::from(ApiError::invalid_request(
"create table commit requires only assert-create requirements",
)));
}
Ok(())
}
fn validate_i64_requirement(
metadata: &serde_json::Value,
requirement: &serde_json::Value,
@@ -3555,20 +3597,104 @@ fn apply_table_commit_updates(
}
fn apply_table_commit_updates_at(
mut metadata: serde_json::Value,
metadata: serde_json::Value,
updates: &[serde_json::Value],
previous_metadata_location: &str,
commit_timestamp_ms: i64,
) -> S3Result<serde_json::Value> {
apply_table_updates_at(
metadata,
updates,
TableMetadataUpdateMode::Existing {
previous_metadata_location,
},
commit_timestamp_ms,
)
}
fn apply_table_create_updates_at(updates: &[serde_json::Value], commit_timestamp_ms: i64) -> S3Result<serde_json::Value> {
let format_version = updates
.iter()
.find(|update| update.get("action").and_then(serde_json::Value::as_str) == Some("upgrade-format-version"))
.and_then(|update| update.get("format-version"))
.and_then(serde_json::Value::as_i64)
.unwrap_or(2);
let mut metadata = serde_json::json!({
"format-version": format_version,
"last-updated-ms": commit_timestamp_ms,
"last-column-id": 0,
"schemas": [],
"current-schema-id": -1,
"partition-specs": [],
"default-spec-id": -1,
"last-partition-id": 999,
"sort-orders": [],
"default-sort-order-id": -1,
"properties": {},
"current-snapshot-id": -1,
"snapshots": [],
"snapshot-log": [],
"metadata-log": [],
"refs": {}
});
if format_version == 2 {
metadata_object_mut(&mut metadata)?.insert("last-sequence-number".to_string(), serde_json::Value::from(0));
}
apply_table_updates_at(metadata, updates, TableMetadataUpdateMode::Create, commit_timestamp_ms)
}
#[derive(Debug, Clone, Copy)]
enum TableMetadataUpdateMode<'a> {
Create,
Existing { previous_metadata_location: &'a str },
}
impl TableMetadataUpdateMode<'_> {
fn initial_catalog_id(self, array_key: &str) -> Option<i64> {
match (self, array_key) {
(Self::Create, "schemas" | "partition-specs") => Some(0),
(Self::Create, "sort-orders") => Some(1),
(Self::Create, _) | (Self::Existing { .. }, _) => None,
}
}
}
fn apply_table_updates_at(
mut metadata: serde_json::Value,
updates: &[serde_json::Value],
mode: TableMetadataUpdateMode<'_>,
commit_timestamp_ms: i64,
) -> S3Result<serde_json::Value> {
if !metadata.is_object() {
return Err(s3_error!(InvalidRequest, "current table metadata must be a JSON object"));
}
if metadata.get("format-version").is_some() {
if matches!(mode, TableMetadataUpdateMode::Existing { .. }) && metadata.get("format-version").is_some() {
crate::table_catalog::synchronize_table_metadata_version_fields(&mut metadata).map_err(catalog_store_error)?;
}
let mut next_schema_id = next_catalog_id_for_updates(&metadata, updates, "add-schema", "schemas", "schema-id")?;
let mut next_spec_id = next_catalog_id_for_updates(&metadata, updates, "add-spec", "partition-specs", "spec-id")?;
let mut next_sort_order_id = next_catalog_id_for_updates(&metadata, updates, "add-sort-order", "sort-orders", "order-id")?;
let mut next_schema_id = next_catalog_id_for_updates(
&metadata,
updates,
"add-schema",
"schemas",
"schema-id",
mode.initial_catalog_id("schemas"),
)?;
let mut next_spec_id = next_catalog_id_for_updates(
&metadata,
updates,
"add-spec",
"partition-specs",
"spec-id",
mode.initial_catalog_id("partition-specs"),
)?;
let mut next_sort_order_id = next_catalog_id_for_updates(
&metadata,
updates,
"add-sort-order",
"sort-orders",
"order-id",
mode.initial_catalog_id("sort-orders"),
)?;
let mut last_added_schema_id = None;
let mut last_added_spec_id = None;
let mut last_added_sort_order_id = None;
@@ -3658,8 +3784,16 @@ fn apply_table_commit_updates_at(
if metadata.get("format-version").is_some() {
crate::table_catalog::synchronize_table_metadata_version_fields(&mut metadata).map_err(catalog_store_error)?;
}
append_previous_metadata_log(&mut metadata, previous_metadata_location)?;
if let TableMetadataUpdateMode::Existing {
previous_metadata_location,
} = mode
{
append_previous_metadata_log(&mut metadata, previous_metadata_location)?;
}
metadata_object_mut(&mut metadata)?.insert("last-updated-ms".to_string(), serde_json::Value::from(commit_timestamp_ms));
if matches!(mode, TableMetadataUpdateMode::Create) {
crate::table_catalog::validate_supported_table_metadata(&metadata).map_err(catalog_store_error)?;
}
Ok(metadata)
}
@@ -3726,7 +3860,7 @@ fn apply_view_commit_updates_at(
if !metadata.is_object() {
return Err(s3_error!(InvalidRequest, "current view metadata must be a JSON object"));
}
let mut next_schema_id = next_catalog_id_for_updates(&metadata, updates, "add-schema", "schemas", "schema-id")?;
let mut next_schema_id = next_catalog_id_for_updates(&metadata, updates, "add-schema", "schemas", "schema-id", None)?;
let mut last_added_schema_id = None;
let mut last_added_view_version_id = None;
let mut added_view_version_timestamps = BTreeMap::new();
@@ -5084,11 +5218,12 @@ fn next_catalog_id_for_updates(
action: &str,
array_key: &str,
id_key: &str,
initial_id: Option<i64>,
) -> S3Result<Option<i64>> {
updates
.iter()
.any(|update| update.get("action").and_then(serde_json::Value::as_str) == Some(action))
.then(|| next_array_object_i64(metadata, array_key, id_key))
.then(|| next_array_object_i64(metadata, array_key, id_key, initial_id))
.transpose()
}
@@ -5103,10 +5238,22 @@ fn take_catalog_assigned_id(next_id: &mut Option<i64>, label: &str) -> S3Result<
Ok(assigned_id)
}
fn next_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: &str) -> S3Result<i64> {
last_array_object_i64(metadata, array_key, id_key)?
.checked_add(1)
.ok_or_else(|| s3_error!(InvalidRequest, "metadata field {array_key} {id_key} exceeds the signed 64-bit range"))
fn next_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: &str, initial_id: Option<i64>) -> S3Result<i64> {
match last_array_object_i64(metadata, array_key, id_key) {
Ok(last_id) => last_id
.checked_add(1)
.ok_or_else(|| s3_error!(InvalidRequest, "metadata field {array_key} {id_key} exceeds the signed 64-bit range")),
Err(_)
if metadata
.get(array_key)
.and_then(serde_json::Value::as_array)
.is_some_and(Vec::is_empty) =>
{
initial_id
.ok_or_else(|| S3Error::from(ApiError::invalid_request(format!("metadata field {array_key} has no {id_key}"))))
}
Err(err) => Err(err),
}
}
fn last_array_object_i64(metadata: &serde_json::Value, array_key: &str, id_key: &str) -> S3Result<i64> {
@@ -5435,11 +5582,40 @@ async fn create_table_response<S>(
namespace: &crate::table_catalog::Namespace,
request: CreateTableRequest,
table_bucket_enabled: bool,
) -> S3Result<RestLoadTableResponse>
) -> S3Result<RestCreateTableResponse>
where
S: crate::table_catalog::TableCatalogStore + ?Sized,
{
let staged = request.stage_create;
let (entry, metadata) = table_entry_from_create_table_request(bucket, namespace, request)?;
if staged {
if !table_bucket_enabled {
return Err(S3Error::from(ApiError::invalid_request(format!("bucket {bucket} is not table-enabled"))));
}
get_namespace_response(store, bucket, namespace).await?;
if store
.load_table(bucket, &namespace.public_name(), &entry.table)
.await
.map_err(catalog_store_error)?
.is_some()
|| store
.load_view(bucket, &namespace.public_name(), &entry.table)
.await
.map_err(catalog_store_error)?
.is_some()
{
return Err(iceberg_rest_error(
ICEBERG_ERROR_ALREADY_EXISTS,
StatusCode::CONFLICT,
format!("catalog object already exists: {}/{}/{}", bucket, namespace.public_name(), entry.table),
));
}
store
.ensure_table_warehouse_location_available(&entry)
.await
.map_err(catalog_store_already_exists_error)?;
return Ok(create_table_response_from_entry(entry, metadata, true));
}
ensure_table_bucket_entry(store, bucket, table_bucket_enabled).await?;
crate::table_catalog::TableCommitPublication::begin_table_bucket(metadata_backend, bucket)
.await
@@ -5465,7 +5641,7 @@ where
.register_table_with_publication(entry.clone(), metadata_backend)
.await
.map_err(catalog_store_already_exists_error)?;
Ok(load_table_response_from_entry(entry, metadata))
Ok(create_table_response_from_entry(entry, metadata, false))
}
async fn create_view_response<S>(
@@ -6080,11 +6256,14 @@ async fn standard_commit_table_response<S>(
where
S: crate::table_catalog::TableCatalogStore + ?Sized,
{
let Some(current) = store
let current = store
.load_table(bucket, &namespace.public_name(), table)
.await
.map_err(catalog_store_error)?
else {
.map_err(catalog_store_error)?;
let Some(current) = current else {
if request_has_assert_create_requirement(&request) {
return commit_staged_table_create_response(store, metadata_backend, bucket, namespace, table, request).await;
}
return Err(iceberg_rest_error(ICEBERG_ERROR_NO_SUCH_TABLE, StatusCode::NOT_FOUND, "table not found"));
};
if let Some(response) =
@@ -6194,6 +6373,104 @@ where
Ok(commit_table_response_from_result(result, next_metadata))
}
async fn commit_staged_table_create_response<S>(
store: &S,
metadata_backend: &impl crate::table_catalog::TableCatalogObjectBackend,
bucket: &str,
namespace: &crate::table_catalog::Namespace,
table: &str,
request: RestCommitTableRequest,
) -> S3Result<RestCommitTableResponse>
where
S: crate::table_catalog::TableCatalogStore + ?Sized,
{
validate_create_table_commit_requirements(&request.requirements)?;
get_namespace_response(store, bucket, namespace).await?;
if store
.load_view(bucket, &namespace.public_name(), table)
.await
.map_err(catalog_store_error)?
.is_some()
{
return Err(iceberg_rest_error(
ICEBERG_ERROR_ALREADY_EXISTS,
StatusCode::CONFLICT,
format!("catalog object already exists: {}/{}/{}", bucket, namespace.public_name(), table),
));
}
let commit_timestamp_ms = current_time_millis();
let metadata = apply_table_create_updates_at(&request.updates, commit_timestamp_ms)?;
validate_metadata_table_location_in_bucket(bucket, &metadata)?;
let table = crate::table_catalog::IdentifierSegment::parse(table.to_string())
.map_err(|err| S3Error::from(ApiError::invalid_request(format!("invalid table name: {err}"))))?;
let (commit_id, _) = standard_commit_ids(request.commit_id.or(request.idempotency_key));
let table_id = Uuid::new_v4().to_string();
let metadata_location =
crate::table_catalog::default_table_metadata_file_path(namespace, &table, &next_metadata_file_name(1, &table_id));
let properties = serde_json::from_value::<BTreeMap<String, String>>(
metadata
.get("properties")
.cloned()
.unwrap_or_else(|| serde_json::Value::Object(serde_json::Map::new())),
)
.map_err(|_| S3Error::from(ApiError::invalid_request("table property values must be strings")))?;
let mut entry = crate::table_catalog::TableEntry {
version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION,
table_bucket: bucket.to_string(),
namespace: namespace.public_name(),
table: table.as_str().to_string(),
table_id,
table_uuid: Uuid::new_v4().to_string(),
format: "ICEBERG".to_string(),
format_version: 1,
warehouse_location: metadata_table_location(&metadata)?.to_string(),
metadata_location,
version_token: format!("token-{}", Uuid::new_v4()),
generation: 1,
state: crate::table_catalog::TableCatalogEntryState::Active,
properties,
created_at: None,
updated_at: None,
};
adopt_registered_metadata_identity(&mut entry, &metadata)?;
validate_table_metadata_snapshot_graph(metadata_backend, bucket, &entry, None, &metadata).await?;
crate::table_catalog::TableCommitPublication::begin_table_bucket(metadata_backend, bucket)
.await
.map_err(catalog_store_error)?;
if !crate::table_catalog::TableCommitPublication::holds_table_bucket(metadata_backend, bucket) {
return Err(catalog_store_error(crate::table_catalog::TableCatalogStoreError::Internal(
"staged table creation requires a table-bucket publication fence".to_string(),
)));
}
let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(metadata_backend);
let metadata_data = serde_json::to_vec(&metadata).map_err(|err| {
S3Error::with_message(S3ErrorCode::InternalError, format!("failed to serialize initial table metadata: {err}"))
})?;
metadata_backend
.put_object(
bucket,
&entry.metadata_location,
metadata_data,
crate::table_catalog::TableCatalogPutPrecondition::IfAbsent,
)
.await
.map_err(catalog_store_already_exists_error)?;
store
.register_table_with_publication(entry.clone(), metadata_backend)
.await
.map_err(catalog_store_already_exists_error)?;
Ok(RestCommitTableResponse {
metadata_location: table_metadata_location_for_client(bucket, &entry.metadata_location),
metadata: table_metadata_for_client(bucket, metadata),
version_token: entry.version_token,
generation: entry.generation,
commit_id,
})
}
async fn table_commit_for_retry<S>(
store: &S,
bucket: &str,
@@ -203,6 +203,10 @@ impl Operation for RestCommitTableHandler {
install_table_catalog_s3_request_info(&mut req, &principal)?;
ensure_table_bucket_enabled_from_extensions(&req.extensions, &warehouse).await?;
let request = read_rest_commit_table_request(std::mem::take(&mut req.input)).await?;
if request_has_assert_create_requirement(&request) {
let namespace_resource = TableCatalogResource::namespace(&warehouse, &namespace);
authorize_table_catalog_resource_request(&req, &namespace_resource, AdminAction::CreateTableAction).await?;
}
let metadata_backend = table_catalog_backend_from_extensions(&req.extensions)?;
let store = table_catalog_store_from_backend(metadata_backend.clone())?;
let commit_backend = TableCommitObjectBackend::for_request(metadata_backend, req);
+498 -17
View File
@@ -668,6 +668,11 @@ fn table_catalog_handlers_require_table_admin_actions() {
);
}
let commit_table_block = operation_block(&src, "RestCommitTableHandler");
assert!(commit_table_block.contains("request_has_assert_create_requirement(&request)"));
assert!(commit_table_block.contains("TableCatalogResource::namespace(&warehouse, &namespace)"));
assert!(commit_table_block.contains("AdminAction::CreateTableAction"));
let sync_bridge_block = operation_block(&src, "SyncExternalCatalogBridgeHandler");
assert!(
sync_bridge_block.contains("AdminAction::RegisterTableAction"),
@@ -2825,7 +2830,7 @@ async fn commit_request_readers_require_standard_arrays_and_preserve_legacy_poin
}
#[test]
fn unsupported_create_and_register_modes_return_iceberg_errors() {
fn register_overwrite_returns_iceberg_unsupported_error() {
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
let register_error = table_entry_from_register_request(
"warehouse",
@@ -2839,17 +2844,6 @@ fn unsupported_create_and_register_modes_return_iceberg_errors() {
.expect_err("register overwrite should remain unsupported");
assert_eq!(register_error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_UNSUPPORTED_OPERATION.into()));
assert_eq!(register_error.status_code(), Some(StatusCode::NOT_ACCEPTABLE));
let create_request: CreateTableRequest = serde_json::from_value(serde_json::json!({
"name": "events",
"schema": {"type": "struct", "schema-id": 0, "fields": []},
"stage-create": true
}))
.expect("stage-create request should parse");
let create_error = table_entry_from_create_table_request("warehouse", &namespace, create_request)
.expect_err("staged create should remain unsupported");
assert_eq!(create_error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_UNSUPPORTED_OPERATION.into()));
assert_eq!(create_error.status_code(), Some(StatusCode::NOT_ACCEPTABLE));
}
#[test]
@@ -2973,10 +2967,10 @@ async fn create_table_response_writes_initial_metadata_for_standard_request() {
.expect("table should exist");
assert_eq!(
response.metadata_location,
format!(
Some(format!(
"s3://warehouse/.rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/00001-{}.metadata.json",
entry.table_id
)
))
);
assert_eq!(response.metadata["table-uuid"], entry.table_uuid);
assert!(
@@ -3380,7 +3374,7 @@ async fn concurrent_create_table_responses_keep_one_catalog_winner_with_distinct
let winner_entry = &tables[0];
assert_eq!(
winner.metadata_location,
table_metadata_location_for_client("warehouse", &winner_entry.metadata_location)
Some(table_metadata_location_for_client("warehouse", &winner_entry.metadata_location))
);
let metadata_prefix = winner_entry
.metadata_location
@@ -3415,6 +3409,423 @@ async fn concurrent_create_table_responses_keep_one_catalog_winner_with_distinct
assert!(table_uuids.contains(&winner_entry.table_uuid));
}
#[tokio::test]
async fn staged_create_is_invisible_until_assert_create_commit_across_backings() {
for mode in [
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
crate::table_catalog::TableCatalogBackingMode::DurableStrong,
] {
let metadata_backend = TestTableCatalogObjectBackend::content_addressed();
let store = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(metadata_backend.clone(), mode);
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
ensure_table_bucket_entry(&store, "warehouse", true)
.await
.expect("table bucket entry should be seeded");
create_namespace_response(
&store,
"warehouse",
CreateNamespaceRequest {
namespace: vec!["analytics".to_string()],
properties: BTreeMap::new(),
},
true,
)
.await
.expect("namespace should be created");
let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone());
let staged =
create_table_response(&store, &commit_backend, "warehouse", &namespace, staged_events_create_request(), true)
.await
.expect("stage-create should return initialized metadata");
assert_eq!(staged.metadata_location, None, "{mode:?}");
let staged_json = serde_json::to_value(&staged).expect("staged response should serialize");
assert!(staged_json["metadata-location"].is_null(), "{mode:?}");
assert!(
store
.load_table("warehouse", "analytics", "events")
.await
.expect("table lookup should succeed")
.is_none(),
"{mode:?}"
);
let metadata_prefix = ".rustfs-table/warehouses/default/namespaces/analytics/tables/events/metadata/";
assert!(
metadata_backend
.list_objects("warehouse", metadata_prefix)
.await
.expect("metadata listing should succeed")
.is_empty(),
"{mode:?}"
);
let committed = commit_table_response(
&store,
&commit_backend,
"warehouse",
&namespace,
"events",
staged_create_commit_request(&staged.metadata, []),
)
.await
.expect("assert-create commit should publish the table");
assert_eq!(committed.generation, 1, "{mode:?}");
assert_eq!(committed.metadata["table-uuid"], staged.metadata["table-uuid"], "{mode:?}");
assert_eq!(committed.metadata["metadata-log"], serde_json::json!([]), "{mode:?}");
let entry = store
.load_table("warehouse", "analytics", "events")
.await
.expect("table lookup should succeed")
.expect("committed table should be visible");
assert_eq!(entry.table_uuid, staged.metadata["table-uuid"].as_str().unwrap(), "{mode:?}");
assert_eq!(
committed.metadata_location,
table_metadata_location_for_client("warehouse", &entry.metadata_location),
"{mode:?}"
);
assert!(
metadata_backend
.object_exists("warehouse", &entry.metadata_location)
.await
.expect("metadata lookup should succeed"),
"{mode:?}"
);
}
}
#[tokio::test]
async fn staged_create_commit_accepts_initial_snapshot_for_spark_ctas() {
let store = TestTableCatalogStore::default();
let metadata_backend = TestTableCatalogObjectBackend::content_addressed();
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
ensure_table_bucket_entry(&store, "warehouse", true)
.await
.expect("table bucket entry should be seeded");
create_namespace_response(
&store,
"warehouse",
CreateNamespaceRequest {
namespace: vec!["analytics".to_string()],
properties: BTreeMap::new(),
},
true,
)
.await
.expect("namespace should be created");
let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone());
let staged = create_table_response(&store, &commit_backend, "warehouse", &namespace, staged_events_create_request(), true)
.await
.expect("stage-create should succeed");
let table_location = staged.metadata["location"]
.as_str()
.expect("table location should be present");
let manifest_list = format!("{table_location}/metadata/snap-10.avro");
let data_file = format!("{table_location}/data/part-10.parquet");
seed_test_snapshot_manifest(&metadata_backend, "warehouse", &manifest_list, 10, 1, &[(&data_file, 0, 1, 10, 1)]).await;
let committed = commit_table_response(
&store,
&commit_backend,
"warehouse",
&namespace,
"events",
staged_create_commit_request(
&staged.metadata,
[
serde_json::json!({
"action": "add-snapshot",
"snapshot": {
"snapshot-id": 10,
"sequence-number": 1,
"timestamp-ms": 1234,
"manifest-list": manifest_list,
"summary": {"operation": "append"}
}
}),
serde_json::json!({
"action": "set-snapshot-ref",
"ref-name": "main",
"snapshot-id": 10,
"type": "branch"
}),
],
),
)
.await
.expect("Spark CTAS-style create commit should succeed");
assert_eq!(committed.metadata["current-snapshot-id"], 10);
assert_eq!(committed.metadata["last-sequence-number"], 1);
assert_eq!(committed.metadata["refs"]["main"]["snapshot-id"], 10);
}
#[test]
fn staged_create_commit_defaults_to_format_version_two_without_upgrade() {
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
let (_, staged_metadata) = table_entry_from_create_table_request("warehouse", &namespace, staged_events_create_request())
.expect("staged create metadata should initialize");
let mut request = staged_create_commit_request(&staged_metadata, []);
request
.updates
.retain(|update| update.get("action").and_then(serde_json::Value::as_str) != Some("upgrade-format-version"));
let metadata = apply_table_create_updates_at(&request.updates, 1234)
.expect("create updates without an explicit upgrade should use the Iceberg default");
assert_eq!(metadata["format-version"], 2);
assert_eq!(metadata["last-sequence-number"], 0);
}
#[tokio::test]
async fn staged_create_rejects_an_active_table_warehouse_location() {
for mode in [
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
crate::table_catalog::TableCatalogBackingMode::DurableStrong,
] {
let catalog_backend = TestTableCatalogObjectBackend::content_addressed();
let metadata_backend = TestTableCatalogObjectBackend::content_addressed();
let store = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(catalog_backend, mode);
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
ensure_table_bucket_entry(&store, "warehouse", true)
.await
.expect("table bucket entry should be seeded");
create_namespace_response(
&store,
"warehouse",
CreateNamespaceRequest {
namespace: vec!["analytics".to_string()],
properties: BTreeMap::new(),
},
true,
)
.await
.expect("namespace should be created");
let commit_backend = TableCommitObjectBackend::trusted(metadata_backend);
let active_request = serde_json::from_value::<CreateTableRequest>(serde_json::json!({
"name": "active_events",
"location": "s3://warehouse/shared/events",
"schema": {"type": "struct", "fields": []}
}))
.expect("active create request should parse");
create_table_response(&store, &commit_backend, "warehouse", &namespace, active_request, true)
.await
.expect("active table should be created");
let staged_request = serde_json::from_value::<CreateTableRequest>(serde_json::json!({
"name": "staged_events",
"location": "s3://warehouse/shared/events/child",
"schema": {"type": "struct", "fields": []},
"stage-create": true
}))
.expect("staged create request should parse");
let error = create_table_response(&store, &commit_backend, "warehouse", &namespace, staged_request, true)
.await
.expect_err("stage-create must reject a location owned by an active table");
assert_eq!(error.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_ALREADY_EXISTS.into()), "{mode:?}");
assert_eq!(error.status_code(), Some(StatusCode::CONFLICT), "{mode:?}");
assert!(
store
.load_table("warehouse", "analytics", "staged_events")
.await
.expect("table lookup should succeed")
.is_none(),
"{mode:?}"
);
}
}
#[tokio::test]
async fn concurrent_staged_create_commits_publish_exactly_one_table() {
for mode in [
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
crate::table_catalog::TableCatalogBackingMode::DurableStrong,
] {
let catalog_backend = TestTableCatalogObjectBackend::content_addressed();
let barrier = Arc::new(tokio::sync::Barrier::new(2));
let metadata_backend = TestTableCatalogObjectBackend {
put_object_barrier: Some(Arc::clone(&barrier)),
..TestTableCatalogObjectBackend::content_addressed()
};
let store = Arc::new(crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(catalog_backend, mode));
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
ensure_table_bucket_entry(store.as_ref(), "warehouse", true)
.await
.expect("table bucket entry should be seeded");
create_namespace_response(
store.as_ref(),
"warehouse",
CreateNamespaceRequest {
namespace: vec!["analytics".to_string()],
properties: BTreeMap::new(),
},
true,
)
.await
.expect("namespace should be created");
let staged = create_table_response(
store.as_ref(),
&TableCommitObjectBackend::trusted(metadata_backend.clone()),
"warehouse",
&namespace,
staged_events_create_request(),
true,
)
.await
.expect("stage-create should succeed");
let first_store = Arc::clone(&store);
let first_namespace = namespace.clone();
let first_backend = TableCommitObjectBackend::trusted(metadata_backend.clone());
let first_request = staged_create_commit_request(&staged.metadata, []);
let first = tokio::spawn(async move {
commit_table_response(
first_store.as_ref(),
&first_backend,
"warehouse",
&first_namespace,
"events",
first_request,
)
.await
});
tokio::time::timeout(StdDuration::from_secs(2), async {
while metadata_backend.state.lock().await.objects.is_empty() {
tokio::task::yield_now().await;
}
})
.await
.expect("first metadata write should reach the publication pause");
metadata_backend.lock_attempts.lock().await.clear();
let second_store = Arc::clone(&store);
let second_namespace = namespace.clone();
let second_backend = TableCommitObjectBackend::trusted(metadata_backend.clone());
let second_request = staged_create_commit_request(&staged.metadata, []);
let second = tokio::spawn(async move {
commit_table_response(
second_store.as_ref(),
&second_backend,
"warehouse",
&second_namespace,
"events",
second_request,
)
.await
});
metadata_backend.wait_for_lock_attempts(1).await;
assert!(
!second.is_finished(),
"second create must wait after observing the table as absent: {mode:?}"
);
barrier.wait().await;
let _winner = tokio::time::timeout(StdDuration::from_secs(2), first)
.await
.expect("first assert-create commit should complete")
.expect("first assert-create task should join")
.expect("first assert-create commit should win");
tokio::time::timeout(StdDuration::from_secs(2), async {
while metadata_backend.state.lock().await.objects.len() < 2 {
tokio::task::yield_now().await;
}
})
.await
.expect("second metadata write should reach the publication pause");
barrier.wait().await;
let loser = tokio::time::timeout(StdDuration::from_secs(2), second)
.await
.expect("second assert-create commit should complete")
.expect("second assert-create task should join")
.expect_err("second assert-create commit must lose at atomic registration");
assert_eq!(loser.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_ALREADY_EXISTS.into()), "{mode:?}");
assert_eq!(loser.status_code(), Some(StatusCode::CONFLICT), "{mode:?}");
assert_eq!(
store
.list_tables("warehouse", "analytics")
.await
.expect("table listing should succeed")
.len(),
1,
"{mode:?}"
);
}
}
#[tokio::test]
async fn malformed_or_conflicting_assert_create_does_not_replace_a_table() {
let store = TestTableCatalogStore::default();
let metadata_backend = TestTableCatalogObjectBackend::content_addressed();
let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse");
ensure_table_bucket_entry(&store, "warehouse", true)
.await
.expect("table bucket entry should be seeded");
create_namespace_response(
&store,
"warehouse",
CreateNamespaceRequest {
namespace: vec!["analytics".to_string()],
properties: BTreeMap::new(),
},
true,
)
.await
.expect("namespace should be created");
let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone());
let malformed = serde_json::from_value(serde_json::json!({
"requirements": [{"type": "assert-create"}],
"updates": []
}))
.expect("malformed create commit should parse");
let error = commit_table_response(&store, &commit_backend, "warehouse", &namespace, "events", malformed)
.await
.expect_err("incomplete create updates must fail");
assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST));
assert!(
store
.load_table("warehouse", "analytics", "events")
.await
.expect("table lookup should succeed")
.is_none()
);
let staged = create_table_response(&store, &commit_backend, "warehouse", &namespace, staged_events_create_request(), true)
.await
.expect("stage-create should succeed");
let first = commit_table_response(
&store,
&commit_backend,
"warehouse",
&namespace,
"events",
staged_create_commit_request(&staged.metadata, []),
)
.await
.expect("first assert-create should succeed");
let error = commit_table_response(
&store,
&commit_backend,
"warehouse",
&namespace,
"events",
staged_create_commit_request(&staged.metadata, []),
)
.await
.expect_err("assert-create must fail after the table exists");
assert_eq!(error.status_code(), Some(StatusCode::CONFLICT));
let current = store
.load_table("warehouse", "analytics", "events")
.await
.expect("table lookup should succeed")
.expect("table should remain visible");
assert_eq!(current.generation, first.generation);
assert_eq!(
table_metadata_location_for_client("warehouse", &current.metadata_location),
first.metadata_location
);
}
#[tokio::test]
async fn standard_commit_applies_updates_and_writes_next_metadata() {
let store = TestTableCatalogStore::default();
@@ -10868,6 +11279,68 @@ async fn seed_test_manifest_data_files(
}
}
fn staged_events_create_request() -> CreateTableRequest {
serde_json::from_value(serde_json::json!({
"name": "events",
"schema": {
"type": "struct",
"schema-id": 0,
"fields": [
{"id": 1, "name": "id", "required": true, "type": "long"},
{"id": 2, "name": "payload", "required": false, "type": "string"}
]
},
"stage-create": true,
"properties": {"write.format.default": "parquet"}
}))
.expect("staged create table request should parse")
}
fn staged_create_commit_request(
staged_metadata: &serde_json::Value,
additional_updates: impl IntoIterator<Item = serde_json::Value>,
) -> RestCommitTableRequest {
let mut updates = vec![
serde_json::json!({
"action": "assign-uuid",
"uuid": staged_metadata["table-uuid"]
}),
serde_json::json!({
"action": "upgrade-format-version",
"format-version": staged_metadata["format-version"]
}),
serde_json::json!({
"action": "add-schema",
"schema": staged_metadata["schemas"][0]
}),
serde_json::json!({"action": "set-current-schema", "schema-id": -1}),
serde_json::json!({
"action": "add-spec",
"spec": staged_metadata["partition-specs"][0]
}),
serde_json::json!({"action": "set-default-spec", "spec-id": -1}),
serde_json::json!({
"action": "add-sort-order",
"sort-order": staged_metadata["sort-orders"][0]
}),
serde_json::json!({"action": "set-default-sort-order", "sort-order-id": -1}),
serde_json::json!({
"action": "set-location",
"location": staged_metadata["location"]
}),
serde_json::json!({
"action": "set-properties",
"updates": staged_metadata["properties"]
}),
];
updates.extend(additional_updates);
serde_json::from_value(serde_json::json!({
"requirements": [{"type": "assert-create"}],
"updates": updates
}))
.expect("staged create commit request should parse")
}
async fn create_standard_events_table<S>(
store: &S,
metadata_backend: &TestTableCatalogObjectBackend,
@@ -10907,9 +11380,17 @@ where
}))
.expect("standard create table request should parse");
let commit_backend = TableCommitObjectBackend::trusted(metadata_backend.clone());
create_table_response(store, &commit_backend, "warehouse", namespace, create_request, true)
let response = create_table_response(store, &commit_backend, "warehouse", namespace, create_request, true)
.await
.expect("table should be created")
.expect("table should be created");
RestLoadTableResponse {
metadata_location: response
.metadata_location
.expect("direct create should return metadata location"),
metadata: response.metadata,
config: response.config,
storage_credentials: response.storage_credentials,
}
}
async fn create_standard_recent_events_view<S>(
@@ -64,6 +64,7 @@ const SIGNATURE_PATH: &str = "envelope.sig";
const RESULT_PATH: &str = "result.json";
const MAX_OBJECT_RESPONSE_BYTES: usize = 16_384;
const CLEANUP_RESERVE_MAX: Duration = Duration::from_millis(250);
const CLEANUP_TIMEOUT: Duration = Duration::from_secs(2);
const OUTPUT_MODE: u32 = 0o600;
static OBJECT_COLLECTOR_ACTIVE: AtomicBool = AtomicBool::new(false);
@@ -374,7 +375,7 @@ impl S3ObjectProbe {
let mut builder = Client::builder()
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.timeout(timeout);
.timeout(timeout.max(CLEANUP_TIMEOUT));
if let Some(root_ca_pem) = root_ca_pem {
let certificate =
reqwest::Certificate::from_pem(root_ca_pem).map_err(|_| ObjectPerformanceError::InvalidRootCertificate)?;
@@ -403,7 +404,6 @@ impl S3ObjectProbe {
let started = Instant::now();
let reserve = CLEANUP_RESERVE_MAX.min(request.duration / 10);
let operation_deadline = started + request.duration.saturating_sub(reserve);
let cleanup_deadline = started + request.duration;
let bucket = format!("rustfs-connect-perf-{}", request.artifact_uid.replace('-', ""));
let object = "synthetic-object";
let bucket_url = self.object_url(&bucket, None)?;
@@ -427,6 +427,9 @@ impl S3ObjectProbe {
}
Err(error) => Err(error),
};
// Cleanup owns two sequential requests and must not inherit only the
// fractional tail of the caller's measurement window.
let cleanup_deadline = Instant::now() + CLEANUP_TIMEOUT;
let cleanup = self.cleanup(object_url, bucket_url, cleanup_deadline).await;
if cleanup.is_err() {
return Err(ObjectProbeError::CleanupFailed);
+27
View File
@@ -245,6 +245,26 @@ pub(crate) trait TableCatalogStore: Send + Sync {
async fn list_all_tables(&self, table_bucket: &str) -> TableCatalogStoreResult<Vec<TableEntry>>;
/// Checks whether an active table already owns any part of the candidate warehouse location.
///
/// This is a preflight for clients that write data before catalog registration. Registration
/// remains the authoritative atomic check for concurrent creators.
async fn ensure_table_warehouse_location_available(&self, candidate: &TableEntry) -> TableCatalogStoreResult<()> {
let candidate_prefix = table_warehouse_object_prefix(candidate)?;
for existing in self.list_all_tables(&candidate.table_bucket).await? {
if existing.state != TableCatalogEntryState::Active || existing.table_id == candidate.table_id {
continue;
}
let existing_prefix = table_warehouse_object_prefix(&existing)?;
if warehouse_object_prefixes_overlap(&existing_prefix, &candidate_prefix) {
return Err(TableCatalogStoreError::Conflict(format!(
"table warehouse location overlaps an active table: {candidate_prefix}"
)));
}
}
Ok(())
}
async fn list_tables_page(
&self,
table_bucket: &str,
@@ -1242,6 +1262,13 @@ where
}
}
async fn ensure_table_warehouse_location_available(&self, candidate: &TableEntry) -> TableCatalogStoreResult<()> {
match self {
Self::ObjectBacked(store) => store.ensure_table_warehouse_location_available(candidate).await,
Self::DurableStrong(store) => store.ensure_table_warehouse_location_available(candidate).await,
}
}
async fn list_tables_page(
&self,
table_bucket: &str,
+4
View File
@@ -5168,6 +5168,10 @@ where
Ok(entries)
}
async fn ensure_table_warehouse_location_available(&self, candidate: &TableEntry) -> TableCatalogStoreResult<()> {
self.ensure_table_warehouse_prefix_available(candidate).await
}
async fn list_tables_page(
&self,
table_bucket: &str,
+10
View File
@@ -2270,6 +2270,16 @@ where
.collect())
}
async fn ensure_table_warehouse_location_available(&self, candidate: &TableEntry) -> TableCatalogStoreResult<()> {
validate_table_entry_version_and_id(candidate)?;
let namespace = parse_namespace_for_store(&candidate.namespace)?;
let table = parse_table_for_store(&candidate.table)?;
self.hydrate_state().await?;
let state = self.state.lock().await;
let key = Self::table_key(&candidate.table_bucket, &namespace, &table);
Self::ensure_table_warehouse_prefix_available_locked(&state, candidate, &key)
}
async fn list_tables_page(
&self,
table_bucket: &str,
+56
View File
@@ -396,6 +396,61 @@ async fn oversized_response_is_rejected_without_buffering_it() {
server.await.expect("response server");
}
#[tokio::test]
async fn cleanup_uses_a_budget_independent_of_the_measurement_window() {
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
let _guard = TEST_LOCK.lock().await;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("response listener");
let address = listener.local_addr().expect("listener address");
let server = tokio::spawn(async move {
for (index, response) in [
b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".as_slice(),
b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".as_slice(),
b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\n\xa5\xa5\xa5\xa5".as_slice(),
b"HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n".as_slice(),
b"HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n".as_slice(),
]
.into_iter()
.enumerate()
{
let (mut socket, _) = listener.accept().await.expect("object connection");
let mut request = vec![0_u8; 16 * 1024];
let _ = socket.read(&mut request).await.expect("request headers");
if index >= 3 {
tokio::time::sleep(Duration::from_millis(600)).await;
}
socket.write_all(response).await.expect("response");
}
});
let probe = S3ObjectProbe::new(
&format!("http://{address}"),
None,
None,
Zeroizing::new("access".to_owned()),
Zeroizing::new("secret".to_owned()),
Zeroizing::new(String::new()),
Duration::from_secs(1),
)
.expect("object probe");
let mut request = request(ObjectOperation::GetObject);
request.traffic_bytes = 4;
let measurement = measure_object(&request, &probe, &CancellationToken::new())
.await
.expect("object measurement");
assert_eq!(
measurement.result.outcome(),
ObjectOutcome::Succeeded,
"target result: {:?}",
measurement.target
);
tokio::time::timeout(Duration::from_secs(3), server)
.await
.expect("cleanup requests should complete")
.expect("response server");
}
#[tokio::test]
async fn signed_result_is_saved_without_overwrite() {
let _guard = TEST_LOCK.lock().await;
@@ -463,6 +518,7 @@ fn real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put() {
}
async fn real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put_body() {
let _guard = TEST_LOCK.lock().await;
let port = match find_available_port() {
Ok(port) => port,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
+2 -2
View File
@@ -649,8 +649,8 @@ python3 scripts/table-catalog/engine_compatibility.py \
--cleanup
```
The generated SQL covers namespace creation, table creation, append, refresh,
count, and optional cleanup. Until Spark execution is enabled in CI through the
The generated SQL covers namespace creation, atomic CTAS through the Iceberg
REST staged-create flow, refresh, count, and optional cleanup. Until Spark execution is enabled in CI through the
explicit live-conformance gate, do not claim Spark support beyond a manually
verified run with the exact RustFS build, Spark version, Iceberg version, and
expected output recorded.
+2 -15
View File
@@ -287,17 +287,6 @@ def negative_sql(args: argparse.Namespace, *, kind: str, seed_table: str, write_
bootstrap = f"bootstrap_{kind}"
sql = profile_sql(args, catalog=bootstrap, table=seed_table)
sql += f"DETACH {engine_compatibility.quote_double_identifier(bootstrap)};\n"
if kind == "stage-create":
catalog = "stage_default"
sql += attach_sql(
args,
catalog=catalog,
rest_path="/iceberg",
signing_name="s3",
compatibility_options=False,
)
sql += f"CREATE TABLE {table_identifier(catalog, args.namespace, table_name(args.table, 'stage'))} (id BIGINT);\n"
return sql
if kind == "purge":
catalog = "purge_requested"
sql += attach_sql(
@@ -435,9 +424,8 @@ def run_smoke(args: argparse.Namespace, deps: pyiceberg_smoke.RuntimeDeps) -> Du
write_table = table_name(args.table, "write")
purge_table = table_name(args.table, "purge")
drop_table = table_name(args.table, "drop")
stage_table = table_name(args.table, "stage")
v3_table = table_name(args.table, "v3")
smoke_tables = [seed_table, write_table, purge_table, drop_table, stage_table, v3_table]
smoke_tables = [seed_table, write_table, purge_table, drop_table, v3_table]
catalog = pyiceberg_smoke.load_rest_catalog(iceberg_args, deps)
namespace_preexisting = bool(catalog.namespace_exists(args.namespace))
prepare_smoke_tables(catalog, args.namespace, smoke_tables, args.replace)
@@ -489,7 +477,6 @@ def run_smoke(args: argparse.Namespace, deps: pyiceberg_smoke.RuntimeDeps) -> Du
checks["s3tables_alias"] = "pass"
negatives = [
("stage-create", "stage-create is not supported"),
("purge", "purgeRequested=true is not supported"),
("format-v3", "unsupported Iceberg table format-version: 3"),
]
@@ -507,7 +494,7 @@ def run_smoke(args: argparse.Namespace, deps: pyiceberg_smoke.RuntimeDeps) -> Du
)
require_duckdb_error(execution, kind, expected_error)
checks[kind] = "failed-closed"
if catalog.table_exists((args.namespace, stage_table)) or catalog.table_exists((args.namespace, v3_table)):
if catalog.table_exists((args.namespace, v3_table)):
raise RuntimeError("a failed DuckDB create probe left a catalog table behind")
if not catalog.table_exists((args.namespace, purge_table)):
raise RuntimeError("purgeRequested=true removed a table despite the expected failure")
@@ -136,8 +136,8 @@ def engine_compatibility_matrix() -> list[dict[str, Any]]:
"entrypoint": "scripts/table-catalog/engine_compatibility.py --print-live-conformance",
"scenarios": [
scenario("create-namespace", "manual-live-harness", "CREATE NAMESPACE IF NOT EXISTS"),
scenario("create-table", "manual-live-harness", "CREATE TABLE USING iceberg"),
scenario("append", "manual-live-harness", "INSERT INTO"),
scenario("create-table", "manual-live-harness", "CREATE TABLE USING iceberg AS SELECT"),
scenario("staged-create", "manual-live-harness", "atomic CTAS through the REST staged-create flow"),
scenario("reload-table", "manual-live-harness", "REFRESH TABLE and SELECT COUNT"),
scenario("drop-table", "manual-live-harness", "DROP TABLE and optional DROP NAMESPACE"),
scenario("commit-conflict", "manual-validation-required", "requires a two-writer Spark or REST conflict harness"),
@@ -162,7 +162,7 @@ def engine_compatibility_matrix() -> list[dict[str, Any]]:
scenario("catalog-attach", "automated", "attach `/iceberg` with s3 signing and `/_iceberg` with s3tables signing"),
scenario("read-table", "automated", "read a PyIceberg-created table through the attached catalog"),
scenario("write-table", "automated", "exercise single-table DDL, DML, schema evolution, snapshots, and PyIceberg cross-read"),
scenario("unsupported-boundaries", "automated", "verify staged create, purge, and format v3 fail closed"),
scenario("unsupported-boundaries", "automated", "verify purge and format v3 fail closed"),
scenario("multi-table-mode", "automated", "verify DuckDB can avoid the multi-table commit endpoint without claiming cross-table atomicity"),
],
},
@@ -363,8 +363,8 @@ def spark_sql_smoke(
statements = [
f"CREATE NAMESPACE IF NOT EXISTS {namespace_identifier};",
f"DROP TABLE IF EXISTS {table_identifier};",
f"CREATE TABLE {table_identifier} (id BIGINT, payload STRING) USING iceberg;",
f"INSERT INTO {table_identifier} VALUES (1, 'alpha'), (2, 'beta');",
f"CREATE TABLE {table_identifier} USING iceberg AS "
"SELECT * FROM VALUES (1L, 'alpha'), (2L, 'beta') AS source(id, payload);",
f"REFRESH TABLE {table_identifier};",
f"SELECT COUNT(*) AS row_count FROM {table_identifier};",
]
@@ -665,7 +665,7 @@ def live_conformance_evidence(
OrderedDict(
[
("client", "Spark Iceberg REST catalog"),
("scenario", "create-namespace-create-table-append-refresh-count-cleanup"),
("scenario", "create-namespace-staged-ctas-refresh-count-cleanup"),
("expected_status", "pass"),
("expected_row_count", 2),
("claim_after_pass", "manual-live-verified"),
@@ -145,17 +145,6 @@ class DuckDBSmokeTest(unittest.TestCase):
def test_boundary_sql_records_required_compatibility_options(self) -> None:
args = self.args()
stage_sql = duckdb_smoke.negative_sql(
args,
kind="stage-create",
seed_table="events_seed",
write_table="events_write",
purge_table="events_purge",
)
stage_attach = stage_sql.split('DETACH "bootstrap_stage-create";', 1)[1]
self.assertNotIn("STAGE_CREATE_TABLES false", stage_attach)
self.assertIn("CREATE TABLE", stage_attach)
purge_sql = duckdb_smoke.negative_sql(
args,
kind="purge",
@@ -32,7 +32,7 @@ class EngineCompatibilityTest(unittest.TestCase):
self.assertEqual(spark["status"], "manual-live-harness")
self.assertContainsScenario(spark, "create-namespace", "manual-live-harness")
self.assertContainsScenario(spark, "create-table", "manual-live-harness")
self.assertContainsScenario(spark, "append", "manual-live-harness")
self.assertContainsScenario(spark, "staged-create", "manual-live-harness")
self.assertContainsScenario(spark, "reload-table", "manual-live-harness")
self.assertContainsScenario(spark, "drop-table", "manual-live-harness")
self.assertContainsScenario(spark, "commit-conflict", "manual-validation-required")
@@ -142,7 +142,7 @@ class EngineCompatibilityTest(unittest.TestCase):
self.assertEqual(config["spark.sql.catalog.rustfs.s3.secret-access-key"], "rustfsadmin")
self.assertEqual(config["spark.sql.catalog.rustfs.rest.signing-name"], "s3tables")
def test_spark_sql_smoke_covers_lifecycle_append_reload_and_cleanup(self) -> None:
def test_spark_sql_smoke_covers_staged_ctas_reload_and_cleanup(self) -> None:
sql = engine_compatibility.spark_sql_smoke(
catalog_name="rustfs",
namespace="sales",
@@ -152,8 +152,9 @@ class EngineCompatibilityTest(unittest.TestCase):
self.assertIn("CREATE NAMESPACE IF NOT EXISTS rustfs.`sales`", sql)
self.assertIn("DROP TABLE IF EXISTS rustfs.`sales`.`orders`", sql)
self.assertIn("CREATE TABLE rustfs.`sales`.`orders`", sql)
self.assertIn("INSERT INTO rustfs.`sales`.`orders`", sql)
self.assertIn("CREATE TABLE rustfs.`sales`.`orders` USING iceberg AS", sql)
self.assertIn("SELECT * FROM VALUES (1L, 'alpha'), (2L, 'beta')", sql)
self.assertNotIn("INSERT INTO rustfs.`sales`.`orders`", sql)
self.assertIn("REFRESH TABLE rustfs.`sales`.`orders`", sql)
self.assertIn("SELECT COUNT(*) AS row_count FROM rustfs.`sales`.`orders`", sql)
self.assertIn("DROP TABLE IF EXISTS rustfs.`sales`.`orders`", sql)
@@ -547,7 +548,7 @@ class EngineCompatibilityTest(unittest.TestCase):
record = engine_compatibility.live_conformance_evidence_record(
client_name="Spark Iceberg REST catalog",
client_version="3.5.4/iceberg-1.7.1",
scenario="create-namespace-create-table-append-refresh-count-cleanup",
scenario="create-namespace-staged-ctas-refresh-count-cleanup",
rustfs_build="rustfs-test",
git_sha="abc123",
catalog_backing="durable-strong",