fix(scanner): account versioned delete markers in usage (#3904)

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
2026-06-26 18:34:21 +08:00
committed by GitHub
parent 0321e4c6ca
commit a8327f8901
11 changed files with 503 additions and 82 deletions
+143 -3
View File
@@ -13,11 +13,40 @@
// limitations under the License.
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
use rustfs_data_usage::DataUsageInfo;
use serial_test::serial;
use tokio::time::{Duration, sleep};
use crate::common::{RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging};
async fn get_data_usage_info(env: &RustFSTestEnvironment) -> Result<DataUsageInfo, Box<dyn std::error::Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/datausageinfo", env.url);
let resp = awscurl_get(&url, &env.access_key, &env.secret_key).await?;
Ok(serde_json::from_str(&resp)?)
}
async fn wait_for_bucket_usage<F>(
env: &RustFSTestEnvironment,
bucket: &str,
mut predicate: F,
) -> Result<DataUsageInfo, Box<dyn std::error::Error + Send + Sync>>
where
F: FnMut(&DataUsageInfo) -> bool,
{
let mut last_usage = DataUsageInfo::default();
for _ in 0..45 {
let usage = get_data_usage_info(env).await?;
if usage.buckets_usage.contains_key(bucket) && predicate(&usage) {
return Ok(usage);
}
last_usage = usage;
sleep(Duration::from_secs(2)).await;
}
Err(format!("bucket usage did not converge for {bucket}; last usage: {last_usage:?}").into())
}
/// Regression test for data usage accuracy (issue #1012).
/// Launches rustfs, writes 1000 objects, then asserts admin data usage reports the full count.
#[tokio::test(flavor = "multi_thread")]
@@ -46,9 +75,7 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
}
// Query admin data usage API
let url = format!("{}/rustfs/admin/v3/datausageinfo", env.url);
let resp = awscurl_get(&url, &env.access_key, &env.secret_key).await?;
let usage: DataUsageInfo = serde_json::from_str(&resp)?;
let usage = get_data_usage_info(&env).await?;
// Assert total object count and per-bucket count are not truncated
let bucket_usage = usage
@@ -71,3 +98,116 @@ async fn data_usage_reports_all_objects() -> Result<(), Box<dyn std::error::Erro
env.stop_server();
Ok(())
}
/// Regression test for issue #3898.
/// Versioned buckets should expose versions and delete markers through admin data usage.
#[tokio::test(flavor = "multi_thread")]
#[serial]
#[ignore = "Starts a rustfs server and requires awscurl; enable when running full E2E"]
async fn data_usage_reports_versioned_objects_and_delete_markers() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let client = env.create_s3_client();
let bucket = "data-usage-versioned";
client.create_bucket().bucket(bucket).send().await?;
client
.put_bucket_versioning()
.bucket(bucket)
.versioning_configuration(
VersioningConfiguration::builder()
.status(BucketVersioningStatus::Enabled)
.build(),
)
.send()
.await?;
client
.put_object()
.bucket(bucket)
.key("alpha")
.body(ByteStream::from_static(b"v1-alpha"))
.send()
.await?;
client
.put_object()
.bucket(bucket)
.key("alpha")
.body(ByteStream::from_static(b"v2-alpha-larger"))
.send()
.await?;
client
.put_object()
.bucket(bucket)
.key("beta")
.body(ByteStream::from_static(b"beta"))
.send()
.await?;
client.delete_object().bucket(bucket).key("alpha").send().await?;
let listed_versions = client.list_object_versions().bucket(bucket).send().await?;
assert_eq!(listed_versions.versions().len(), 3, "S3 version listing should report stored versions");
assert_eq!(
listed_versions.delete_markers().len(),
1,
"S3 version listing should report the delete marker"
);
let usage = wait_for_bucket_usage(&env, bucket, |usage| {
usage
.buckets_usage
.get(bucket)
.map(|bucket_usage| {
bucket_usage.objects_count == 2 && bucket_usage.versions_count == 3 && bucket_usage.delete_markers_count == 1
})
.unwrap_or(false)
})
.await?;
let bucket_usage = usage
.buckets_usage
.get(bucket)
.expect("bucket usage should exist after convergence");
assert_eq!(
bucket_usage.objects_count, 2,
"current object count should exclude the deleted current alpha"
);
assert_eq!(bucket_usage.versions_count, 3, "version count should include all stored object versions");
assert_eq!(
bucket_usage.delete_markers_count, 1,
"delete marker count should include the alpha marker"
);
assert_eq!(usage.objects_total_count, 2, "total current object count should match bucket usage");
assert_eq!(usage.versions_total_count, 3, "total version count should match bucket usage");
assert_eq!(usage.delete_markers_total_count, 1, "total delete marker count should match bucket usage");
env.stop_server();
env.start_rustfs_server(vec![]).await?;
let restarted_usage = wait_for_bucket_usage(&env, bucket, |usage| {
usage
.buckets_usage
.get(bucket)
.map(|bucket_usage| {
bucket_usage.objects_count == 2 && bucket_usage.versions_count == 3 && bucket_usage.delete_markers_count == 1
})
.unwrap_or(false)
})
.await?;
let restarted_bucket_usage = restarted_usage
.buckets_usage
.get(bucket)
.expect("bucket usage should exist after restart");
assert_eq!(restarted_bucket_usage.objects_count, 2, "object count should persist after restart");
assert_eq!(restarted_bucket_usage.versions_count, 3, "version count should persist after restart");
assert_eq!(
restarted_bucket_usage.delete_markers_count, 1,
"delete marker count should persist after restart"
);
env.stop_server();
Ok(())
}
+2 -1
View File
@@ -69,7 +69,8 @@ pub mod config {
pub mod data_usage {
pub use crate::data_usage::{
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, load_data_usage_from_backend,
record_bucket_object_delete_memory, record_bucket_object_write_memory, remove_bucket_usage_from_backend,
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
record_bucket_object_write_memory, refresh_versioned_bucket_usage_from_object_layer, remove_bucket_usage_from_backend,
replace_bucket_usage_memory_from_info,
};
}
+178 -48
View File
@@ -16,7 +16,7 @@ pub mod local_snapshot;
use crate::storage_api_contracts::{list::ListOperations as _, object::ObjectIO as _};
use crate::{
bucket::metadata_sys::get_replication_config,
bucket::{metadata_sys::get_replication_config, versioning::VersioningApi as _, versioning_sys::BucketVersioningSys},
config::com::read_config,
disk::DiskAPI,
error::{Error, classify_system_path_failure_reason},
@@ -398,8 +398,9 @@ pub async fn aggregate_local_snapshots(store: Arc<ECStore>) -> Result<(Vec<DiskU
/// Calculate accurate bucket usage statistics by enumerating objects through the object layer.
pub async fn compute_bucket_usage(store: Arc<ECStore>, bucket_name: &str) -> Result<BucketUsageInfo, Error> {
let mut continuation: Option<String> = None;
let mut objects_count: u64 = 0;
let mut marker: Option<String> = None;
let mut version_marker: Option<String> = None;
let mut object_names: HashSet<String> = HashSet::new();
let mut versions_count: u64 = 0;
let mut total_size: u64 = 0;
let mut delete_markers: u64 = 0;
@@ -407,15 +408,13 @@ pub async fn compute_bucket_usage(store: Arc<ECStore>, bucket_name: &str) -> Res
loop {
let result = store
.clone()
.list_objects_v2(
.list_object_versions(
bucket_name,
"", // prefix
continuation.clone(),
None, // delimiter
1000, // max_keys
false, // fetch_owner
None, // start_after
false, // incl_deleted
marker.clone(),
version_marker.clone(),
None, // delimiter
1000, // max_keys
)
.await?;
@@ -430,34 +429,27 @@ pub async fn compute_bucket_usage(store: Arc<ECStore>, bucket_name: &str) -> Res
}
let object_size = object.size.max(0) as u64;
objects_count = objects_count.saturating_add(1);
object_names.insert(object.name.clone());
total_size = total_size.saturating_add(object_size);
let detected_versions = if object.num_versions > 0 {
object.num_versions as u64
} else {
1
};
versions_count = versions_count.saturating_add(detected_versions);
versions_count = versions_count.saturating_add(1);
}
if !result.is_truncated {
break;
}
continuation = result.next_continuation_token.clone();
if continuation.is_none() {
marker = result.next_marker.clone();
version_marker = result.next_version_idmarker.clone();
if marker.is_none() {
info!(
"Bucket {} listing marked truncated but no continuation token returned; stopping early",
"Bucket {} version listing marked truncated but no marker returned; stopping early",
bucket_name
);
break;
}
}
if versions_count == 0 {
versions_count = objects_count;
}
let objects_count = object_names.len() as u64;
let usage = BucketUsageInfo {
size: total_size,
@@ -470,6 +462,42 @@ pub async fn compute_bucket_usage(store: Arc<ECStore>, bucket_name: &str) -> Res
Ok(usage)
}
pub async fn refresh_versioned_bucket_usage_from_object_layer(store: Arc<ECStore>, data_usage_info: &mut DataUsageInfo) {
let buckets = data_usage_info.buckets_usage.keys().cloned().collect::<Vec<String>>();
let mut changed = false;
for bucket in buckets {
let Ok(versioning) = BucketVersioningSys::get(&bucket).await else {
continue;
};
if !versioning.enabled() && !versioning.suspended() {
continue;
}
let usage = match compute_bucket_usage(store.clone(), &bucket).await {
Ok(usage) => usage,
Err(err) => {
debug!(
bucket = %bucket,
error = %err,
"failed to refresh versioned bucket usage from object layer"
);
continue;
}
};
data_usage_info.bucket_sizes.insert(bucket.clone(), usage.size);
data_usage_info.buckets_usage.insert(bucket, usage);
changed = true;
}
if changed {
set_buckets_count_from_usage(data_usage_info);
data_usage_info.calculate_totals();
}
}
async fn ensure_bucket_usage_cached(bucket: &str) {
let cache = memory_cache().read().await;
if cache.contains_key(bucket) {
@@ -514,6 +542,20 @@ fn bucket_usage_counts_match(left: &BucketUsageInfo, right: &BucketUsageInfo) ->
/// Fast in-memory update for immediate quota and admin usage consistency.
pub async fn record_bucket_object_write_memory(bucket: &str, previous_current_size: Option<u64>, new_size: u64) {
record_bucket_object_write_memory_inner(bucket, previous_current_size, new_size, false).await;
}
/// Fast in-memory update for versioned object writes.
pub async fn record_bucket_object_version_write_memory(bucket: &str, previous_current_size: Option<u64>, new_size: u64) {
record_bucket_object_write_memory_inner(bucket, previous_current_size, new_size, true).await;
}
async fn record_bucket_object_write_memory_inner(
bucket: &str,
previous_current_size: Option<u64>,
new_size: u64,
creates_new_version: bool,
) {
ensure_bucket_usage_cached(bucket).await;
let mut cache = memory_cache().write().await;
@@ -521,14 +563,22 @@ pub async fn record_bucket_object_write_memory(bucket: &str, previous_current_si
.entry(bucket.to_string())
.or_insert_with(|| cached_bucket_usage_now(BucketUsageInfo::default()));
match previous_current_size {
Some(previous_size) => {
entry.usage.size = entry.usage.size.saturating_sub(previous_size).saturating_add(new_size);
}
None => {
entry.usage.size = entry.usage.size.saturating_add(new_size);
if creates_new_version {
entry.usage.size = entry.usage.size.saturating_add(new_size);
if previous_current_size.is_none() {
entry.usage.objects_count = entry.usage.objects_count.saturating_add(1);
entry.usage.versions_count = entry.usage.versions_count.saturating_add(1);
}
entry.usage.versions_count = entry.usage.versions_count.saturating_add(1);
} else {
match previous_current_size {
Some(previous_size) => {
entry.usage.size = entry.usage.size.saturating_sub(previous_size).saturating_add(new_size);
}
None => {
entry.usage.size = entry.usage.size.saturating_add(new_size);
entry.usage.objects_count = entry.usage.objects_count.saturating_add(1);
entry.usage.versions_count = entry.usage.versions_count.saturating_add(1);
}
}
}
@@ -566,6 +616,24 @@ pub async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64,
entry.stale_snapshot_pending = false;
}
/// Fast in-memory update for successful delete marker creation.
pub async fn record_bucket_delete_marker_memory(bucket: &str) {
ensure_bucket_usage_cached(bucket).await;
let mut cache = memory_cache().write().await;
let entry = cache
.entry(bucket.to_string())
.or_insert_with(|| cached_bucket_usage_now(BucketUsageInfo::default()));
entry.usage.delete_markers_count = entry.usage.delete_markers_count.saturating_add(1);
let now = SystemTime::now();
entry.refreshed_at = now;
entry.usage_updated_at = now;
entry.dirty = true;
entry.stale_snapshot_pending = false;
}
/// Fast in-memory decrement for immediate quota consistency
pub async fn decrement_bucket_usage_memory(bucket: &str, size_decrement: u64) {
record_bucket_object_delete_memory(bucket, size_decrement, size_decrement > 0).await;
@@ -605,20 +673,12 @@ async fn update_usage_cache_if_needed() {
*updating = true;
drop(updating);
let cache_clone = (*memory_cache()).clone();
let updating_clone = (*cache_updating()).clone();
tokio::spawn(async move {
if let Some(store) = runtime_sources::object_store_handle()
&& let Ok(data_usage_info) = load_data_usage_from_backend(store.clone()).await
{
let mut cache = cache_clone.write().await;
let usage_updated_at = data_usage_info_updated_at(&data_usage_info);
for (bucket_name, bucket_usage) in data_usage_info.buckets_usage.iter() {
cache.insert(
bucket_name.clone(),
cached_bucket_usage_from_backend(bucket_usage.clone(), usage_updated_at),
);
}
replace_bucket_usage_memory_from_info(&data_usage_info).await;
}
let mut updating = updating_clone.write().await;
*updating = false;
@@ -642,14 +702,7 @@ async fn update_usage_cache_if_needed() {
if let Some(store) = runtime_sources::object_store_handle()
&& let Ok(data_usage_info) = load_data_usage_from_backend(store.clone()).await
{
let mut cache = memory_cache().write().await;
let usage_updated_at = data_usage_info_updated_at(&data_usage_info);
for (bucket_name, bucket_usage) in data_usage_info.buckets_usage.iter() {
cache.insert(
bucket_name.clone(),
cached_bucket_usage_from_backend(bucket_usage.clone(), usage_updated_at),
);
}
replace_bucket_usage_memory_from_info(&data_usage_info).await;
}
let mut updating = cache_updating().write().await;
@@ -1161,6 +1214,83 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn memory_overlay_counts_versioned_overwrite_as_new_version() {
clear_usage_memory_cache_for_test().await;
let persisted = data_usage_info_for_test("bucket-a", 1, 10, SystemTime::now() - Duration::from_secs(10));
replace_bucket_usage_memory_from_info(&persisted).await;
record_bucket_object_version_write_memory("bucket-a", Some(10), 20).await;
let mut response = persisted.clone();
apply_bucket_usage_memory_overlay(&mut response).await;
assert_eq!(response.objects_total_count, 1);
assert_eq!(response.versions_total_count, 2);
assert_eq!(response.objects_total_size, 30);
assert_eq!(
response
.buckets_usage
.get("bucket-a")
.map(|usage| (usage.objects_count, usage.versions_count, usage.size)),
Some((1, 2, 30))
);
}
#[tokio::test]
#[serial]
async fn memory_overlay_records_delete_marker_without_removing_versions() {
clear_usage_memory_cache_for_test().await;
let persisted = data_usage_info_for_test("bucket-a", 2, 30, SystemTime::now() - Duration::from_secs(10));
replace_bucket_usage_memory_from_info(&persisted).await;
record_bucket_delete_marker_memory("bucket-a").await;
let mut response = persisted.clone();
apply_bucket_usage_memory_overlay(&mut response).await;
assert_eq!(response.objects_total_count, 2);
assert_eq!(response.versions_total_count, 2);
assert_eq!(response.delete_markers_total_count, 1);
assert_eq!(response.objects_total_size, 30);
assert_eq!(
response
.buckets_usage
.get("bucket-a")
.map(|usage| { (usage.objects_count, usage.versions_count, usage.delete_markers_count, usage.size,) }),
Some((2, 2, 1, 30))
);
}
#[tokio::test]
#[serial]
async fn scanner_sync_preserves_dirty_delete_marker_with_later_snapshot() {
clear_usage_memory_cache_for_test().await;
let now = SystemTime::now();
let old_persisted = data_usage_info_for_test("bucket-a", 2, 30, now - Duration::from_secs(10));
replace_bucket_usage_memory_from_info(&old_persisted).await;
record_bucket_delete_marker_memory("bucket-a").await;
let scanner_without_marker = data_usage_info_for_test("bucket-a", 2, 30, now + Duration::from_secs(10));
replace_bucket_usage_memory_from_info(&scanner_without_marker).await;
let mut response = scanner_without_marker.clone();
apply_bucket_usage_memory_overlay(&mut response).await;
assert_eq!(response.objects_total_count, 2);
assert_eq!(response.versions_total_count, 2);
assert_eq!(response.delete_markers_total_count, 1);
assert_eq!(
response
.buckets_usage
.get("bucket-a")
.map(|usage| { (usage.objects_count, usage.versions_count, usage.delete_markers_count, usage.size,) }),
Some((2, 2, 1, 30))
);
}
#[tokio::test]
#[serial]
async fn memory_overlay_does_not_replace_newer_persisted_usage() {
+18 -1
View File
@@ -172,6 +172,7 @@ impl SizeSummary {
pub fn actions_accounting(&mut self, oi: &ObjectInfo, size: i64, actual_size: i64) {
if oi.delete_marker {
self.delete_markers += 1;
return;
}
if oi.version_id.is_some_and(|v| !v.is_nil()) && size == actual_size {
@@ -180,7 +181,7 @@ impl SizeSummary {
self.total_size += if size > 0 { size as usize } else { 0 };
if oi.delete_marker || oi.transitioned_object.free_version {
if oi.transitioned_object.free_version {
return;
}
@@ -1065,6 +1066,22 @@ mod tests {
assert_eq!(summary1.versions, 15);
}
#[test]
fn size_summary_counts_delete_markers_separately_from_versions() {
let mut summary = SizeSummary::new();
let marker = ObjectInfo {
delete_marker: true,
version_id: Some(uuid::Uuid::new_v4()),
..Default::default()
};
summary.actions_accounting(&marker, 0, 0);
assert_eq!(summary.delete_markers, 1);
assert_eq!(summary.versions, 0);
assert_eq!(summary.total_size, 0);
}
#[test]
fn test_data_usage_entry_merge_sums_failed_objects() {
let mut left = DataUsageEntry {
+3 -3
View File
@@ -697,10 +697,10 @@ impl ScannerItem {
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
bucket = %self.bucket,
state = "versioning_lookup_failed",
"Scanner lifecycle action skipped"
state = "versioning_lookup_failed_defaulting",
"Scanner lifecycle action falling back to default bucket versioning"
);
return;
Default::default()
}
};
+72
View File
@@ -1543,8 +1543,10 @@ mod tests {
use super::*;
use crate::scanner_folder::ScannerItem;
use crate::{DiskOption, Endpoint, new_disk, path2_bucket_object_with_base_path};
use rustfs_filemeta::FileInfo;
use serial_test::serial;
use temp_env::with_var;
use time::OffsetDateTime;
use uuid::Uuid;
fn bucket_info(name: &str) -> BucketInfo {
@@ -1770,6 +1772,76 @@ mod tests {
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
}
#[tokio::test]
async fn get_size_counts_delete_markers_separately_from_versions() {
let temp_dir = std::env::temp_dir().join(format!("rustfs-scanner-versioned-usage-{}", Uuid::new_v4()));
let bucket = "bucket";
let object = "object";
let object_dir = temp_dir.join(bucket).join(object);
let metadata_path = object_dir.join(STORAGE_FORMAT_FILE);
tokio::fs::create_dir_all(&object_dir)
.await
.expect("failed to create object directory");
let mut meta = FileMeta::new();
for (size, timestamp) in [(10, 10), (20, 20)] {
let mut fi = FileInfo::new(object, 1, 1);
fi.version_id = Some(Uuid::new_v4());
fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(timestamp).expect("timestamp should be valid"));
fi.size = size;
meta.add_version(fi).expect("object version should be added");
}
let mut delete_marker = FileInfo::new(object, 1, 1);
delete_marker.version_id = Some(Uuid::new_v4());
delete_marker.mod_time = Some(OffsetDateTime::from_unix_timestamp(30).expect("timestamp should be valid"));
delete_marker.deleted = true;
meta.add_version(delete_marker).expect("delete marker should be added");
tokio::fs::write(&metadata_path, meta.marshal_msg().expect("metadata should marshal"))
.await
.expect("failed to write metadata");
let endpoint = Endpoint::try_from(temp_dir.to_string_lossy().as_ref()).expect("failed to create endpoint");
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("failed to open local disk");
let relative_path = metadata_path.to_string_lossy().to_string();
let (_, scanner_path) = path2_bucket_object_with_base_path(temp_dir.to_string_lossy().as_ref(), relative_path.as_str());
let file_type = tokio::fs::metadata(&metadata_path)
.await
.expect("failed to stat metadata")
.file_type();
let item = ScannerItem {
path: scanner_path,
bucket: bucket.to_string(),
prefix: object.to_string(),
object_name: STORAGE_FORMAT_FILE.to_string(),
file_type,
lifecycle: None,
replication: None,
heal_enabled: false,
heal_bitrot: false,
debug: false,
};
let summary = disk.get_size(item).await.expect("scanner should read versioned metadata");
assert_eq!(summary.versions, 2);
assert_eq!(summary.delete_markers, 1);
assert_eq!(summary.total_size, 30);
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
}
#[test]
fn cache_root_entry_info_flattens_bucket_children() {
let mut cache = DataUsageCache {
+6 -1
View File
@@ -19,7 +19,10 @@ use super::storage_api::admin_usecase::admin::get_server_info;
use super::storage_api::admin_usecase::capacity::{
PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free,
};
use super::storage_api::admin_usecase::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend};
use super::storage_api::admin_usecase::data_usage::{
apply_bucket_usage_memory_overlay, load_data_usage_from_backend, refresh_versioned_bucket_usage_from_object_layer,
replace_bucket_usage_memory_from_info,
};
use super::storage_api::admin_usecase::{ECStore, EndpointServerPools};
use crate::app::runtime_sources::{
AppContext, current_app_context, resolve_endpoints_handle, resolve_object_store_handle_for_context,
@@ -242,6 +245,8 @@ impl DefaultAdminUsecase {
error!("load_data_usage_from_backend failed {:?}", e);
Self::app_error(S3ErrorCode::InternalError, "load_data_usage_from_backend failed")
})?;
refresh_versioned_bucket_usage_from_object_layer(store.clone(), &mut info).await;
replace_bucket_usage_memory_from_info(&info).await;
apply_bucket_usage_memory_overlay(&mut info).await;
let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
+10 -2
View File
@@ -27,7 +27,9 @@ use super::storage_api::multipart_usecase::bucket::{
versioning_sys::BucketVersioningSys,
};
use super::storage_api::multipart_usecase::compression::is_disk_compressible;
use super::storage_api::multipart_usecase::data_usage::record_bucket_object_write_memory;
use super::storage_api::multipart_usecase::data_usage::{
record_bucket_object_version_write_memory, record_bucket_object_write_memory,
};
use super::storage_api::multipart_usecase::error::{StorageError, is_err_object_not_found, is_err_version_not_found};
use super::storage_api::multipart_usecase::helper::OperationHelper;
#[cfg(test)]
@@ -489,7 +491,13 @@ impl DefaultMultipartUsecase {
));
}
// Update quota tracking after successful multipart upload
record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
let mpu_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
if mpu_versioned {
record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64)
.await;
} else {
record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
}
}
Err(e) => {
warn!("Quota check failed for bucket {}: {}, allowing operation", bucket, e);
+38 -21
View File
@@ -50,7 +50,10 @@ use super::storage_api::object_usecase::concurrency::{
self, ConcurrencyManager, GetObjectGuard, PutObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager,
get_put_concurrency_aware_buffer_size,
};
use super::storage_api::object_usecase::data_usage::{record_bucket_object_delete_memory, record_bucket_object_write_memory};
use super::storage_api::object_usecase::data_usage::{
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
record_bucket_object_write_memory,
};
use super::storage_api::object_usecase::deadlock_detector;
use super::storage_api::object_usecase::ecfs::FS;
use super::storage_api::object_usecase::error::{
@@ -2778,8 +2781,13 @@ impl DefaultObjectUsecase {
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
let put_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
// Fast in-memory update for immediate quota and admin usage consistency
record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
if put_versioned {
record_bucket_object_version_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
} else {
record_bucket_object_write_memory(&bucket, previous_current_size, obj_info.size.max(0) as u64).await;
}
let raw_version = obj_info.version_id.map(|v| v.to_string());
@@ -2788,11 +2796,7 @@ impl DefaultObjectUsecase {
helper = helper.version_id(version_id.clone());
}
let put_version = if BucketVersioningSys::prefix_enabled(&bucket, &key).await {
raw_version
} else {
None
};
let put_version = if put_versioned { raw_version } else { None };
let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag));
@@ -3769,17 +3773,18 @@ impl DefaultObjectUsecase {
maybe_enqueue_transition_immediate(&oi, LcEventSrc::S3CopyObject).await;
let dest_versioned = BucketVersioningSys::prefix_enabled(&bucket, &key).await;
// Update quota tracking after successful copy
if has_bucket_metadata {
record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await;
if dest_versioned {
record_bucket_object_version_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await;
} else {
record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64).await;
}
}
let raw_dest_version = oi.version_id.map(|v| v.to_string());
let dest_version = if BucketVersioningSys::prefix_enabled(&bucket, &key).await {
raw_dest_version
} else {
None
};
let dest_version = if dest_versioned { raw_dest_version } else { None };
// warn!("copy_object oi {:?}", &oi);
let object_info = oi.clone();
@@ -3994,6 +3999,7 @@ impl DefaultObjectUsecase {
&bucket,
object_to_delete.clone(),
ObjectOptions {
versioned: version_cfg.enabled(),
version_suspended: version_cfg.suspended(),
..Default::default()
},
@@ -4047,13 +4053,20 @@ impl DefaultObjectUsecase {
"failed to persist transitioned object cleanup journal"
);
}
let size = object_sizes[i].max(0) as u64;
record_bucket_object_delete_memory(
&bucket,
size,
existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
)
.await;
let creates_delete_marker = object_to_delete[i].version_id.is_none()
&& version_cfg.prefix_enabled(object_to_delete[i].object_name.as_str())
&& !version_cfg.suspended();
if creates_delete_marker {
record_bucket_delete_marker_memory(&bucket).await;
} else {
let size = object_sizes[i].max(0) as u64;
record_bucket_object_delete_memory(
&bucket,
size,
existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
)
.await;
}
continue;
}
@@ -4285,7 +4298,11 @@ impl DefaultObjectUsecase {
}
// Fast in-memory update for immediate quota and admin usage consistency
record_bucket_object_delete_memory(&bucket, obj_info.size.max(0) as u64, opts.version_id.is_none()).await;
if delete_creates_delete_marker(&opts) {
record_bucket_delete_marker_memory(&bucket).await;
} else {
record_bucket_object_delete_memory(&bucket, obj_info.size.max(0) as u64, opts.version_id.is_none()).await;
}
if obj_info.name.is_empty() {
if replicate_force_delete {
+29
View File
@@ -55,11 +55,35 @@ pub(crate) mod data_usage {
crate::storage::ecstore_data_usage::load_data_usage_from_backend(store).await
}
pub(crate) async fn refresh_versioned_bucket_usage_from_object_layer(
store: Arc<super::ECStore>,
data_usage_info: &mut rustfs_data_usage::DataUsageInfo,
) {
crate::storage::ecstore_data_usage::refresh_versioned_bucket_usage_from_object_layer(store, data_usage_info).await;
}
pub(crate) async fn replace_bucket_usage_memory_from_info(data_usage_info: &rustfs_data_usage::DataUsageInfo) {
crate::storage::ecstore_data_usage::replace_bucket_usage_memory_from_info(data_usage_info).await;
}
pub(crate) async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, removed_current_object: bool) {
crate::storage::ecstore_data_usage::record_bucket_object_delete_memory(bucket, deleted_size, removed_current_object)
.await;
}
pub(crate) async fn record_bucket_delete_marker_memory(bucket: &str) {
crate::storage::ecstore_data_usage::record_bucket_delete_marker_memory(bucket).await;
}
pub(crate) async fn record_bucket_object_version_write_memory(
bucket: &str,
previous_current_size: Option<u64>,
new_size: u64,
) {
crate::storage::ecstore_data_usage::record_bucket_object_version_write_memory(bucket, previous_current_size, new_size)
.await;
}
pub(crate) async fn record_bucket_object_write_memory(bucket: &str, previous_current_size: Option<u64>, new_size: u64) {
crate::storage::ecstore_data_usage::record_bucket_object_write_memory(bucket, previous_current_size, new_size).await;
}
@@ -214,11 +238,16 @@ pub(crate) mod bucket {
}
pub(crate) trait VersioningConfigExt {
fn enabled(&self) -> bool;
fn prefix_enabled(&self, prefix: &str) -> bool;
fn suspended(&self) -> bool;
}
impl VersioningConfigExt for s3s::dto::VersioningConfiguration {
fn enabled(&self) -> bool {
<s3s::dto::VersioningConfiguration as crate::storage::ecstore_bucket::versioning::VersioningApi>::enabled(self)
}
fn prefix_enabled(&self, prefix: &str) -> bool {
<s3s::dto::VersioningConfiguration as crate::storage::ecstore_bucket::versioning::VersioningApi>::prefix_enabled(
self, prefix,
+4 -2
View File
@@ -81,8 +81,10 @@ pub(crate) mod ecstore_config {
pub(crate) mod ecstore_data_usage {
pub(crate) use rustfs_ecstore::api::data_usage::{
apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory,
record_bucket_object_write_memory, remove_bucket_usage_from_backend,
apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_delete_marker_memory,
record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
refresh_versioned_bucket_usage_from_object_layer, remove_bucket_usage_from_backend,
replace_bucket_usage_memory_from_info,
};
}