mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-28 07:57:01 +00:00
refactor: clean external operation consumers (#3565)
This commit is contained in:
@@ -33,8 +33,9 @@ use rustfs_ecstore::{
|
|||||||
config::{com::save_config, storageclass},
|
config::{com::save_config, storageclass},
|
||||||
disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
|
disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
|
||||||
error::{Error, Result as StorageResult, StorageError},
|
error::{Error, Result as StorageResult, StorageError},
|
||||||
store_api::{ObjectIO, ObjectInfo, ObjectOptions},
|
store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
|
||||||
};
|
};
|
||||||
|
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO};
|
||||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
||||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
@@ -60,6 +61,32 @@ const EVENT_SCANNER_CACHE_LOAD_STATE: &str = "scanner_cache_load_state";
|
|||||||
const EVENT_SCANNER_CACHE_SAVE_STATE: &str = "scanner_cache_save_state";
|
const EVENT_SCANNER_CACHE_SAVE_STATE: &str = "scanner_cache_save_state";
|
||||||
static CACHE_SAVE_METRICS_ONCE: Once = Once::new();
|
static CACHE_SAVE_METRICS_ONCE: Once = Once::new();
|
||||||
|
|
||||||
|
pub trait ScannerObjectIO:
|
||||||
|
ObjectIO<
|
||||||
|
Error = Error,
|
||||||
|
RangeSpec = HTTPRangeSpec,
|
||||||
|
HeaderMap = HeaderMap,
|
||||||
|
ObjectOptions = ObjectOptions,
|
||||||
|
ObjectInfo = ObjectInfo,
|
||||||
|
GetObjectReader = GetObjectReader,
|
||||||
|
PutObjectReader = PutObjReader,
|
||||||
|
>
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> ScannerObjectIO for T where
|
||||||
|
T: ObjectIO<
|
||||||
|
Error = Error,
|
||||||
|
RangeSpec = HTTPRangeSpec,
|
||||||
|
HeaderMap = HeaderMap,
|
||||||
|
ObjectOptions = ObjectOptions,
|
||||||
|
ObjectInfo = ObjectInfo,
|
||||||
|
GetObjectReader = GetObjectReader,
|
||||||
|
PutObjectReader = PutObjReader,
|
||||||
|
>
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
pub const DATA_USAGE_SCAN_CHECKPOINT_VERSION: u16 = 1;
|
pub const DATA_USAGE_SCAN_CHECKPOINT_VERSION: u16 = 1;
|
||||||
|
|
||||||
// Data usage paths (computed at runtime)
|
// Data usage paths (computed at runtime)
|
||||||
@@ -616,7 +643,7 @@ impl DataUsageCache {
|
|||||||
/// Only backend errors are returned as errors.
|
/// Only backend errors are returned as errors.
|
||||||
/// The loader is optimistic and has no locking, but tries 5 times before giving up.
|
/// The loader is optimistic and has no locking, but tries 5 times before giving up.
|
||||||
/// If the object is not found, a nil error with empty data usage cache is returned.
|
/// If the object is not found, a nil error with empty data usage cache is returned.
|
||||||
pub async fn load<S: ObjectIO>(&mut self, store: Arc<S>, name: &str) -> StorageResult<()> {
|
pub async fn load<S: ScannerObjectIO>(&mut self, store: Arc<S>, name: &str) -> StorageResult<()> {
|
||||||
// By default, empty data usage cache
|
// By default, empty data usage cache
|
||||||
*self = DataUsageCache::default();
|
*self = DataUsageCache::default();
|
||||||
|
|
||||||
@@ -671,7 +698,7 @@ impl DataUsageCache {
|
|||||||
}
|
}
|
||||||
// Inner load function that attempts to load from a specific path
|
// Inner load function that attempts to load from a specific path
|
||||||
// Returns (should_retry, cache_option, error_option)
|
// Returns (should_retry, cache_option, error_option)
|
||||||
async fn try_load_inner<S: ObjectIO>(
|
async fn try_load_inner<S: ScannerObjectIO>(
|
||||||
store: Arc<S>,
|
store: Arc<S>,
|
||||||
load_name: &str,
|
load_name: &str,
|
||||||
timeout_duration: Duration,
|
timeout_duration: Duration,
|
||||||
@@ -866,7 +893,7 @@ impl DataUsageCache {
|
|||||||
Err(last_err.unwrap_or_else(|| StorageError::other("Failed to save data usage cache".to_string())))
|
Err(last_err.unwrap_or_else(|| StorageError::other("Failed to save data usage cache".to_string())))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn save_path_with_retry<S: ObjectIO>(
|
async fn save_path_with_retry<S: ScannerObjectIO>(
|
||||||
store: Arc<S>,
|
store: Arc<S>,
|
||||||
path: &str,
|
path: &str,
|
||||||
buf: &[u8],
|
buf: &[u8],
|
||||||
@@ -889,7 +916,7 @@ impl DataUsageCache {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn save<S: ObjectIO>(&self, store: Arc<S>, name: &str) -> StorageResult<()> {
|
pub async fn save<S: ScannerObjectIO>(&self, store: Arc<S>, name: &str) -> StorageResult<()> {
|
||||||
let mut buf = Vec::new();
|
let mut buf = Vec::new();
|
||||||
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
||||||
let timeout_duration = Self::cache_save_timeout();
|
let timeout_duration = Self::cache_save_timeout();
|
||||||
|
|||||||
@@ -14,7 +14,9 @@
|
|||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use crate::data_usage_define::{BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH};
|
use crate::data_usage_define::{
|
||||||
|
BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, ScannerObjectIO,
|
||||||
|
};
|
||||||
use crate::runtime_config::{
|
use crate::runtime_config::{
|
||||||
current_scanner_runtime_config, lookup_scanner_runtime_config, refresh_scanner_runtime_config_from_global,
|
current_scanner_runtime_config, lookup_scanner_runtime_config, refresh_scanner_runtime_config_from_global,
|
||||||
scanner_bitrot_cycle, scanner_cycle_interval, scanner_start_delay, set_scanner_default_cycle_secs,
|
scanner_bitrot_cycle, scanner_cycle_interval, scanner_start_delay, set_scanner_default_cycle_secs,
|
||||||
@@ -45,7 +47,6 @@ use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
|
|||||||
use rustfs_ecstore::error::Error as EcstoreError;
|
use rustfs_ecstore::error::Error as EcstoreError;
|
||||||
use rustfs_ecstore::global::is_erasure_sd;
|
use rustfs_ecstore::global::is_erasure_sd;
|
||||||
use rustfs_ecstore::store::ECStore;
|
use rustfs_ecstore::store::ECStore;
|
||||||
use rustfs_ecstore::store_api::ObjectIO;
|
|
||||||
use rustfs_storage_api::{BucketOperations, BucketOptions, NamespaceLocking as _};
|
use rustfs_storage_api::{BucketOperations, BucketOptions, NamespaceLocking as _};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
@@ -976,7 +977,7 @@ impl Drop for ScannerScanModeGuard {
|
|||||||
#[instrument(skip(ctx, storeapi))]
|
#[instrument(skip(ctx, storeapi))]
|
||||||
pub async fn store_data_usage_in_backend(
|
pub async fn store_data_usage_in_backend(
|
||||||
ctx: CancellationToken,
|
ctx: CancellationToken,
|
||||||
storeapi: Arc<impl ObjectIO>,
|
storeapi: Arc<impl ScannerObjectIO>,
|
||||||
mut receiver: mpsc::Receiver<DataUsageInfo>,
|
mut receiver: mpsc::Receiver<DataUsageInfo>,
|
||||||
) {
|
) {
|
||||||
let mut attempts = 1u32;
|
let mut attempts = 1u32;
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ use crate::scanner_folder::{ScannerItem, scan_data_folder};
|
|||||||
use crate::sleeper::SCANNER_SLEEPER;
|
use crate::sleeper::SCANNER_SLEEPER;
|
||||||
use crate::{
|
use crate::{
|
||||||
DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageEntryInfo,
|
DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageEntryInfo,
|
||||||
DataUsageInfo, ScannerError, SizeSummary, TierStats,
|
DataUsageInfo, ScannerError, ScannerObjectIO, SizeSummary, TierStats,
|
||||||
};
|
};
|
||||||
use futures::future::join_all;
|
use futures::future::join_all;
|
||||||
use metrics::counter;
|
use metrics::counter;
|
||||||
@@ -41,7 +41,7 @@ use rustfs_ecstore::error::{Error, StorageError};
|
|||||||
use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
|
use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
|
||||||
use rustfs_ecstore::resolve_object_store_handle;
|
use rustfs_ecstore::resolve_object_store_handle;
|
||||||
use rustfs_ecstore::set_disk::SetDisks;
|
use rustfs_ecstore::set_disk::SetDisks;
|
||||||
use rustfs_ecstore::store_api::{ObjectIO, ObjectInfo};
|
use rustfs_ecstore::store_api::ObjectInfo;
|
||||||
use rustfs_ecstore::{error::Result, store::ECStore};
|
use rustfs_ecstore::{error::Result, store::ECStore};
|
||||||
use rustfs_filemeta::FileMeta;
|
use rustfs_filemeta::FileMeta;
|
||||||
use rustfs_storage_api::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
|
use rustfs_storage_api::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
|
||||||
@@ -407,7 +407,7 @@ async fn send_cache_root_entry_info(
|
|||||||
bucket_result_tx.lock().await.send(cache_root_entry_info(cache)).await
|
bucket_result_tx.lock().await.send(cache_root_entry_info(cache)).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn persist_and_publish_cache_snapshot<S: ObjectIO>(
|
async fn persist_and_publish_cache_snapshot<S: ScannerObjectIO>(
|
||||||
store: Arc<S>,
|
store: Arc<S>,
|
||||||
updates: &mpsc::Sender<DataUsageCache>,
|
updates: &mpsc::Sender<DataUsageCache>,
|
||||||
cache_snapshot: DataUsageCache,
|
cache_snapshot: DataUsageCache,
|
||||||
|
|||||||
@@ -111,3 +111,9 @@ Outer RustFS/IAM consumers must use `rustfs-storage-api` generic list response
|
|||||||
contracts directly for `ListObjectsV2Info`, `ListObjectVersionsInfo`, and
|
contracts directly for `ListObjectsV2Info`, `ListObjectVersionsInfo`, and
|
||||||
`ObjectInfoOrErr`; ECStore keeps the concrete aliases only for internal
|
`ObjectInfoOrErr`; ECStore keeps the concrete aliases only for internal
|
||||||
implementation and compatibility.
|
implementation and compatibility.
|
||||||
|
|
||||||
|
Outer RustFS/scanner consumers must use `rustfs-storage-api` operation traits
|
||||||
|
directly for `ObjectIO`, `ObjectOperations`, `ListOperations`,
|
||||||
|
`MultipartOperations`, `HealOperations`, and `NamespaceLocking`; ECStore keeps
|
||||||
|
the concrete compatibility traits only for internal implementation and
|
||||||
|
downstream compatibility.
|
||||||
|
|||||||
@@ -5,19 +5,20 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
## Current Context
|
## Current Context
|
||||||
|
|
||||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||||
- Branch: `overtrue/arch-storage-shared-operation-bounds`
|
- Branch: `overtrue/arch-storage-operation-bounds-cleanup`
|
||||||
- Baseline: `main` at `46e200c290e3fc0887f4c19172bcc146e019d737`
|
- Baseline: `main` at `a30cafa73fde3e7e88c160e70c290e74a0c2235f`
|
||||||
after the heal/namespace-lock operation contract merge.
|
after `rustfs/rustfs#3563` merged.
|
||||||
- PR type for this branch: `consumer-migration`
|
- PR type for this branch: `consumer-migration`
|
||||||
- Runtime behavior changes: no external behavior change expected.
|
- Runtime behavior changes: no external behavior change expected.
|
||||||
- Rust code changes: migrate RustFS bucket response builders and IAM walk
|
- Rust code changes: migrate scanner cache persistence, RustFS object
|
||||||
channel typing away from ECStore list response aliases to the generic
|
namespace-lock helper, and table catalog storage bounds away from ECStore
|
||||||
`rustfs-storage-api` list contracts while keeping ECStore-owned
|
compatibility operation traits to `rustfs-storage-api` operation traits with
|
||||||
`ObjectInfo`, `ObjectOptions`, readers, delete DTOs, and implementation
|
ECStore concrete associated-type bindings. ECStore-owned `ObjectInfo`,
|
||||||
behavior in ECStore.
|
`ObjectOptions`, readers, delete DTOs, walk filters, lock wrappers, and
|
||||||
- CI/script changes: extend migration guards so outer RustFS/IAM list response
|
implementation behavior stay in ECStore.
|
||||||
consumers do not drift back to ECStore alias imports.
|
- CI/script changes: extend migration guards so outer consumers do not drift
|
||||||
- Docs changes: record the shared list operation consumer cleanup slice.
|
back to ECStore operation trait imports.
|
||||||
|
- Docs changes: record the larger operation consumer-bound cleanup slice.
|
||||||
|
|
||||||
## Phase 0 Tasks
|
## Phase 0 Tasks
|
||||||
|
|
||||||
@@ -753,6 +754,27 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
formatting, diff hygiene, Rust risk scan, full pre-commit, and required
|
formatting, diff hygiene, Rust risk scan, full pre-commit, and required
|
||||||
three-expert review passed.
|
three-expert review passed.
|
||||||
|
|
||||||
|
- [x] `API-025` Clean external operation consumer bounds.
|
||||||
|
- Completed slice: migrate scanner data-usage cache storage bounds, RustFS
|
||||||
|
object-usecase namespace-lock helper bounds, and table catalog object
|
||||||
|
backend storage bounds from ECStore compatibility operation traits to
|
||||||
|
`rustfs-storage-api` operation traits with explicit ECStore concrete
|
||||||
|
associated-type bindings.
|
||||||
|
- Acceptance: outer RustFS/scanner consumers no longer import ECStore
|
||||||
|
operation traits, ECStore keeps compatibility traits for internal
|
||||||
|
implementation and downstream compatibility, and migration guards reject
|
||||||
|
restoring old outer-consumer operation trait imports.
|
||||||
|
- Must preserve: scanner cache load/save behavior, scanner backend timeout
|
||||||
|
and retry behavior, object self-copy namespace-lock quorum/error mapping,
|
||||||
|
table catalog object read/write/list/lock behavior, ECStore object metadata
|
||||||
|
shape, reader shape, walk filter shape, and storage error conversion.
|
||||||
|
- Risk defense: this slice changes only generic bounds/import ownership;
|
||||||
|
ECStore still owns concrete object DTOs, readers, delete DTOs, lock wrappers,
|
||||||
|
walk filters, and implementation bodies.
|
||||||
|
- Verification: focused RustFS/scanner compile and tests, migration/layer
|
||||||
|
guards, formatting, diff hygiene, Rust risk scan, full pre-commit, and
|
||||||
|
required three-expert review passed.
|
||||||
|
|
||||||
## Phase 8 Background Controller Tasks
|
## Phase 8 Background Controller Tasks
|
||||||
|
|
||||||
- [x] `BGC-001` Inventory background services.
|
- [x] `BGC-001` Inventory background services.
|
||||||
@@ -1029,17 +1051,20 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
|
|
||||||
| Expert | Status | Notes |
|
| Expert | Status | Notes |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Quality/architecture | passed | Outer RustFS/IAM list response consumers now use the generic storage-api contracts; ECStore keeps concrete aliases only for implementation and compatibility. |
|
| Quality/architecture | passed | Outer scanner/RustFS operation consumers now use storage-api operation traits with explicit ECStore concrete associated-type bindings; ECStore keeps compatibility traits only for implementation and downstream compatibility. |
|
||||||
| Migration preservation | passed | S3 list response mapping and IAM walk item/error handling keep the same ECStore `ObjectInfo`, error conversion, and walk option inference behavior. |
|
| Migration preservation | passed | Scanner cache persistence, object self-copy namespace locking, and table catalog object storage keep the same ECStore DTOs, readers, lock wrappers, walk filter shape, and storage error conversion behavior. |
|
||||||
| Testing/verification | passed | Focused RustFS/IAM compile/tests, migration/layer guards, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` passed. |
|
| Testing/verification | passed | Focused RustFS/scanner compile/tests, migration/layer guards, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` passed. |
|
||||||
|
|
||||||
## Verification Notes
|
## Verification Notes
|
||||||
|
|
||||||
Passed before push:
|
Passed before push:
|
||||||
|
|
||||||
- `cargo check --tests -p rustfs -p rustfs-iam`: passed.
|
- `cargo check --tests -p rustfs -p rustfs-scanner`: passed.
|
||||||
- `cargo test -p rustfs --lib storage::s3_api::bucket`: passed; 22 passed.
|
- `cargo test -p rustfs-scanner`: passed; 151 unit tests passed, lifecycle
|
||||||
- `cargo test -p rustfs-iam`: passed; 150 passed.
|
focused test passed, 14 lifecycle integration tests ignored by default.
|
||||||
|
- `cargo test -p rustfs --lib table_catalog`: passed; 168 passed.
|
||||||
|
- `cargo test -p rustfs --lib app::object_usecase`: passed; 86 passed, 2
|
||||||
|
ignored.
|
||||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||||
- `./scripts/check_layer_dependencies.sh`: passed.
|
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||||
- `cargo fmt --all --check`: passed.
|
- `cargo fmt --all --check`: passed.
|
||||||
@@ -1052,20 +1077,20 @@ Passed before push:
|
|||||||
|
|
||||||
Notes:
|
Notes:
|
||||||
|
|
||||||
- This slice follows the heal and namespace-lock operation contract branch and
|
- This slice follows the shared list operation consumer cleanup and keeps the
|
||||||
keeps the old aggregate facade, DTO/helper, response, and operation contract
|
old aggregate facade, DTO/helper, response, and operation contract guards
|
||||||
guards active.
|
active.
|
||||||
- Outer RustFS/IAM consumers now bind low-coupling list response/channel
|
- Outer scanner/RustFS operation consumers now bind operation traits through
|
||||||
containers through `rustfs-storage-api`; ECStore keeps the concrete aliases,
|
`rustfs-storage-api`; ECStore keeps concrete object metadata, options,
|
||||||
object metadata, filemeta-bound walk filter, readers, delete DTOs, and list
|
readers, delete DTOs, lock wrappers, filemeta-bound walk filters, and
|
||||||
implementation behavior.
|
implementation behavior.
|
||||||
- The slice does not alter list, walk, bucket, object, delete, reader,
|
- The slice does not alter scanner cache load/save behavior, object self-copy
|
||||||
tag/metadata, heal, namespace-lock, multipart, or storage error runtime
|
lock behavior, table catalog object operations, list/walk implementation,
|
||||||
behavior.
|
object reader/writer behavior, or storage error runtime behavior.
|
||||||
|
|
||||||
## Handoff Notes
|
## Handoff Notes
|
||||||
|
|
||||||
- Shared list operation consumer cleanup is rebased onto `main` after
|
- External operation consumer cleanup is based on `main` after
|
||||||
`rustfs/rustfs#3560` merged.
|
`rustfs/rustfs#3563` merged.
|
||||||
- After this lands, continue with larger consumer-migration batches instead of
|
- Continue with larger consumer-migration batches; avoid moving ECStore-owned
|
||||||
single-alias slices.
|
DTOs/readers/walk filters until their concrete behavior is isolated.
|
||||||
|
|||||||
@@ -74,11 +74,13 @@ use rustfs_ecstore::client::object_api_utils::to_s3s_etag;
|
|||||||
use rustfs_ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
use rustfs_ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
||||||
use rustfs_ecstore::config::storageclass;
|
use rustfs_ecstore::config::storageclass;
|
||||||
use rustfs_ecstore::disk::{error::DiskError, error_reduce::is_all_buckets_not_found};
|
use rustfs_ecstore::disk::{error::DiskError, error_reduce::is_all_buckets_not_found};
|
||||||
use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found};
|
use rustfs_ecstore::error::{
|
||||||
|
Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||||
|
};
|
||||||
use rustfs_ecstore::rio::{DynReader, HashReader, WritePlan, wrap_reader};
|
use rustfs_ecstore::rio::{DynReader, HashReader, WritePlan, wrap_reader};
|
||||||
use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
|
use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
|
||||||
use rustfs_ecstore::store::ECStore;
|
use rustfs_ecstore::store::ECStore;
|
||||||
use rustfs_ecstore::store_api::{NamespaceLocking, ObjectInfo, ObjectOptions, ObjectToDelete, PutObjReader};
|
use rustfs_ecstore::store_api::{ObjectInfo, ObjectOptions, ObjectToDelete, PutObjReader};
|
||||||
use rustfs_filemeta::{
|
use rustfs_filemeta::{
|
||||||
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType,
|
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType,
|
||||||
ReplicationType, RestoreStatusOps, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
|
ReplicationType, RestoreStatusOps, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
|
||||||
@@ -93,7 +95,7 @@ use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_fo
|
|||||||
use rustfs_s3select_api::object_store::bytes_stream;
|
use rustfs_s3select_api::object_store::bytes_stream;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use rustfs_storage_api::HTTPPreconditions;
|
use rustfs_storage_api::HTTPPreconditions;
|
||||||
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
|
use rustfs_storage_api::{HTTPRangeSpec, NamespaceLocking, ObjectIO as _, ObjectOperations as _};
|
||||||
use rustfs_targets::{
|
use rustfs_targets::{
|
||||||
EventName, extract_params_header, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent,
|
EventName, extract_params_header, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent,
|
||||||
};
|
};
|
||||||
@@ -876,11 +878,10 @@ fn copy_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn acquire_self_copy_namespace_lock<S: NamespaceLocking + ?Sized>(
|
async fn acquire_self_copy_namespace_lock<S>(store: &S, bucket: &str, object: &str) -> S3Result<NamespaceLockGuard>
|
||||||
store: &S,
|
where
|
||||||
bucket: &str,
|
S: NamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper> + ?Sized,
|
||||||
object: &str,
|
{
|
||||||
) -> S3Result<NamespaceLockGuard> {
|
|
||||||
let object = encode_dir_object(object);
|
let object = encode_dir_object(object);
|
||||||
let lock = store.new_ns_lock(bucket, &object).await.map_err(ApiError::from)?;
|
let lock = store.new_ns_lock(bucket, &object).await.map_err(ApiError::from)?;
|
||||||
lock.get_write_lock(get_lock_acquire_timeout())
|
lock.get_write_lock(get_lock_acquire_timeout())
|
||||||
|
|||||||
@@ -42,12 +42,18 @@ use rustfs_ecstore::bucket::{
|
|||||||
metadata_sys,
|
metadata_sys,
|
||||||
};
|
};
|
||||||
use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
|
use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
|
||||||
use rustfs_ecstore::error::StorageError;
|
use rustfs_ecstore::error::{Error as EcstoreError, StorageError};
|
||||||
use rustfs_ecstore::{
|
use rustfs_ecstore::{
|
||||||
set_disk::get_lock_acquire_timeout,
|
set_disk::get_lock_acquire_timeout,
|
||||||
store_api::{ListOperations, NamespaceLocking, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader},
|
store_api::{DeletedObject, GetObjectReader, ObjectInfo, ObjectOptions, ObjectToDelete, PutObjReader},
|
||||||
|
};
|
||||||
|
use rustfs_filemeta::FileInfo;
|
||||||
|
use rustfs_storage_api::{
|
||||||
|
HTTPPreconditions, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
|
||||||
|
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as StorageListOperations,
|
||||||
|
NamespaceLocking as StorageNamespaceLocking, ObjectIO as StorageObjectIO, ObjectInfoOrErr as StorageObjectInfoOrErr,
|
||||||
|
ObjectOperations as StorageObjectOperations, WalkOptions as StorageWalkOptions,
|
||||||
};
|
};
|
||||||
use rustfs_storage_api::HTTPPreconditions;
|
|
||||||
use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
||||||
use time::{Duration, OffsetDateTime};
|
use time::{Duration, OffsetDateTime};
|
||||||
use tokio::io::AsyncReadExt;
|
use tokio::io::AsyncReadExt;
|
||||||
@@ -105,6 +111,67 @@ const ICEBERG_REF_MIN_SNAPSHOTS_TO_KEEP_FIELD: &str = "min-snapshots-to-keep";
|
|||||||
const ICEBERG_REF_MAX_SNAPSHOT_AGE_MS_FIELD: &str = "max-snapshot-age-ms";
|
const ICEBERG_REF_MAX_SNAPSHOT_AGE_MS_FIELD: &str = "max-snapshot-age-ms";
|
||||||
const ICEBERG_REF_MAX_REF_AGE_MS_FIELD: &str = "max-ref-age-ms";
|
const ICEBERG_REF_MAX_REF_AGE_MS_FIELD: &str = "max-ref-age-ms";
|
||||||
|
|
||||||
|
type CatalogListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||||
|
type CatalogListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||||
|
type CatalogObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, EcstoreError>;
|
||||||
|
type CatalogWalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||||
|
|
||||||
|
pub(crate) trait TableCatalogStorage:
|
||||||
|
StorageObjectIO<
|
||||||
|
Error = EcstoreError,
|
||||||
|
RangeSpec = HTTPRangeSpec,
|
||||||
|
HeaderMap = HeaderMap,
|
||||||
|
ObjectOptions = ObjectOptions,
|
||||||
|
ObjectInfo = ObjectInfo,
|
||||||
|
GetObjectReader = GetObjectReader,
|
||||||
|
PutObjectReader = PutObjReader,
|
||||||
|
> + StorageObjectOperations<
|
||||||
|
Error = EcstoreError,
|
||||||
|
ObjectInfo = ObjectInfo,
|
||||||
|
ObjectOptions = ObjectOptions,
|
||||||
|
FileInfo = FileInfo,
|
||||||
|
ObjectToDelete = ObjectToDelete,
|
||||||
|
DeletedObject = DeletedObject,
|
||||||
|
> + StorageListOperations<
|
||||||
|
Error = EcstoreError,
|
||||||
|
ListObjectsV2Info = CatalogListObjectsV2Info,
|
||||||
|
ListObjectVersionsInfo = CatalogListObjectVersionsInfo,
|
||||||
|
ObjectInfoOrErr = CatalogObjectInfoOrErr,
|
||||||
|
WalkOptions = CatalogWalkOptions,
|
||||||
|
WalkCancellation = tokio_util::sync::CancellationToken,
|
||||||
|
WalkResultSender = tokio::sync::mpsc::Sender<CatalogObjectInfoOrErr>,
|
||||||
|
> + StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> TableCatalogStorage for T where
|
||||||
|
T: StorageObjectIO<
|
||||||
|
Error = EcstoreError,
|
||||||
|
RangeSpec = HTTPRangeSpec,
|
||||||
|
HeaderMap = HeaderMap,
|
||||||
|
ObjectOptions = ObjectOptions,
|
||||||
|
ObjectInfo = ObjectInfo,
|
||||||
|
GetObjectReader = GetObjectReader,
|
||||||
|
PutObjectReader = PutObjReader,
|
||||||
|
> + StorageObjectOperations<
|
||||||
|
Error = EcstoreError,
|
||||||
|
ObjectInfo = ObjectInfo,
|
||||||
|
ObjectOptions = ObjectOptions,
|
||||||
|
FileInfo = FileInfo,
|
||||||
|
ObjectToDelete = ObjectToDelete,
|
||||||
|
DeletedObject = DeletedObject,
|
||||||
|
> + StorageListOperations<
|
||||||
|
Error = EcstoreError,
|
||||||
|
ListObjectsV2Info = CatalogListObjectsV2Info,
|
||||||
|
ListObjectVersionsInfo = CatalogListObjectVersionsInfo,
|
||||||
|
ObjectInfoOrErr = CatalogObjectInfoOrErr,
|
||||||
|
WalkOptions = CatalogWalkOptions,
|
||||||
|
WalkCancellation = tokio_util::sync::CancellationToken,
|
||||||
|
WalkResultSender = tokio::sync::mpsc::Sender<CatalogObjectInfoOrErr>,
|
||||||
|
> + StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub enum CatalogIdentifierError {
|
pub enum CatalogIdentifierError {
|
||||||
Empty,
|
Empty,
|
||||||
@@ -3487,7 +3554,7 @@ impl<S> Clone for EcStoreTableCatalogObjectBackend<S> {
|
|||||||
|
|
||||||
impl<S> EcStoreTableCatalogObjectBackend<S>
|
impl<S> EcStoreTableCatalogObjectBackend<S>
|
||||||
where
|
where
|
||||||
S: ObjectIO + ObjectOperations + ListOperations + NamespaceLocking,
|
S: TableCatalogStorage,
|
||||||
{
|
{
|
||||||
pub fn new(store: Arc<S>) -> Self {
|
pub fn new(store: Arc<S>) -> Self {
|
||||||
Self { store }
|
Self { store }
|
||||||
@@ -3499,7 +3566,7 @@ pub(crate) type EcStoreTableCatalogStore<S> = ObjectTableCatalogStore<EcStoreTab
|
|||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
impl<S> TableCatalogObjectBackend for EcStoreTableCatalogObjectBackend<S>
|
impl<S> TableCatalogObjectBackend for EcStoreTableCatalogObjectBackend<S>
|
||||||
where
|
where
|
||||||
S: ObjectIO + ObjectOperations + ListOperations + NamespaceLocking,
|
S: TableCatalogStorage,
|
||||||
{
|
{
|
||||||
async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<Option<TableCatalogObject>> {
|
async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<Option<TableCatalogObject>> {
|
||||||
self.read_object_with_options(bucket, object, ObjectOptions::default()).await
|
self.read_object_with_options(bucket, object, ObjectOptions::default()).await
|
||||||
@@ -3598,7 +3665,7 @@ where
|
|||||||
|
|
||||||
impl<S> EcStoreTableCatalogObjectBackend<S>
|
impl<S> EcStoreTableCatalogObjectBackend<S>
|
||||||
where
|
where
|
||||||
S: ObjectIO + ObjectOperations + ListOperations + NamespaceLocking,
|
S: TableCatalogStorage,
|
||||||
{
|
{
|
||||||
async fn read_object_with_options(
|
async fn read_object_with_options(
|
||||||
&self,
|
&self,
|
||||||
|
|||||||
@@ -58,6 +58,7 @@ STORE_API_RANGE_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_range_helper_reexpor
|
|||||||
STORE_API_LIST_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_list_helper_reexports.txt"
|
STORE_API_LIST_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_list_helper_reexports.txt"
|
||||||
STORE_API_LIST_RESPONSE_REEXPORTS_FILE="${TMP_DIR}/store_api_list_response_reexports.txt"
|
STORE_API_LIST_RESPONSE_REEXPORTS_FILE="${TMP_DIR}/store_api_list_response_reexports.txt"
|
||||||
STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_list_consumer_hits.txt"
|
STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_list_consumer_hits.txt"
|
||||||
|
STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_operation_consumer_hits.txt"
|
||||||
STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE="${TMP_DIR}/store_api_object_operation_local_method_hits.txt"
|
STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE="${TMP_DIR}/store_api_object_operation_local_method_hits.txt"
|
||||||
|
|
||||||
awk '
|
awk '
|
||||||
@@ -309,6 +310,16 @@ if [[ -s "$STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE" ]]; then
|
|||||||
report_failure "external list response consumers must use rustfs-storage-api contracts: $(paste -sd '; ' "$STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE")"
|
report_failure "external list response consumers must use rustfs-storage-api contracts: $(paste -sd '; ' "$STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE")"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
(
|
||||||
|
cd "$ROOT_DIR"
|
||||||
|
rg -n --no-heading 'rustfs_ecstore::store_api(?:::\{[^}]*\b(?:ObjectIO|ObjectOperations|ListOperations|MultipartOperations|HealOperations|NamespaceLocking)\b|::(?:ObjectIO|ObjectOperations|ListOperations|MultipartOperations|HealOperations|NamespaceLocking)\b)' \
|
||||||
|
rustfs/src crates --glob '!crates/ecstore/**' || true
|
||||||
|
) >"$STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE"
|
||||||
|
|
||||||
|
if [[ -s "$STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE" ]]; then
|
||||||
|
report_failure "external operation consumers must use rustfs-storage-api traits: $(paste -sd '; ' "$STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE")"
|
||||||
|
fi
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
rg -n --no-heading 'async fn (get_object_reader|put_object|get_object_info|verify_object_integrity|copy_object|delete_object_version|delete_object|delete_objects|put_object_metadata|get_object_tags|put_object_tags|delete_object_tags|add_partial|transition_object|restore_transitioned_object|list_multipart_uploads|new_multipart_upload|copy_object_part|put_object_part|get_multipart_info|list_object_parts|abort_multipart_upload|complete_multipart_upload|heal_format|heal_bucket|heal_object|get_pool_and_set|check_abandoned_parts|new_ns_lock)\b' \
|
rg -n --no-heading 'async fn (get_object_reader|put_object|get_object_info|verify_object_integrity|copy_object|delete_object_version|delete_object|delete_objects|put_object_metadata|get_object_tags|put_object_tags|delete_object_tags|add_partial|transition_object|restore_transitioned_object|list_multipart_uploads|new_multipart_upload|copy_object_part|put_object_part|get_multipart_info|list_object_parts|abort_multipart_upload|complete_multipart_upload|heal_format|heal_bucket|heal_object|get_pool_and_set|check_abandoned_parts|new_ns_lock)\b' \
|
||||||
|
|||||||
Reference in New Issue
Block a user