mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-28 09:08:58 +00:00
refactor: clean external operation consumers (#3565)
This commit is contained in:
@@ -33,8 +33,9 @@ use rustfs_ecstore::{
|
||||
config::{com::save_config, storageclass},
|
||||
disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
|
||||
error::{Error, Result as StorageResult, StorageError},
|
||||
store_api::{ObjectIO, ObjectInfo, ObjectOptions},
|
||||
store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
|
||||
};
|
||||
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO};
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
use tracing::warn;
|
||||
@@ -60,6 +61,32 @@ const EVENT_SCANNER_CACHE_LOAD_STATE: &str = "scanner_cache_load_state";
|
||||
const EVENT_SCANNER_CACHE_SAVE_STATE: &str = "scanner_cache_save_state";
|
||||
static CACHE_SAVE_METRICS_ONCE: Once = Once::new();
|
||||
|
||||
pub trait ScannerObjectIO:
|
||||
ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> ScannerObjectIO for T where
|
||||
T: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>
|
||||
{
|
||||
}
|
||||
|
||||
pub const DATA_USAGE_SCAN_CHECKPOINT_VERSION: u16 = 1;
|
||||
|
||||
// Data usage paths (computed at runtime)
|
||||
@@ -616,7 +643,7 @@ impl DataUsageCache {
|
||||
/// Only backend errors are returned as errors.
|
||||
/// The loader is optimistic and has no locking, but tries 5 times before giving up.
|
||||
/// If the object is not found, a nil error with empty data usage cache is returned.
|
||||
pub async fn load<S: ObjectIO>(&mut self, store: Arc<S>, name: &str) -> StorageResult<()> {
|
||||
pub async fn load<S: ScannerObjectIO>(&mut self, store: Arc<S>, name: &str) -> StorageResult<()> {
|
||||
// By default, empty data usage cache
|
||||
*self = DataUsageCache::default();
|
||||
|
||||
@@ -671,7 +698,7 @@ impl DataUsageCache {
|
||||
}
|
||||
// Inner load function that attempts to load from a specific path
|
||||
// Returns (should_retry, cache_option, error_option)
|
||||
async fn try_load_inner<S: ObjectIO>(
|
||||
async fn try_load_inner<S: ScannerObjectIO>(
|
||||
store: Arc<S>,
|
||||
load_name: &str,
|
||||
timeout_duration: Duration,
|
||||
@@ -866,7 +893,7 @@ impl DataUsageCache {
|
||||
Err(last_err.unwrap_or_else(|| StorageError::other("Failed to save data usage cache".to_string())))
|
||||
}
|
||||
|
||||
async fn save_path_with_retry<S: ObjectIO>(
|
||||
async fn save_path_with_retry<S: ScannerObjectIO>(
|
||||
store: Arc<S>,
|
||||
path: &str,
|
||||
buf: &[u8],
|
||||
@@ -889,7 +916,7 @@ impl DataUsageCache {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn save<S: ObjectIO>(&self, store: Arc<S>, name: &str) -> StorageResult<()> {
|
||||
pub async fn save<S: ScannerObjectIO>(&self, store: Arc<S>, name: &str) -> StorageResult<()> {
|
||||
let mut buf = Vec::new();
|
||||
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
||||
let timeout_duration = Self::cache_save_timeout();
|
||||
|
||||
@@ -14,7 +14,9 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::data_usage_define::{BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH};
|
||||
use crate::data_usage_define::{
|
||||
BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, ScannerObjectIO,
|
||||
};
|
||||
use crate::runtime_config::{
|
||||
current_scanner_runtime_config, lookup_scanner_runtime_config, refresh_scanner_runtime_config_from_global,
|
||||
scanner_bitrot_cycle, scanner_cycle_interval, scanner_start_delay, set_scanner_default_cycle_secs,
|
||||
@@ -45,7 +47,6 @@ use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
|
||||
use rustfs_ecstore::error::Error as EcstoreError;
|
||||
use rustfs_ecstore::global::is_erasure_sd;
|
||||
use rustfs_ecstore::store::ECStore;
|
||||
use rustfs_ecstore::store_api::ObjectIO;
|
||||
use rustfs_storage_api::{BucketOperations, BucketOptions, NamespaceLocking as _};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::mpsc;
|
||||
@@ -976,7 +977,7 @@ impl Drop for ScannerScanModeGuard {
|
||||
#[instrument(skip(ctx, storeapi))]
|
||||
pub async fn store_data_usage_in_backend(
|
||||
ctx: CancellationToken,
|
||||
storeapi: Arc<impl ObjectIO>,
|
||||
storeapi: Arc<impl ScannerObjectIO>,
|
||||
mut receiver: mpsc::Receiver<DataUsageInfo>,
|
||||
) {
|
||||
let mut attempts = 1u32;
|
||||
|
||||
@@ -17,7 +17,7 @@ use crate::scanner_folder::{ScannerItem, scan_data_folder};
|
||||
use crate::sleeper::SCANNER_SLEEPER;
|
||||
use crate::{
|
||||
DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageEntryInfo,
|
||||
DataUsageInfo, ScannerError, SizeSummary, TierStats,
|
||||
DataUsageInfo, ScannerError, ScannerObjectIO, SizeSummary, TierStats,
|
||||
};
|
||||
use futures::future::join_all;
|
||||
use metrics::counter;
|
||||
@@ -41,7 +41,7 @@ use rustfs_ecstore::error::{Error, StorageError};
|
||||
use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
|
||||
use rustfs_ecstore::resolve_object_store_handle;
|
||||
use rustfs_ecstore::set_disk::SetDisks;
|
||||
use rustfs_ecstore::store_api::{ObjectIO, ObjectInfo};
|
||||
use rustfs_ecstore::store_api::ObjectInfo;
|
||||
use rustfs_ecstore::{error::Result, store::ECStore};
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_storage_api::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
|
||||
@@ -407,7 +407,7 @@ async fn send_cache_root_entry_info(
|
||||
bucket_result_tx.lock().await.send(cache_root_entry_info(cache)).await
|
||||
}
|
||||
|
||||
async fn persist_and_publish_cache_snapshot<S: ObjectIO>(
|
||||
async fn persist_and_publish_cache_snapshot<S: ScannerObjectIO>(
|
||||
store: Arc<S>,
|
||||
updates: &mpsc::Sender<DataUsageCache>,
|
||||
cache_snapshot: DataUsageCache,
|
||||
|
||||
@@ -111,3 +111,9 @@ Outer RustFS/IAM consumers must use `rustfs-storage-api` generic list response
|
||||
contracts directly for `ListObjectsV2Info`, `ListObjectVersionsInfo`, and
|
||||
`ObjectInfoOrErr`; ECStore keeps the concrete aliases only for internal
|
||||
implementation and compatibility.
|
||||
|
||||
Outer RustFS/scanner consumers must use `rustfs-storage-api` operation traits
|
||||
directly for `ObjectIO`, `ObjectOperations`, `ListOperations`,
|
||||
`MultipartOperations`, `HealOperations`, and `NamespaceLocking`; ECStore keeps
|
||||
the concrete compatibility traits only for internal implementation and
|
||||
downstream compatibility.
|
||||
|
||||
@@ -5,19 +5,20 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
## Current Context
|
||||
|
||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||
- Branch: `overtrue/arch-storage-shared-operation-bounds`
|
||||
- Baseline: `main` at `46e200c290e3fc0887f4c19172bcc146e019d737`
|
||||
after the heal/namespace-lock operation contract merge.
|
||||
- Branch: `overtrue/arch-storage-operation-bounds-cleanup`
|
||||
- Baseline: `main` at `a30cafa73fde3e7e88c160e70c290e74a0c2235f`
|
||||
after `rustfs/rustfs#3563` merged.
|
||||
- PR type for this branch: `consumer-migration`
|
||||
- Runtime behavior changes: no external behavior change expected.
|
||||
- Rust code changes: migrate RustFS bucket response builders and IAM walk
|
||||
channel typing away from ECStore list response aliases to the generic
|
||||
`rustfs-storage-api` list contracts while keeping ECStore-owned
|
||||
`ObjectInfo`, `ObjectOptions`, readers, delete DTOs, and implementation
|
||||
behavior in ECStore.
|
||||
- CI/script changes: extend migration guards so outer RustFS/IAM list response
|
||||
consumers do not drift back to ECStore alias imports.
|
||||
- Docs changes: record the shared list operation consumer cleanup slice.
|
||||
- Rust code changes: migrate scanner cache persistence, RustFS object
|
||||
namespace-lock helper, and table catalog storage bounds away from ECStore
|
||||
compatibility operation traits to `rustfs-storage-api` operation traits with
|
||||
ECStore concrete associated-type bindings. ECStore-owned `ObjectInfo`,
|
||||
`ObjectOptions`, readers, delete DTOs, walk filters, lock wrappers, and
|
||||
implementation behavior stay in ECStore.
|
||||
- CI/script changes: extend migration guards so outer consumers do not drift
|
||||
back to ECStore operation trait imports.
|
||||
- Docs changes: record the larger operation consumer-bound cleanup slice.
|
||||
|
||||
## Phase 0 Tasks
|
||||
|
||||
@@ -753,6 +754,27 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
formatting, diff hygiene, Rust risk scan, full pre-commit, and required
|
||||
three-expert review passed.
|
||||
|
||||
- [x] `API-025` Clean external operation consumer bounds.
|
||||
- Completed slice: migrate scanner data-usage cache storage bounds, RustFS
|
||||
object-usecase namespace-lock helper bounds, and table catalog object
|
||||
backend storage bounds from ECStore compatibility operation traits to
|
||||
`rustfs-storage-api` operation traits with explicit ECStore concrete
|
||||
associated-type bindings.
|
||||
- Acceptance: outer RustFS/scanner consumers no longer import ECStore
|
||||
operation traits, ECStore keeps compatibility traits for internal
|
||||
implementation and downstream compatibility, and migration guards reject
|
||||
restoring old outer-consumer operation trait imports.
|
||||
- Must preserve: scanner cache load/save behavior, scanner backend timeout
|
||||
and retry behavior, object self-copy namespace-lock quorum/error mapping,
|
||||
table catalog object read/write/list/lock behavior, ECStore object metadata
|
||||
shape, reader shape, walk filter shape, and storage error conversion.
|
||||
- Risk defense: this slice changes only generic bounds/import ownership;
|
||||
ECStore still owns concrete object DTOs, readers, delete DTOs, lock wrappers,
|
||||
walk filters, and implementation bodies.
|
||||
- Verification: focused RustFS/scanner compile and tests, migration/layer
|
||||
guards, formatting, diff hygiene, Rust risk scan, full pre-commit, and
|
||||
required three-expert review passed.
|
||||
|
||||
## Phase 8 Background Controller Tasks
|
||||
|
||||
- [x] `BGC-001` Inventory background services.
|
||||
@@ -1029,17 +1051,20 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
|
||||
| Expert | Status | Notes |
|
||||
|---|---|---|
|
||||
| Quality/architecture | passed | Outer RustFS/IAM list response consumers now use the generic storage-api contracts; ECStore keeps concrete aliases only for implementation and compatibility. |
|
||||
| Migration preservation | passed | S3 list response mapping and IAM walk item/error handling keep the same ECStore `ObjectInfo`, error conversion, and walk option inference behavior. |
|
||||
| Testing/verification | passed | Focused RustFS/IAM compile/tests, migration/layer guards, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` passed. |
|
||||
| Quality/architecture | passed | Outer scanner/RustFS operation consumers now use storage-api operation traits with explicit ECStore concrete associated-type bindings; ECStore keeps compatibility traits only for implementation and downstream compatibility. |
|
||||
| Migration preservation | passed | Scanner cache persistence, object self-copy namespace locking, and table catalog object storage keep the same ECStore DTOs, readers, lock wrappers, walk filter shape, and storage error conversion behavior. |
|
||||
| Testing/verification | passed | Focused RustFS/scanner compile/tests, migration/layer guards, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` passed. |
|
||||
|
||||
## Verification Notes
|
||||
|
||||
Passed before push:
|
||||
|
||||
- `cargo check --tests -p rustfs -p rustfs-iam`: passed.
|
||||
- `cargo test -p rustfs --lib storage::s3_api::bucket`: passed; 22 passed.
|
||||
- `cargo test -p rustfs-iam`: passed; 150 passed.
|
||||
- `cargo check --tests -p rustfs -p rustfs-scanner`: passed.
|
||||
- `cargo test -p rustfs-scanner`: passed; 151 unit tests passed, lifecycle
|
||||
focused test passed, 14 lifecycle integration tests ignored by default.
|
||||
- `cargo test -p rustfs --lib table_catalog`: passed; 168 passed.
|
||||
- `cargo test -p rustfs --lib app::object_usecase`: passed; 86 passed, 2
|
||||
ignored.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
@@ -1052,20 +1077,20 @@ Passed before push:
|
||||
|
||||
Notes:
|
||||
|
||||
- This slice follows the heal and namespace-lock operation contract branch and
|
||||
keeps the old aggregate facade, DTO/helper, response, and operation contract
|
||||
guards active.
|
||||
- Outer RustFS/IAM consumers now bind low-coupling list response/channel
|
||||
containers through `rustfs-storage-api`; ECStore keeps the concrete aliases,
|
||||
object metadata, filemeta-bound walk filter, readers, delete DTOs, and list
|
||||
- This slice follows the shared list operation consumer cleanup and keeps the
|
||||
old aggregate facade, DTO/helper, response, and operation contract guards
|
||||
active.
|
||||
- Outer scanner/RustFS operation consumers now bind operation traits through
|
||||
`rustfs-storage-api`; ECStore keeps concrete object metadata, options,
|
||||
readers, delete DTOs, lock wrappers, filemeta-bound walk filters, and
|
||||
implementation behavior.
|
||||
- The slice does not alter list, walk, bucket, object, delete, reader,
|
||||
tag/metadata, heal, namespace-lock, multipart, or storage error runtime
|
||||
behavior.
|
||||
- The slice does not alter scanner cache load/save behavior, object self-copy
|
||||
lock behavior, table catalog object operations, list/walk implementation,
|
||||
object reader/writer behavior, or storage error runtime behavior.
|
||||
|
||||
## Handoff Notes
|
||||
|
||||
- Shared list operation consumer cleanup is rebased onto `main` after
|
||||
`rustfs/rustfs#3560` merged.
|
||||
- After this lands, continue with larger consumer-migration batches instead of
|
||||
single-alias slices.
|
||||
- External operation consumer cleanup is based on `main` after
|
||||
`rustfs/rustfs#3563` merged.
|
||||
- Continue with larger consumer-migration batches; avoid moving ECStore-owned
|
||||
DTOs/readers/walk filters until their concrete behavior is isolated.
|
||||
|
||||
@@ -74,11 +74,13 @@ use rustfs_ecstore::client::object_api_utils::to_s3s_etag;
|
||||
use rustfs_ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
||||
use rustfs_ecstore::config::storageclass;
|
||||
use rustfs_ecstore::disk::{error::DiskError, error_reduce::is_all_buckets_not_found};
|
||||
use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found};
|
||||
use rustfs_ecstore::error::{
|
||||
Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||
};
|
||||
use rustfs_ecstore::rio::{DynReader, HashReader, WritePlan, wrap_reader};
|
||||
use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
|
||||
use rustfs_ecstore::store::ECStore;
|
||||
use rustfs_ecstore::store_api::{NamespaceLocking, ObjectInfo, ObjectOptions, ObjectToDelete, PutObjReader};
|
||||
use rustfs_ecstore::store_api::{ObjectInfo, ObjectOptions, ObjectToDelete, PutObjReader};
|
||||
use rustfs_filemeta::{
|
||||
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType,
|
||||
ReplicationType, RestoreStatusOps, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
|
||||
@@ -93,7 +95,7 @@ use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_fo
|
||||
use rustfs_s3select_api::object_store::bytes_stream;
|
||||
#[cfg(test)]
|
||||
use rustfs_storage_api::HTTPPreconditions;
|
||||
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
|
||||
use rustfs_storage_api::{HTTPRangeSpec, NamespaceLocking, ObjectIO as _, ObjectOperations as _};
|
||||
use rustfs_targets::{
|
||||
EventName, extract_params_header, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent,
|
||||
};
|
||||
@@ -876,11 +878,10 @@ fn copy_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err
|
||||
}
|
||||
}
|
||||
|
||||
async fn acquire_self_copy_namespace_lock<S: NamespaceLocking + ?Sized>(
|
||||
store: &S,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> S3Result<NamespaceLockGuard> {
|
||||
async fn acquire_self_copy_namespace_lock<S>(store: &S, bucket: &str, object: &str) -> S3Result<NamespaceLockGuard>
|
||||
where
|
||||
S: NamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper> + ?Sized,
|
||||
{
|
||||
let object = encode_dir_object(object);
|
||||
let lock = store.new_ns_lock(bucket, &object).await.map_err(ApiError::from)?;
|
||||
lock.get_write_lock(get_lock_acquire_timeout())
|
||||
|
||||
@@ -42,12 +42,18 @@ use rustfs_ecstore::bucket::{
|
||||
metadata_sys,
|
||||
};
|
||||
use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
|
||||
use rustfs_ecstore::error::StorageError;
|
||||
use rustfs_ecstore::error::{Error as EcstoreError, StorageError};
|
||||
use rustfs_ecstore::{
|
||||
set_disk::get_lock_acquire_timeout,
|
||||
store_api::{ListOperations, NamespaceLocking, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader},
|
||||
store_api::{DeletedObject, GetObjectReader, ObjectInfo, ObjectOptions, ObjectToDelete, PutObjReader},
|
||||
};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_storage_api::{
|
||||
HTTPPreconditions, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
|
||||
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as StorageListOperations,
|
||||
NamespaceLocking as StorageNamespaceLocking, ObjectIO as StorageObjectIO, ObjectInfoOrErr as StorageObjectInfoOrErr,
|
||||
ObjectOperations as StorageObjectOperations, WalkOptions as StorageWalkOptions,
|
||||
};
|
||||
use rustfs_storage_api::HTTPPreconditions;
|
||||
use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
||||
use time::{Duration, OffsetDateTime};
|
||||
use tokio::io::AsyncReadExt;
|
||||
@@ -105,6 +111,67 @@ const ICEBERG_REF_MIN_SNAPSHOTS_TO_KEEP_FIELD: &str = "min-snapshots-to-keep";
|
||||
const ICEBERG_REF_MAX_SNAPSHOT_AGE_MS_FIELD: &str = "max-snapshot-age-ms";
|
||||
const ICEBERG_REF_MAX_REF_AGE_MS_FIELD: &str = "max-ref-age-ms";
|
||||
|
||||
type CatalogListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
type CatalogListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type CatalogObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, EcstoreError>;
|
||||
type CatalogWalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
pub(crate) trait TableCatalogStorage:
|
||||
StorageObjectIO<
|
||||
Error = EcstoreError,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + StorageObjectOperations<
|
||||
Error = EcstoreError,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
> + StorageListOperations<
|
||||
Error = EcstoreError,
|
||||
ListObjectsV2Info = CatalogListObjectsV2Info,
|
||||
ListObjectVersionsInfo = CatalogListObjectVersionsInfo,
|
||||
ObjectInfoOrErr = CatalogObjectInfoOrErr,
|
||||
WalkOptions = CatalogWalkOptions,
|
||||
WalkCancellation = tokio_util::sync::CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<CatalogObjectInfoOrErr>,
|
||||
> + StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> TableCatalogStorage for T where
|
||||
T: StorageObjectIO<
|
||||
Error = EcstoreError,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + StorageObjectOperations<
|
||||
Error = EcstoreError,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
> + StorageListOperations<
|
||||
Error = EcstoreError,
|
||||
ListObjectsV2Info = CatalogListObjectsV2Info,
|
||||
ListObjectVersionsInfo = CatalogListObjectVersionsInfo,
|
||||
ObjectInfoOrErr = CatalogObjectInfoOrErr,
|
||||
WalkOptions = CatalogWalkOptions,
|
||||
WalkCancellation = tokio_util::sync::CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<CatalogObjectInfoOrErr>,
|
||||
> + StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
{
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CatalogIdentifierError {
|
||||
Empty,
|
||||
@@ -3487,7 +3554,7 @@ impl<S> Clone for EcStoreTableCatalogObjectBackend<S> {
|
||||
|
||||
impl<S> EcStoreTableCatalogObjectBackend<S>
|
||||
where
|
||||
S: ObjectIO + ObjectOperations + ListOperations + NamespaceLocking,
|
||||
S: TableCatalogStorage,
|
||||
{
|
||||
pub fn new(store: Arc<S>) -> Self {
|
||||
Self { store }
|
||||
@@ -3499,7 +3566,7 @@ pub(crate) type EcStoreTableCatalogStore<S> = ObjectTableCatalogStore<EcStoreTab
|
||||
#[async_trait::async_trait]
|
||||
impl<S> TableCatalogObjectBackend for EcStoreTableCatalogObjectBackend<S>
|
||||
where
|
||||
S: ObjectIO + ObjectOperations + ListOperations + NamespaceLocking,
|
||||
S: TableCatalogStorage,
|
||||
{
|
||||
async fn read_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<Option<TableCatalogObject>> {
|
||||
self.read_object_with_options(bucket, object, ObjectOptions::default()).await
|
||||
@@ -3598,7 +3665,7 @@ where
|
||||
|
||||
impl<S> EcStoreTableCatalogObjectBackend<S>
|
||||
where
|
||||
S: ObjectIO + ObjectOperations + ListOperations + NamespaceLocking,
|
||||
S: TableCatalogStorage,
|
||||
{
|
||||
async fn read_object_with_options(
|
||||
&self,
|
||||
|
||||
@@ -58,6 +58,7 @@ STORE_API_RANGE_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_range_helper_reexpor
|
||||
STORE_API_LIST_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_list_helper_reexports.txt"
|
||||
STORE_API_LIST_RESPONSE_REEXPORTS_FILE="${TMP_DIR}/store_api_list_response_reexports.txt"
|
||||
STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_list_consumer_hits.txt"
|
||||
STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_operation_consumer_hits.txt"
|
||||
STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE="${TMP_DIR}/store_api_object_operation_local_method_hits.txt"
|
||||
|
||||
awk '
|
||||
@@ -309,6 +310,16 @@ if [[ -s "$STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE" ]]; then
|
||||
report_failure "external list response consumers must use rustfs-storage-api contracts: $(paste -sd '; ' "$STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading 'rustfs_ecstore::store_api(?:::\{[^}]*\b(?:ObjectIO|ObjectOperations|ListOperations|MultipartOperations|HealOperations|NamespaceLocking)\b|::(?:ObjectIO|ObjectOperations|ListOperations|MultipartOperations|HealOperations|NamespaceLocking)\b)' \
|
||||
rustfs/src crates --glob '!crates/ecstore/**' || true
|
||||
) >"$STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE"
|
||||
|
||||
if [[ -s "$STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE" ]]; then
|
||||
report_failure "external operation consumers must use rustfs-storage-api traits: $(paste -sd '; ' "$STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading 'async fn (get_object_reader|put_object|get_object_info|verify_object_integrity|copy_object|delete_object_version|delete_object|delete_objects|put_object_metadata|get_object_tags|put_object_tags|delete_object_tags|add_partial|transition_object|restore_transitioned_object|list_multipart_uploads|new_multipart_upload|copy_object_part|put_object_part|get_multipart_info|list_object_parts|abort_multipart_upload|complete_multipart_upload|heal_format|heal_bucket|heal_object|get_pool_and_set|check_abandoned_parts|new_ns_lock)\b' \
|
||||
|
||||
Reference in New Issue
Block a user