mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
refactor: clean remaining storage DTO imports (#3568)
This commit is contained in:
@@ -19,6 +19,7 @@ pub mod manager;
|
||||
pub mod progress;
|
||||
pub mod resume;
|
||||
pub mod storage;
|
||||
mod storage_compat;
|
||||
pub mod task;
|
||||
pub mod utils;
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@ use rustfs_ecstore::{
|
||||
disk::{DiskStore, endpoint::Endpoint},
|
||||
error::StorageError,
|
||||
store::ECStore,
|
||||
store_api::{ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions, PutObjReader as EcstorePutObjReader},
|
||||
};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_storage_api::{
|
||||
@@ -29,6 +28,8 @@ use rustfs_storage_api::{
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
pub use super::storage_compat::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_STORAGE: &str = "storage";
|
||||
const EVENT_HEAL_STORAGE_OBJECT_IO: &str = "heal_storage_object_io";
|
||||
@@ -37,10 +38,6 @@ const EVENT_HEAL_STORAGE_OBJECT_VERIFY: &str = "heal_storage_object_verify";
|
||||
const EVENT_HEAL_STORAGE_ADMIN_OP: &str = "heal_storage_admin_op";
|
||||
const EVENT_HEAL_STORAGE_REPAIR_OP: &str = "heal_storage_repair_op";
|
||||
|
||||
pub type HealObjectInfo = EcstoreObjectInfo;
|
||||
pub type HealObjectOptions = EcstoreObjectOptions;
|
||||
pub type HealPutObjReader = EcstorePutObjReader;
|
||||
|
||||
/// Disk status for heal operations
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum DiskStatus {
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_ecstore::store_api::{
|
||||
ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions, PutObjReader as EcstorePutObjReader,
|
||||
};
|
||||
|
||||
pub type HealObjectInfo = EcstoreObjectInfo;
|
||||
pub type HealObjectOptions = EcstoreObjectOptions;
|
||||
pub type HealPutObjReader = EcstorePutObjReader;
|
||||
@@ -18,11 +18,10 @@ use rustfs_ecstore::{
|
||||
disk::endpoint::Endpoint,
|
||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
store::ECStore,
|
||||
store_api::{ObjectOptions, PutObjReader},
|
||||
};
|
||||
use rustfs_heal::heal::{
|
||||
manager::{HealConfig, HealManager},
|
||||
storage::{ECStoreHealStorage, HealStorageAPI},
|
||||
storage::{ECStoreHealStorage, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI},
|
||||
task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
|
||||
};
|
||||
use rustfs_storage_api::{BucketOperations, ObjectIO as _, ObjectOperations as _};
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub mod object;
|
||||
mod storage_compat;
|
||||
|
||||
use crate::cache::Cache;
|
||||
use crate::error::Result;
|
||||
|
||||
@@ -29,7 +29,6 @@ use rustfs_ecstore::{
|
||||
com::{delete_config, read_config_no_lock, read_config_with_metadata, save_config, save_config_with_opts},
|
||||
},
|
||||
store::ECStore,
|
||||
store_api::{ObjectInfo, ObjectOptions},
|
||||
};
|
||||
use rustfs_io_metrics::record_system_path_failure;
|
||||
use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc};
|
||||
@@ -43,6 +42,8 @@ use tokio::sync::mpsc::{self, Sender};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use super::storage_compat::{IamObjectInfo, IamObjectOptions};
|
||||
|
||||
pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam"));
|
||||
pub static IAM_CONFIG_USERS_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam/users/"));
|
||||
pub static IAM_CONFIG_SERVICE_ACCOUNTS_PREFIX: LazyLock<String> =
|
||||
@@ -60,7 +61,7 @@ pub static IAM_CONFIG_POLICY_DB_SERVICE_ACCOUNTS_PREFIX: LazyLock<String> =
|
||||
pub static IAM_CONFIG_POLICY_DB_GROUPS_PREFIX: LazyLock<String> =
|
||||
LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam/policydb/groups/"));
|
||||
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, EcstoreError>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<IamObjectInfo, EcstoreError>;
|
||||
|
||||
const IAM_IDENTITY_FILE: &str = "identity.json";
|
||||
const IAM_POLICY_FILE: &str = "policy.json";
|
||||
@@ -282,7 +283,7 @@ impl ObjectStore {
|
||||
entry.cooldown_until = Some(Instant::now() + IAM_LAZY_REWRITE_COOLDOWN);
|
||||
}
|
||||
|
||||
fn maybe_schedule_lazy_rewrite(&self, path: &str, outcome: &DecryptOutcome, object_info: &ObjectInfo) {
|
||||
fn maybe_schedule_lazy_rewrite(&self, path: &str, outcome: &DecryptOutcome, object_info: &IamObjectInfo) {
|
||||
if !Self::should_lazy_rewrite(outcome.source) {
|
||||
return;
|
||||
}
|
||||
@@ -322,7 +323,7 @@ impl ObjectStore {
|
||||
async fn lazy_rewrite_iam_config(&self, path: &str, plain: &[u8], etag: &str) -> std::result::Result<(), StorageError> {
|
||||
let encrypted = Self::encrypt_data_with_master_key(plain).map_err(StorageError::other)?;
|
||||
|
||||
let mut opts = ObjectOptions {
|
||||
let mut opts = IamObjectOptions {
|
||||
max_parity: true,
|
||||
..Default::default()
|
||||
};
|
||||
@@ -343,9 +344,9 @@ impl ObjectStore {
|
||||
Self::prepare_data_for_storage(data)
|
||||
}
|
||||
|
||||
async fn load_iamconfig_bytes_with_metadata(&self, path: impl AsRef<str> + Send) -> Result<(Vec<u8>, ObjectInfo)> {
|
||||
async fn load_iamconfig_bytes_with_metadata(&self, path: impl AsRef<str> + Send) -> Result<(Vec<u8>, IamObjectInfo)> {
|
||||
let path_ref = path.as_ref();
|
||||
let (data, obj) = read_config_with_metadata(self.object_api.clone(), path_ref, &ObjectOptions::default()).await?;
|
||||
let (data, obj) = read_config_with_metadata(self.object_api.clone(), path_ref, &IamObjectOptions::default()).await?;
|
||||
let outcome = Self::decrypt_data_with_source(&data)?;
|
||||
self.maybe_schedule_lazy_rewrite(path_ref, &outcome, &obj);
|
||||
|
||||
@@ -640,7 +641,7 @@ impl Store for ObjectStore {
|
||||
}
|
||||
async fn load_iam_config<Item: DeserializeOwned>(&self, path: impl AsRef<str> + Send) -> Result<Item> {
|
||||
let path_ref = path.as_ref();
|
||||
let (data, obj) = read_config_with_metadata(self.object_api.clone(), path_ref, &ObjectOptions::default()).await?;
|
||||
let (data, obj) = read_config_with_metadata(self.object_api.clone(), path_ref, &IamObjectOptions::default()).await?;
|
||||
|
||||
let outcome = match Self::decrypt_data_with_source(&data) {
|
||||
Ok(v) => v,
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_ecstore::store_api::{ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions};
|
||||
|
||||
pub(super) type IamObjectInfo = EcstoreObjectInfo;
|
||||
pub(super) type IamObjectOptions = EcstoreObjectOptions;
|
||||
@@ -19,7 +19,7 @@ use rustfs_s3_types::{EventName, event_schema_version};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use url::form_urlencoded;
|
||||
|
||||
pub type NotifyObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
||||
use crate::storage_compat::NotifyObjectInfo;
|
||||
|
||||
/// Represents the identity of the user who triggered the event
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
|
||||
@@ -36,11 +36,12 @@ mod runtime_facade;
|
||||
mod runtime_view;
|
||||
mod services;
|
||||
mod status_view;
|
||||
mod storage_compat;
|
||||
|
||||
pub use bucket_config_manager::NotifyBucketConfigManager;
|
||||
pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem};
|
||||
pub use error::{LifecycleError, NotificationError};
|
||||
pub use event::{Event, EventArgs, EventArgsBuilder, NotifyObjectInfo};
|
||||
pub use event::{Event, EventArgs, EventArgsBuilder};
|
||||
pub use event_bridge::{LiveEventHistory, NotifyEventBridge};
|
||||
pub use global::{
|
||||
initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot, notification_system,
|
||||
@@ -54,3 +55,4 @@ pub use runtime_facade::NotifyRuntimeFacade;
|
||||
pub use runtime_view::NotifyRuntimeView;
|
||||
pub use services::NotifyServices;
|
||||
pub use status_view::NotifyStatusView;
|
||||
pub use storage_compat::NotifyObjectInfo;
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub type NotifyObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
||||
@@ -51,6 +51,7 @@ pub mod ratelimit;
|
||||
pub mod router;
|
||||
pub mod slo;
|
||||
pub mod staticweb;
|
||||
pub mod storage_compat;
|
||||
pub mod symlink;
|
||||
pub mod sync;
|
||||
pub mod tempurl;
|
||||
|
||||
@@ -55,25 +55,18 @@ use super::{SwiftError, SwiftResult};
|
||||
use axum::http::HeaderMap;
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_ecstore::resolve_object_store_handle;
|
||||
use rustfs_ecstore::store_api::{
|
||||
GetObjectReader as EcstoreGetObjectReader, ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions,
|
||||
PutObjReader as EcstorePutObjReader,
|
||||
};
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_storage_api::{BucketOperations, BucketOptions, ObjectIO as _, ObjectOperations as _};
|
||||
use std::collections::HashMap;
|
||||
use tracing::debug;
|
||||
use tracing::error;
|
||||
|
||||
pub use super::storage_compat::{SwiftGetObjectReader, SwiftObjectInfo, SwiftObjectOptions, SwiftPutObjReader};
|
||||
|
||||
const LOG_COMPONENT_PROTOCOLS: &str = "protocols";
|
||||
const LOG_SUBSYSTEM_SWIFT_OBJECT: &str = "swift_object";
|
||||
const EVENT_SWIFT_OBJECT_STORAGE_STATE: &str = "swift_object_storage_state";
|
||||
|
||||
pub type SwiftGetObjectReader = EcstoreGetObjectReader;
|
||||
pub type SwiftObjectInfo = EcstoreObjectInfo;
|
||||
pub type SwiftObjectOptions = EcstoreObjectOptions;
|
||||
pub type SwiftPutObjReader = EcstorePutObjReader;
|
||||
|
||||
/// Maximum number of metadata headers allowed per object (Swift standard)
|
||||
const MAX_METADATA_COUNT: usize = 90;
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_ecstore::store_api::{
|
||||
GetObjectReader as EcstoreGetObjectReader, ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions,
|
||||
PutObjReader as EcstorePutObjReader,
|
||||
};
|
||||
|
||||
pub type SwiftGetObjectReader = EcstoreGetObjectReader;
|
||||
pub type SwiftObjectInfo = EcstoreObjectInfo;
|
||||
pub type SwiftObjectOptions = EcstoreObjectOptions;
|
||||
pub type SwiftPutObjReader = EcstorePutObjReader;
|
||||
@@ -19,6 +19,7 @@ use std::fmt::Display;
|
||||
pub mod object_store;
|
||||
pub mod query;
|
||||
pub mod server;
|
||||
mod storage_compat;
|
||||
|
||||
#[cfg(test)]
|
||||
mod test;
|
||||
|
||||
@@ -29,9 +29,6 @@ use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object
|
||||
use rustfs_ecstore::resolve_object_store_handle;
|
||||
use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||
use rustfs_ecstore::store::ECStore;
|
||||
use rustfs_ecstore::store_api::{
|
||||
GetObjectReader as EcstoreGetObjectReader, ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions,
|
||||
};
|
||||
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
|
||||
use s3s::S3Result;
|
||||
use s3s::dto::SelectObjectContentInput;
|
||||
@@ -51,6 +48,8 @@ use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio_util::io::ReaderStream;
|
||||
use transform_stream::AsyncTryStream;
|
||||
|
||||
use crate::storage_compat::{SelectGetObjectReader, SelectObjectInfo, SelectObjectOptions};
|
||||
|
||||
/// Maximum allowed object size for JSON DOCUMENT mode.
|
||||
///
|
||||
/// JSON DOCUMENT format requires loading the entire file into memory for DOM
|
||||
@@ -68,10 +67,6 @@ pub const MAX_JSON_DOCUMENT_BYTES: u64 = 128 * 1024 * 1024;
|
||||
pub const INVALID_SCAN_RANGE_MESSAGE: &str =
|
||||
"The value of a parameter in ScanRange element is invalid. Check the service API documentation and try again.";
|
||||
|
||||
type SelectGetObjectReader = EcstoreGetObjectReader;
|
||||
type SelectObjectInfo = EcstoreObjectInfo;
|
||||
type SelectObjectOptions = EcstoreObjectOptions;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct EcObjectStore {
|
||||
input: Arc<SelectObjectContentInput>,
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_ecstore::store_api::{
|
||||
GetObjectReader as EcstoreGetObjectReader, ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions,
|
||||
};
|
||||
|
||||
pub(crate) type SelectGetObjectReader = EcstoreGetObjectReader;
|
||||
pub(crate) type SelectObjectInfo = EcstoreObjectInfo;
|
||||
pub(crate) type SelectObjectOptions = EcstoreObjectOptions;
|
||||
@@ -33,13 +33,16 @@ use rustfs_ecstore::{
|
||||
config::{com::save_config, storageclass},
|
||||
disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
|
||||
error::{Error, Result as StorageResult, StorageError},
|
||||
store_api::{GetObjectReader, ObjectInfo, ObjectOptions, ObjectToDelete, PutObjReader},
|
||||
};
|
||||
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO};
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
use tracing::warn;
|
||||
|
||||
pub use crate::storage_compat::{
|
||||
ScannerGetObjectReader, ScannerObjectIO, ScannerObjectInfo, ScannerObjectOptions, ScannerObjectToDelete, ScannerPutObjReader,
|
||||
};
|
||||
use crate::storage_compat::{ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions};
|
||||
|
||||
// Data usage constants
|
||||
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
|
||||
|
||||
@@ -61,38 +64,6 @@ const EVENT_SCANNER_CACHE_LOAD_STATE: &str = "scanner_cache_load_state";
|
||||
const EVENT_SCANNER_CACHE_SAVE_STATE: &str = "scanner_cache_save_state";
|
||||
static CACHE_SAVE_METRICS_ONCE: Once = Once::new();
|
||||
|
||||
pub type ScannerGetObjectReader = GetObjectReader;
|
||||
pub type ScannerObjectInfo = ObjectInfo;
|
||||
pub type ScannerObjectOptions = ObjectOptions;
|
||||
pub type ScannerObjectToDelete = ObjectToDelete;
|
||||
pub type ScannerPutObjReader = PutObjReader;
|
||||
|
||||
pub trait ScannerObjectIO:
|
||||
ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> ScannerObjectIO for T where
|
||||
T: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>
|
||||
{
|
||||
}
|
||||
|
||||
pub const DATA_USAGE_SCAN_CHECKPOINT_VERSION: u16 = 1;
|
||||
|
||||
// Data usage paths (computed at runtime)
|
||||
|
||||
@@ -28,6 +28,7 @@ pub mod scanner_budget;
|
||||
pub mod scanner_folder;
|
||||
pub mod scanner_io;
|
||||
pub mod sleeper;
|
||||
mod storage_compat;
|
||||
|
||||
pub use data_usage_define::*;
|
||||
pub use error::ScannerError;
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use http::HeaderMap;
|
||||
use rustfs_ecstore::{
|
||||
error::Error,
|
||||
store_api::{
|
||||
GetObjectReader as EcstoreGetObjectReader, ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions,
|
||||
ObjectToDelete as EcstoreObjectToDelete, PutObjReader as EcstorePutObjReader,
|
||||
},
|
||||
};
|
||||
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO};
|
||||
|
||||
pub type ScannerGetObjectReader = EcstoreGetObjectReader;
|
||||
pub type ScannerObjectInfo = EcstoreObjectInfo;
|
||||
pub type ScannerObjectOptions = EcstoreObjectOptions;
|
||||
pub type ScannerObjectToDelete = EcstoreObjectToDelete;
|
||||
pub type ScannerPutObjReader = EcstorePutObjReader;
|
||||
|
||||
pub trait ScannerObjectIO:
|
||||
ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ScannerObjectOptions,
|
||||
ObjectInfo = ScannerObjectInfo,
|
||||
GetObjectReader = ScannerGetObjectReader,
|
||||
PutObjectReader = ScannerPutObjReader,
|
||||
>
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> ScannerObjectIO for T where
|
||||
T: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ScannerObjectOptions,
|
||||
ObjectInfo = ScannerObjectInfo,
|
||||
GetObjectReader = ScannerGetObjectReader,
|
||||
PutObjectReader = ScannerPutObjReader,
|
||||
>
|
||||
{
|
||||
}
|
||||
@@ -800,6 +800,30 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
diff hygiene, direct import scan, Rust risk scan, full pre-commit, and
|
||||
required three-expert review passed.
|
||||
|
||||
- [x] `API-027` Clean remaining external storage DTO imports.
|
||||
- Current branch: `overtrue/arch-storage-compat-contract-cleanup`.
|
||||
- Completed slice: move table catalog, IAM object-store, admin zip-download,
|
||||
capacity dirty-scope tests, heal integration tests, scanner, Swift, S3
|
||||
Select, and notify event payloads from raw ECStore `store_api` DTO imports
|
||||
to crate-local compatibility aliases/modules.
|
||||
- Acceptance: non-ECStore direct `rustfs_ecstore::store_api` references are
|
||||
limited to explicit boundary alias points in RustFS storage plus scanner,
|
||||
heal, IAM, notify, Swift, and S3 Select compatibility modules; table
|
||||
catalog, affected tests, and protocol/scanner/notification consumers
|
||||
consume those boundary names instead of raw ECStore DTO paths.
|
||||
- Must preserve: table catalog storage trait bindings, IAM metadata/lazy
|
||||
rewrite behavior, object zip preflight/read semantics, capacity dirty-disk
|
||||
assertions, heal integration object read/write behavior, scanner cache
|
||||
load/save semantics, Swift object read/write/copy/delete behavior, S3
|
||||
Select object-store reads, notify event payload shape, and ECStore-owned DTO
|
||||
concrete shapes.
|
||||
- Risk defense: this slice changes import ownership and type aliases only; it
|
||||
does not move DTO definitions, alter serialization, change object-store
|
||||
implementation bodies, or adjust runtime control flow.
|
||||
- Verification: focused compile/tests, migration/layer guards, formatting,
|
||||
diff hygiene, direct import scan, Rust risk scan, full pre-commit, and
|
||||
required three-expert review passed.
|
||||
|
||||
## Phase 8 Background Controller Tasks
|
||||
|
||||
- [x] `BGC-001` Inventory background services.
|
||||
@@ -1076,70 +1100,54 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
|
||||
| Expert | Status | Notes |
|
||||
|---|---|---|
|
||||
| Quality/architecture | passed | Scanner, heal, notify, Swift, S3 Select, and RustFS storage/app consumers now use crate-local DTO aliases; the erasure CI fix is isolated to zero-length shard reconstruction. |
|
||||
| Migration preservation | passed | ECStore remains the owner of object DTOs/readers/walk filters/implementation behavior; aliases preserve concrete types and non-empty erasure encode/decode paths still use the existing encoders. |
|
||||
| Testing/verification | passed | Focused DTO/erasure compile/tests, migration/layer guards, direct import scan, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` passed. |
|
||||
| Quality/architecture | passed | Table catalog, IAM, admin zip, capacity, heal integration, scanner, Swift, S3 Select, and notify consumers now use local DTO compatibility aliases instead of raw ECStore `store_api` imports. |
|
||||
| Migration preservation | passed | ECStore remains the owner of object DTO/readers; alias boundaries preserve concrete associated types for storage traits, IAM config reads, zip preflight reads, scanner cache I/O, Swift/S3 Select reads, notify payloads, capacity assertions, and heal integration writes. |
|
||||
| Testing/verification | passed | Focused compile/tests, migration/layer guards, direct import scan, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` passed. |
|
||||
|
||||
## Verification Notes
|
||||
|
||||
Passed before push:
|
||||
|
||||
- `cargo check --tests -p rustfs -p rustfs-scanner -p rustfs-heal -p
|
||||
rustfs-protocols -p rustfs-notify -p rustfs-s3select-api`: passed.
|
||||
- `cargo test -p rustfs-scanner -p rustfs-heal -p rustfs-notify -p
|
||||
rustfs-protocols -p rustfs-s3select-api`: passed.
|
||||
- `cargo test -p rustfs --lib app::object_usecase::tests`: passed; 85 passed,
|
||||
2 ignored.
|
||||
- `cargo test -p rustfs --lib app::bucket_usecase::tests`: passed; 64 passed.
|
||||
- `cargo test -p rustfs --lib app::multipart_usecase::tests`: passed; 24
|
||||
passed, 4 ignored.
|
||||
- `cargo test -p rustfs --lib storage::s3_api::bucket::tests`: passed; 22
|
||||
- `cargo check --tests -p rustfs -p rustfs-iam -p rustfs-heal`: passed.
|
||||
- `cargo check --tests -p rustfs-iam -p rustfs-heal -p rustfs-scanner -p
|
||||
rustfs-protocols -p rustfs-s3select-api -p rustfs-notify`: passed.
|
||||
- `cargo test -p rustfs --lib table_catalog_store_trait`: passed; 2 passed.
|
||||
- `cargo test -p rustfs --lib admin::handlers::object_zip_download::tests`:
|
||||
passed; 28 passed.
|
||||
- `cargo test -p rustfs --lib app::capacity_dirty_scope_test`: passed; 2
|
||||
passed.
|
||||
- `cargo test -p rustfs --lib storage::ecfs_test::tests`: passed; 59 passed,
|
||||
14 ignored.
|
||||
- `rg -n 'rustfs_ecstore::store_api' rustfs/src crates --glob
|
||||
- `cargo test -p rustfs-iam store::object::tests`: passed; 8 passed.
|
||||
- `cargo test -p rustfs-heal --test heal_integration_test`: passed; 5 passed.
|
||||
- `cargo check --tests -p rustfs-scanner -p rustfs-protocols -p
|
||||
rustfs-s3select-api -p rustfs-notify`: passed.
|
||||
- `cargo test -p rustfs-scanner -p rustfs-protocols -p rustfs-s3select-api -p
|
||||
rustfs-notify`: passed; notify 82 passed, protocols 13 passed, S3 Select 49
|
||||
passed, scanner 151 passed plus lifecycle integration 1 passed and 14
|
||||
ignored.
|
||||
- `rg -n 'rustfs_ecstore::store_api|store_api::\{' rustfs/src crates --glob
|
||||
'!crates/ecstore/**' --glob '*.rs'`: remaining matches are deliberate
|
||||
boundary alias definitions.
|
||||
- `cargo test -p rustfs-ecstore erasure_coding::erasure::tests`: passed; 31
|
||||
passed.
|
||||
- `PROPTEST_CASES=1024 cargo test -p rustfs-ecstore
|
||||
erasure_coding::erasure::tests::decode_data_and_parity_round_trips_bounded_recoverability`:
|
||||
passed.
|
||||
- `cargo test -p rustfs-ecstore
|
||||
rpc::peer_s3_client::tests::local_get_bucket_info_survives_prior_walk_timeout`:
|
||||
passed after a full-crate `cargo test -p rustfs-ecstore` run exposed this
|
||||
unrelated ordinary-harness global-state interference.
|
||||
boundary alias definitions in RustFS storage and crate-local compat modules.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- Rust risk scan: new hits are crate-local alias casts plus the empty-erasure
|
||||
test expectations; no new production `unwrap`/`expect`, panic/todo markers,
|
||||
`unsafe`, process-spawning calls, println/eprintln, or relaxed ordering in
|
||||
added Rust lines.
|
||||
- `make pre-commit`: passed; nextest reported 6218 tests passed and 111
|
||||
skipped, and doctests passed.
|
||||
- Rust risk scan: new hits are crate-local alias casts only; no new
|
||||
`unwrap`/`expect`, panic/todo markers, `unsafe`, process-spawning calls,
|
||||
println/eprintln, or relaxed ordering in added Rust lines.
|
||||
- `make pre-commit`: passed.
|
||||
|
||||
Notes:
|
||||
|
||||
- This slice follows the external operation consumer cleanup and keeps the old
|
||||
aggregate facade, DTO/helper, response, and operation contract guards active.
|
||||
- External DTO consumers now name ECStore-owned object DTOs through local
|
||||
semantic aliases in their owning crates or RustFS storage boundary module.
|
||||
- The slice does not alter object metadata shape, options defaults,
|
||||
reader/writer behavior, delete replication DTO handling, scanner cache
|
||||
semantics, heal storage metadata semantics, Swift/S3 Select object reads,
|
||||
notification event payloads, S3 response DTO mapping, or runtime storage
|
||||
behavior.
|
||||
- The CI follow-up preserves empty-object erasure reconstruction by filling
|
||||
missing zero-length shards before calling encoders that reject zero-byte SIMD
|
||||
shard sizes.
|
||||
- This slice is stacked on API-026 while `rustfs/rustfs#3566` is pending.
|
||||
- Direct ECStore `store_api` references outside ECStore now remain only at
|
||||
explicit alias boundary points.
|
||||
- The slice does not alter table catalog storage behavior, IAM config storage,
|
||||
admin zip download object reads, capacity dirty-disk behavior, heal
|
||||
integration semantics, scanner cache I/O, Swift/S3 Select object reads,
|
||||
notification event payloads, or ECStore DTO definitions.
|
||||
|
||||
## Handoff Notes
|
||||
|
||||
- External DTO consumer cleanup is based on `main` after
|
||||
`rustfs/rustfs#3565` merged and carries the empty-object erasure recovery
|
||||
CI follow-up because the merged PR did not include that post-merge fix.
|
||||
- Continue with larger consumer-migration batches; keep ECStore-owned
|
||||
DTOs/readers/walk filters in ECStore until their concrete behavior is isolated
|
||||
enough for a pure-move slice.
|
||||
|
||||
@@ -31,7 +31,7 @@ use matchit::Params;
|
||||
use rand::RngExt;
|
||||
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
|
||||
use rustfs_credentials::get_global_action_cred;
|
||||
use rustfs_ecstore::{global::get_global_region, store_api::ObjectOptions};
|
||||
use rustfs_ecstore::global::get_global_region;
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_storage_api::{BucketOperations, ListOperations as _, ObjectIO as _, ObjectOperations as _, bucket::BucketOptions};
|
||||
use rustfs_trusted_proxies::{ClientInfo, ValidationMode};
|
||||
@@ -49,6 +49,8 @@ use tokio_util::io::ReaderStream;
|
||||
use url::form_urlencoded;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::storage::StorageObjectOptions as ObjectOptions;
|
||||
|
||||
const OBJECT_ZIP_DOWNLOAD_TOKEN_TTL: Duration = Duration::minutes(5);
|
||||
const ZIP_STREAM_BUFFER_SIZE: usize = 1024 * 1024;
|
||||
const ZIP_OBJECT_BUFFER_SIZE: usize = 128 * 1024;
|
||||
|
||||
@@ -18,7 +18,6 @@ use rustfs_ecstore::{
|
||||
disk::endpoint::Endpoint,
|
||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
store::ECStore,
|
||||
store_api::{ObjectOptions, PutObjReader},
|
||||
};
|
||||
use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager};
|
||||
use rustfs_storage_api::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _};
|
||||
@@ -35,6 +34,8 @@ use tokio::fs;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::storage::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader};
|
||||
|
||||
static CAPACITY_DIRTY_SCOPE_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>, TempDir)> = OnceLock::new();
|
||||
static CAPACITY_DIRTY_SCOPE_INIT: Once = Once::new();
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ pub mod timeout_wrapper;
|
||||
pub mod tonic_service;
|
||||
|
||||
pub(crate) type StorageDeletedObject = rustfs_ecstore::store_api::DeletedObject;
|
||||
pub(crate) type StorageGetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
|
||||
pub(crate) type StorageObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
||||
pub(crate) type StorageObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
|
||||
pub(crate) type StorageObjectToDelete = rustfs_ecstore::store_api::ObjectToDelete;
|
||||
|
||||
@@ -43,10 +43,7 @@ use rustfs_ecstore::bucket::{
|
||||
};
|
||||
use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
|
||||
use rustfs_ecstore::error::{Error as EcstoreError, StorageError};
|
||||
use rustfs_ecstore::{
|
||||
set_disk::get_lock_acquire_timeout,
|
||||
store_api::{DeletedObject, GetObjectReader, ObjectInfo, ObjectOptions, ObjectToDelete, PutObjReader},
|
||||
};
|
||||
use rustfs_ecstore::set_disk::get_lock_acquire_timeout;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_storage_api::{
|
||||
HTTPPreconditions, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
|
||||
@@ -59,6 +56,11 @@ use time::{Duration, OffsetDateTime};
|
||||
use tokio::io::AsyncReadExt;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::storage::{
|
||||
StorageDeletedObject as DeletedObject, StorageGetObjectReader as GetObjectReader, StorageObjectInfo as ObjectInfo,
|
||||
StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader,
|
||||
};
|
||||
|
||||
pub(crate) const TABLE_BUCKET_MARKER_CONFIG: &str = BUCKET_TABLE_CONFIG;
|
||||
pub(crate) const RESERVED_CATALOG_OBJECT_MESSAGE: &str = "Object key is reserved for the table catalog";
|
||||
pub(crate) const TABLE_BUCKET_CATALOG_TYPE: &str = "iceberg-rest";
|
||||
|
||||
Reference in New Issue
Block a user