From dda841d8dee6c7552beef1a08b062c0684654f1b Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Thu, 27 Aug 2026 08:01:18 +0800 Subject: [PATCH] refactor(ecstore): retire set_disk lint blankets via explicit imports (#6697) refactor(ecstore): retire the set_disk lint blankets by making the prelude explicit backlog#1823 step 1 / backlog#2029 road 2. Removes the last two module-level lint blankets in ecstore: set_disk/mod.rs #![allow(unused_imports)] and #![allow(unused_variables)], restoring both lints for the whole 40K-line subtree, and deletes the register line for the unused_variables blanket in the same diff (the guard from #6155 is a bidirectional exact match). The unused_imports blanket existed because 14 submodules consumed mod.rs as a glob prelude (use super::* / use super::super::*), and rustc does not track consumption through glob re-exports. Each glob is now an explicit use super::{...} list, keeping mod.rs as the single import hub while making every import lint-checkable. Names consumed only by test or test-util units carry #[cfg(test)] / #[cfg(all(test, feature = "test-util"))] / #[cfg(any(test, feature = "test-util"))] gates matching their consumers; storage-api traits are routed through the storage_api_contracts facade per the architecture guard. The sweep then deleted the genuinely dead imports the blanket was hiding (chrono::Utc, glob::Pattern, futures::task::AtomicWaker, rustfs_lock LocalLock, AsyncBatchProcessor, rand::Rng, std::future::Future among others in mod.rs, plus stale scoped imports and one empty test module shell across the subtree). One unused_variables finding surfaced: flush_read_version_coalescer_pending's lane_key is read only by the #[cfg(test)] counter block, handled with the cfg(not(test)) let _ pattern established in #6158. Verification: cargo check zero warnings versus the 9cf276ed2 baseline on five lanes (default lib / --tests / rio-v2 --tests / test-util --tests / test-util,rio-v2 --tests; the --tests lane keeps the same three pre-existing core/pools.rs and store/object.rs dead-code warnings main already has); clippy --lib --tests -D warnings clean with test-util,rio-v2; cargo nextest run 4567 passed; make pre-commit exit 0. --- .../src/set_disk/core/io_primitives.rs | 84 ++++++--- crates/ecstore/src/set_disk/ctx.rs | 2 +- crates/ecstore/src/set_disk/metadata.rs | 13 +- crates/ecstore/src/set_disk/mod.rs | 76 ++++---- .../src/set_disk/ops/bitrot_self_verify.rs | 12 +- crates/ecstore/src/set_disk/ops/bucket.rs | 7 +- crates/ecstore/src/set_disk/ops/heal.rs | 13 +- crates/ecstore/src/set_disk/ops/heal_walk.rs | 8 +- crates/ecstore/src/set_disk/ops/list.rs | 6 +- crates/ecstore/src/set_disk/ops/locking.rs | 12 +- crates/ecstore/src/set_disk/ops/multipart.rs | 69 ++++++- crates/ecstore/src/set_disk/ops/object.rs | 134 ++++++++++---- crates/ecstore/src/set_disk/read.rs | 170 +++++++++++++++--- crates/ecstore/src/set_disk/replication.rs | 7 +- .../src/set_disk/transition_matrix_tests.rs | 6 +- scripts/ecstore-module-lint-register.txt | 1 - 16 files changed, 477 insertions(+), 143 deletions(-) diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index a05884f72..30b221256 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -22,20 +22,47 @@ //! byte-identical to the pre-move sources; only the module header //! (`use super::*;` -> `use super::super::*;`) and item visibility change. -use super::super::*; +#[cfg(test)] +use super::super::ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE; +#[cfg(test)] +use super::super::ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT; +#[cfg(test)] +use super::super::ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE; +#[cfg(test)] +use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET; +#[cfg(test)] +use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS; +#[cfg(test)] +use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS; +#[cfg(test)] +use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX; +#[cfg(test)] +use super::super::get_metadata_slowtail_fault_delay; +use super::super::{ + Bytes, CHECK_PART_DISK_NOT_FOUND, DeleteOptions, DiskError, DiskStore, EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED, + EVENT_SET_DISK_WRITE, Error, FileInfo, FileMeta, FileMetaShallowVersion, GetCodecStreamingFallbackReason, + GetObjectMetadataCacheEntry, HTTPPreconditions, HashAlgorithm, HealAdmissionResult, HealChannelPriority, HealRequestSource, + LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, MultipartWriteQuorumContext, OBJECT_OP_IGNORED_ERRS, ObjectOptions, + ObjectPartInfo, OffsetDateTime, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RawFileInfo, ReadMultipleReq, + ReadMultipleResp, ReadOptions, Result, SLASH_SEPARATOR, STORAGE_FORMAT_FILE, SetDisks, SnapshotLeaseToken, StorageError, + UpdateMetadataOpts, Uuid, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct, + capacity_scope_from_disks, coding, collect_inline_data_shard_fileinfos_by_index_or_reason, current_dirty_generation, debug, + disk, file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info, inline_erasure_shard_file_offset, + inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found, is_get_metadata_data_read_early_stop_enabled, + is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled, is_object_dangling, + is_version_early_stop_enabled, issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure, + merge_file_meta_versions, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs, reduce_write_quorum_errs, + send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn, +}; +#[cfg(test)] +use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; +#[cfg(test)] +use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH; +#[cfg(test)] +use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD; use crate::diagnostics::get::{ - GET_DIRECT_MEMORY_SUBPATH_DISK_DATA_BLOCKS, GET_DIRECT_MEMORY_SUBPATH_INLINE_BUFFERED, GET_METADATA_CACHE_DECISION_HIT, - GET_METADATA_CACHE_DECISION_MISS, GET_METADATA_CACHE_DECISION_REJECT, GET_METADATA_CACHE_DECISION_SKIP, - GET_METADATA_CACHE_REASON_DATA_MOVEMENT, GET_METADATA_CACHE_REASON_DELETE_MARKER, GET_METADATA_CACHE_REASON_DIST_ERASURE, - GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS, GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM, - GET_METADATA_CACHE_REASON_META_BUCKET, GET_METADATA_CACHE_REASON_NO_LOCK, GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED, - GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER, - GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, - GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED, GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, - GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH, - GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, @@ -45,16 +72,26 @@ use crate::diagnostics::get::{ GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID, - GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY, - GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, - GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, - GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, - GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, - GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure, - record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled, + GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META, + GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, + GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, + GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled, record_get_stage_duration_if_enabled, }; -use crate::disk::disk_store::{DiskStoreRenameDataExt, get_drive_metadata_timeout}; +#[cfg(test)] +use crate::disk::CHECK_PART_FILE_NOT_FOUND; +use crate::disk::DiskAPI; +#[cfg(test)] +use crate::disk::DiskOption; +#[cfg(test)] +use crate::disk::RUSTFS_META_TMP_BUCKET; +use crate::disk::disk_store::get_drive_metadata_timeout; +#[cfg(test)] +use crate::disk::endpoint::Endpoint; +#[cfg(test)] +use crate::disk::format::FormatV3; use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX; +#[cfg(test)] +use crate::disk::new_disk; use crate::disk::{ BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, @@ -65,9 +102,11 @@ use crate::io_support::bitrot::ShardReader; use crate::io_support::bitrot::{ BitrotReaderStageMetrics, DeferredReaderStripeHandle, adjust_shard_read_params, create_bitrot_reader_from_bytes_with_stage_metrics, create_deferred_bitrot_reader_with_stripe_handle, - object_mmap_read_enabled, object_mmap_read_max_length, + object_mmap_read_max_length, }; +use crate::set_disk::runtime_sources; use crate::set_disk::shard_source::ShardReadCost; +use crate::storage_api_contracts::object::ObjectOperations; use futures::FutureExt as _; use futures::stream::{FuturesUnordered, StreamExt}; use metrics::counter; @@ -281,6 +320,9 @@ async fn flush_read_version_coalescer_pending( return; } + // Only the #[cfg(test)] counter-recording block below reads this. + #[cfg(not(test))] + let _ = lane_key; #[cfg(test)] { let mut observed_paths = HashSet::new(); @@ -6762,7 +6804,7 @@ mod tests { use super::*; use std::io::Cursor; use tempfile::TempDir; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::io::AsyncReadExt; #[test] fn write_precondition_lookup_errors_fail_closed_unless_absence_is_known() { diff --git a/crates/ecstore/src/set_disk/ctx.rs b/crates/ecstore/src/set_disk/ctx.rs index 3b7f681b3..a2af75199 100644 --- a/crates/ecstore/src/set_disk/ctx.rs +++ b/crates/ecstore/src/set_disk/ctx.rs @@ -23,7 +23,7 @@ //! This module only establishes the borrow handle. It moves no trait impl and //! changes no runtime behavior. -use super::*; +use super::{Arc, DiskStore, Endpoint, FormatV3, LockClient, RwLock, SetDisks}; /// Lightweight, `Copy` handle borrowing the shared [`SetDisks`] core state. /// diff --git a/crates/ecstore/src/set_disk/metadata.rs b/crates/ecstore/src/set_disk/metadata.rs index 57a51f861..8337b8559 100644 --- a/crates/ecstore/src/set_disk/metadata.rs +++ b/crates/ecstore/src/set_disk/metadata.rs @@ -12,8 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::*; +use super::{ + Bytes, DATA_MOVEMENT_MULTIPART_PREFIX, DiskError, DiskStore, FileInfo, HashMap, HashSet, OBJECT_OP_IGNORED_ERRS, ObjProps, + OffsetDateTime, SetDisks, Sha256, TRANSITION_COMPLETE, Uuid, debug, disk, error, file_info_is_valid_for_metadata, hex, + reduce_read_quorum_errs, warn, +}; +#[cfg(test)] +use crate::disk::DiskOption; +#[cfg(test)] +use crate::disk::endpoint::Endpoint; +#[cfg(test)] +use crate::disk::new_disk; use rustfs_utils::http; +use sha2::Digest; #[derive(Clone, Copy)] struct FileInfoIdentityGroup { diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 0358ae613..83398c02e 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -38,23 +38,15 @@ //! read primitives it drives. //! - `metadata.rs`, `replication.rs`, `shard_source.rs` — supporting helpers. -// #730: SetDisks still hosts staged read/heal/write migration helpers. -#![allow(unused_imports)] -#![allow(unused_variables)] - use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; use crate::bucket::metadata_sys; use crate::bucket::metadata_sys::ObjectLockConfigState; use crate::bucket::object_lock::objectlock_sys::{ - check_object_lock_for_deletion_with_default_retention, check_object_lock_for_deletion_with_state, - check_retention_for_modification, replication_write_may_pass_worm_gate, + check_object_lock_for_deletion_with_state, check_retention_for_modification, replication_write_may_pass_worm_gate, }; -use crate::bucket::replication::{ - ReplicateDecision, ReplicationObjectBridge, ReplicationState, ReplicationStatusType, VersionPurgeStatusType, - replication_state_to_filemeta, -}; -use crate::bucket::versioning::VersioningApi; -use crate::bucket::versioning_sys::BucketVersioningSys; +#[cfg(test)] +use crate::bucket::replication::ReplicationState; +use crate::bucket::replication::{ReplicateDecision, ReplicationObjectBridge, ReplicationStatusType, VersionPurgeStatusType}; use crate::cluster::rpc::heal_bucket_local_on_disks; use crate::data_usage::record_compression_total_memory; use crate::diagnostics::get::{ @@ -73,9 +65,11 @@ use crate::disk::error_reduce::{ BUCKET_OP_IGNORED_ERRS, OBJECT_OP_IGNORED_ERRS, build_write_quorum_failure_summary, count_errs, reduce_read_quorum_errs, reduce_write_quorum_errs, }; +#[cfg(test)] +use crate::disk::has_part_err; use crate::disk::{ self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, - conv_part_err_to_int, has_part_err, + conv_part_err_to_int, }; use crate::disk::{STORAGE_FORMAT_FILE, count_part_not_success}; use crate::erasure::codec::bridge::{ @@ -90,15 +84,14 @@ use crate::object_api::get_object_body_cache_hook; use crate::object_api::object_api_utils::get_raw_etag; use crate::runtime::instance::{InstanceContext, bootstrap_ctx}; use crate::runtime::sources as runtime_sources; -use crate::services::batch_processor::AsyncBatchProcessor; +#[cfg(test)] +use crate::storage_api_contracts::multipart::MultipartOperations; use crate::storage_api_contracts::{ bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions}, list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions}, - multipart::{ - CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations as _, MultipartUploadResult, PartInfo, - }, + multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo}, namespace::NamespaceLocking as _, - object::{DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete}, + object::{DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectToDelete}, range::HTTPRangeSpec, }; use crate::store::utils::is_reserved_or_invalid_bucket; @@ -109,7 +102,7 @@ use crate::{ disk::{ CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, FileInfoVersions, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, - SnapshotLeaseToken, UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk, + SnapshotLeaseToken, UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, }, error::{StorageError, to_object_err}, object_api::{GetObjectReader, NamespaceLockFence, ObjectInfo, ObjectLockConfigSnapshot, PutObjReader}, @@ -122,18 +115,14 @@ use crate::{ }; use bytes::Bytes; use bytesize::ByteSize; -use chrono::Utc; use futures::future::join_all; -use futures::task::AtomicWaker; -use glob::Pattern; use http::HeaderMap; use md5::{Digest as Md5Digest, Md5}; -use rand::{Rng, seq::SliceRandom}; use regex::Regex; use rustfs_config::MI_B; use rustfs_filemeta::{ - FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ObjectPartInfo, - RawFileInfo, file_info_from_raw, merge_file_meta_versions, + FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, ObjectPartInfo, RawFileInfo, + merge_file_meta_versions, }; use rustfs_heal_contracts::heal_channel::{ DriveState, HealAdmissionResult, HealChannelPriority, HealItemType, HealOpts, HealRequestSource, HealScanMode, @@ -144,9 +133,10 @@ use rustfs_io_metrics::{ record_object_lock_diag_slow_acquire, record_object_lock_diag_slow_hold, }; use rustfs_lock::LockClient; +#[cfg(test)] +use rustfs_lock::LockManager; use rustfs_lock::fast_lock::types::LockResult; -use rustfs_lock::local_lock::LocalLock; -use rustfs_lock::{FastLockGuard, LockManager, NamespaceLock, NamespaceLockGuard, NamespaceLockWrapper, ObjectKey}; +use rustfs_lock::{FastLockGuard, NamespaceLock, NamespaceLockGuard, NamespaceLockWrapper, ObjectKey}; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos}; use rustfs_object_capacity::capacity_scope::{ CapacityScope, CapacityScopeDisk, current_dirty_generation, record_capacity_scope, record_global_dirty_scope, @@ -158,7 +148,7 @@ use rustfs_utils::http::SSEC_ALGORITHM_HEADER; use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING; use rustfs_utils::http::headers::AMZ_STORAGE_CLASS; use rustfs_utils::http::headers::{ - CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _, + CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, }; use rustfs_utils::http::{ SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, @@ -171,30 +161,29 @@ use rustfs_utils::{ path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf}, }; use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_RESTORE}; -use sha2::{Digest, Sha256}; -use std::future::Future; +use sha2::Sha256; use std::hash::{BuildHasher, Hash, Hasher}; use std::mem::{self}; use std::pin::Pin; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, OnceLock}; use std::task::{Context, Poll}; use std::time::{Instant, SystemTime, UNIX_EPOCH}; use std::{ collections::{HashMap, HashSet}, - io::{Cursor, Write}, + io::Cursor, path::Path, time::Duration, }; use time::OffsetDateTime; +#[cfg(test)] +use tokio::sync::mpsc; +use tokio::sync::mpsc::Sender; +#[cfg(test)] +use tokio::time::timeout; use tokio::{ - io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader, ReadBuf}, - sync::{RwLock, broadcast}, -}; -use tokio::{ - select, - sync::mpsc::{self, Sender}, - time::{interval, timeout}, + io::{AsyncRead, AsyncWrite, BufReader, ReadBuf}, + sync::RwLock, }; use tokio_util::sync::CancellationToken; use tracing::error; @@ -347,7 +336,7 @@ const ENV_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: &str = "RUSTFS_MULTIP const DEFAULT_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: usize = 128 * 1024 * 1024; static CACHED_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: OnceLock = OnceLock::new(); -use crate::io_support::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _}; +use crate::io_support::rio::HashReader; pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024; pub const MAX_PARTS_COUNT: usize = 10000; @@ -872,7 +861,7 @@ pub(crate) use ops::object::DeleteObjectCommitBarrier; #[cfg(feature = "test-util")] pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier; pub(crate) use ops::object::body_cache_plaintext_len; -#[cfg(test)] +#[cfg(all(test, feature = "test-util"))] pub(crate) use ops::object::cleanup_rejected_transition_upload_durably; #[cfg(any(test, feature = "test-util"))] pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause}; @@ -1029,8 +1018,7 @@ mod prepared_get_object_metadata_tests { use crate::ecstore_validation_blackbox::make_local_set_disks; use crate::object_api::{BLOCK_SIZE_V2, PutObjReader}; use crate::set_disk::core::io_primitives::{bounded_metadata_fanout_order, disk_call_counters, rename_fanout_barrier}; - use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions}; - use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + use crate::storage_api_contracts::bucket::MakeBucketOptions; use crate::test_metrics::CapturingRecorder; use http::HeaderMap; use tokio::io::AsyncReadExt; @@ -5971,7 +5959,7 @@ mod tests { use crate::set_disk::core::io_primitives::rename_fanout_barrier; use crate::storage_api_contracts::{ heal::HealOperations as _, lifecycle::TransitionedObject, list::ListOperations as _, multipart::CompletePart, - namespace::NamespaceLocking as _, object::ObjectIO as _, object::ObjectOperations as _, + object::ObjectOperations as _, }; use crate::store::init_format::save_format_file; use crate::store::list_objects::ListPathOptions; diff --git a/crates/ecstore/src/set_disk/ops/bitrot_self_verify.rs b/crates/ecstore/src/set_disk/ops/bitrot_self_verify.rs index 93c038e6a..52af4a3d0 100644 --- a/crates/ecstore/src/set_disk/ops/bitrot_self_verify.rs +++ b/crates/ecstore/src/set_disk/ops/bitrot_self_verify.rs @@ -12,7 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::*; +use super::super::{ + Cursor, DiskStore, EVENT_SET_DISK_WRITE, Error, FileInfo, HashAlgorithm, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, + Result, join_all, warn, +}; +use crate::disk::DiskAPI; +#[cfg(test)] +use crate::disk::RUSTFS_META_TMP_BUCKET; +use crate::set_disk::coding; +#[cfg(test)] +use bytes::Bytes; /// Null out any disk whose shard writer failed (or was never created) so its /// truncated/absent shard is not committed by the final rename, and return the @@ -132,7 +141,6 @@ pub(in crate::set_disk::ops) async fn verify_written_bitrot_shards( mod tests { use super::super::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity; use super::*; - use crate::disk::DiskAPI as _; async fn encode_streaming_shard(data: &[u8], shard_size: usize) -> Bytes { let mut writer = coding::BitrotWriter::new(Cursor::new(Vec::new()), shard_size, HashAlgorithm::HighwayHash256S); diff --git a/crates/ecstore/src/set_disk/ops/bucket.rs b/crates/ecstore/src/set_disk/ops/bucket.rs index 904e59e19..50e84ee68 100644 --- a/crates/ecstore/src/set_disk/ops/bucket.rs +++ b/crates/ecstore/src/set_disk/ops/bucket.rs @@ -19,7 +19,12 @@ //! `for SetDisks`, so its associated-type bounds are unchanged and runtime //! behavior is the same. -use super::super::*; +use super::super::{ + BUCKET_OP_IGNORED_ERRS, BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DiskError, Error, HashMap, + MakeBucketOptions, Result, SetDisks, is_reserved_or_invalid_bucket, join_all, reduce_write_quorum_errs, +}; +use crate::api::bucket::metadata_sys; +use crate::disk::DiskAPI; impl SetDisks { pub(crate) async fn list_bucket_for_scanner(&self, _opts: &BucketOptions) -> Result<(Vec, bool)> { diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index d1a5c9281..171e83fc2 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -12,9 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::*; +use super::super::{ + Bytes, CHECK_PART_FILE_CORRUPT, CHECK_PART_SUCCESS, DeleteOptions, DiskError, DiskStore, DriveState, EVENT_SET_DISK_HEAL, + Error, FileInfo, HashAlgorithm, HashMap, HealDriveInfo, HealItemType, HealOpts, HealResultItem, HealScanMode, Infos, + LOG_SUBSYSTEM_SET_DISK, ObjectInfo, ObjectOptions, ObjectPartInfo, Path, RUSTFS_META_TMP_BUCKET, ReadOptions, Result, + SLASH_SEPARATOR, SetDisks, StorageError, Uuid, coding, count_errs, count_part_not_success, create_bitrot_reader, + create_bitrot_writer, debug, disk, disks_with_all_parts, encode_dir_object, error, file_info_is_valid_for_metadata, + formats_match_reference_slots, get_format_erasure_in_quorum, get_lock_acquire_timeout, has_suffix, + heal_bucket_local_on_disks, is_object_dir_dangling, join_all, load_format_erasure_all, path_join_buf, save_format_file, + should_heal_object_on_disk, stat_all_dirs, to_object_err, warn, +}; use crate::disk::DataDirDeleteStatus; -use crate::disk::disk_store::DiskStoreRenameDataExt; +use crate::disk::DiskAPI; use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX; use crate::io_support::bitrot::object_mmap_read_enabled; use crate::storage_api_contracts::namespace::NamespaceLocking as _; diff --git a/crates/ecstore/src/set_disk/ops/heal_walk.rs b/crates/ecstore/src/set_disk/ops/heal_walk.rs index fe0ce30fc..aa627e415 100644 --- a/crates/ecstore/src/set_disk/ops/heal_walk.rs +++ b/crates/ecstore/src/set_disk/ops/heal_walk.rs @@ -22,8 +22,14 @@ //! every `(object, version)` present on ANY disk, feeding each to the existing //! per-version `SetDisks::heal_object`. -use super::super::*; +use super::super::{ + Arc, CancellationToken, DiskError, ListPathRawOptions, MetaCacheEntries, MetaCacheEntry, SetDisks, debug, disk, list_path_raw, +}; +#[cfg(test)] +use crate::disk::DiskAPI; use crate::object_api::ObjectInfo; +#[cfg(test)] +use rustfs_filemeta::FileMeta; use std::collections::HashSet; use std::sync::Mutex; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; diff --git a/crates/ecstore/src/set_disk/ops/list.rs b/crates/ecstore/src/set_disk/ops/list.rs index 59dc1401a..a25258f5a 100644 --- a/crates/ecstore/src/set_disk/ops/list.rs +++ b/crates/ecstore/src/set_disk/ops/list.rs @@ -22,7 +22,11 @@ //! runtime behavior is unchanged. use super::super::ctx::SetDisksCtx; -use super::super::*; +use super::super::{ + Arc, CancellationToken, DeleteOptions, DiskError, DiskStore, Error, ListObjectVersionsInfo, ListObjectsV2Info, + OBJECT_OP_IGNORED_ERRS, ObjectInfoOrErr, Result, Sender, SetDisks, WalkOptions, debug, join_all, reduce_write_quorum_errs, +}; +use crate::disk::DiskAPI; impl SetDisks { #[tracing::instrument(skip(self))] diff --git a/crates/ecstore/src/set_disk/ops/locking.rs b/crates/ecstore/src/set_disk/ops/locking.rs index be33c0c43..e2674ac38 100644 --- a/crates/ecstore/src/set_disk/ops/locking.rs +++ b/crates/ecstore/src/set_disk/ops/locking.rs @@ -19,9 +19,19 @@ //! here; the contract stays implemented `for SetDisks`, so its associated-type //! bounds are unchanged and helper access is via inherent calls. -use super::super::*; +use super::super::{ + Arc, DiskError, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, Endpoint, Error, FormatV3, HealChannelPriority, LockResult, + NamespaceLock, NamespaceLockWrapper, ObjectKey, Result, SetDisks, StorageError, debug, disk, info, load_format_erasure, + send_heal_disk, warn, +}; +use crate::disk::DiskAPI; use crate::disk::health_state::DriveMembershipSnapshot; +#[cfg(test)] +use crate::disk::new_disk; use crate::runtime::sources as runtime_sources; +use rand::prelude::SliceRandom; +#[cfg(test)] +use uuid::Uuid; #[async_trait::async_trait] impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks { diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 7aafc24d2..10a98b215 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -20,24 +20,79 @@ //! contract stays implemented `for SetDisks`, so its associated-type bounds are //! unchanged; method bodies are moved verbatim and runtime behavior is the same. -use super::super::*; +#[cfg(test)] +use super::super::GetObjectMetadataCacheKey; +#[cfg(test)] +use super::super::MetadataCacheInvalidationProbe; +#[cfg(test)] +use super::super::capacity_scope_from_disks; +use super::super::{ + AMZ_STORAGE_CLASS, Arc, Bytes, CompletePart, Cursor, DiskError, DiskStore, EVENT_SET_DISK_MULTIPART, Error, FileInfo, + GLOBAL_MIN_PART_SIZE, HashAlgorithm, HashMap, HashReader, HashSet, HealChannelPriority, Instant, LOG_COMPONENT_ECSTORE, + LOG_SUBSYSTEM_SET_DISK, ListMultipartsInfo, ListPartsInfo, MAX_PARTS_COUNT, MULTIPART_WRITE_QUORUM_RENAME_PART, + MULTIPART_WRITE_QUORUM_UPLOAD_METADATA, MULTIPART_WRITE_QUORUM_WRITER_SETUP, MultipartInfo, MultipartUploadResult, + MultipartWriteQuorumContext, NamespaceLockFence, OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectLockDiagGuard, ObjectOptions, + ObjectPartInfo, OffsetDateTime, PartInfo, PutObjReader, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, + RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, Result, SLASH_SEPARATOR, SUFFIX_ACTUAL_OBJECT_SIZE_CAP, + SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, + SUFFIX_RESTORE_OPERATION_ID, SetDisks, SmallWritePath, StorageError, Uuid, WriteLayout, + check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding, complete_multipart_part_error, + complete_multipart_part_error_result, complete_part_checksum, completed_multipart_object_part, contains_key_str, + create_bitrot_writer, debug, disk, error, get_complete_multipart_md5, get_header_map, get_str, insert_str, + is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size, log_multipart_write_quorum_failure, + parts_after_marker, path_join_buf, record_compression_total_memory, reduce_read_quorum_errs, reduce_write_quorum_errs, + remove_header_map, resolve_write_layout, restore_commit_operation_id_from_metadata, should_persist_encryption_original_size, + strip_internal_multipart_metadata, to_object_err, warn, +}; use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards}; +#[cfg(test)] +use super::object::old_data_cleanup_receipt_path; use super::object::{ assign_object_transaction_epoch, object_transaction_fencing_fleet_proof, object_transaction_fencing_fleet_proof_matches, - object_transaction_fencing_requested, old_data_cleanup_receipt_path, read_object_transaction_epoch_fence, - verify_object_transaction_epoch_fence, + object_transaction_fencing_requested, read_object_transaction_epoch_fence, verify_object_transaction_epoch_fence, }; +use crate::api::config::storageclass; +#[cfg(test)] +use crate::bucket::metadata_sys::ObjectLockConfigState; use crate::bucket::quota::reservation; use crate::crash_inject::{self, CrashPoint}; +use crate::disk::DiskAPI; +#[cfg(test)] +use crate::disk::DiskOption; +#[cfg(test)] +use crate::disk::STORAGE_FORMAT_FILE; +#[cfg(test)] +use crate::disk::new_disk; use crate::multipart_listing::paginate_multipart_listing; +#[cfg(test)] +use crate::object_api::ObjectLockConfigSnapshot; use crate::set_disk::core::io_primitives::finish_rename_tail_heal; +use crate::set_disk::mem; +use crate::set_disk::metadata_sys; +use crate::set_disk::runtime_sources; +#[cfg(test)] +use crate::storage_api_contracts::multipart::MultipartOperations; +#[cfg(test)] +use crate::storage_api_contracts::object::HTTPPreconditions; +use crate::storage_api_contracts::object::ObjectOperations; use futures::{StreamExt, stream}; +#[cfg(test)] +use http::HeaderMap; +use rustfs_rio::EtagResolvable; +use rustfs_rio::TryGetIndex; +#[cfg(test)] +use rustfs_utils::http::SSEC_ALGORITHM_HEADER; +#[cfg(test)] +use rustfs_utils::http::SUFFIX_COMPRESSION; use std::future::Future; #[cfg(test)] use std::sync::atomic::AtomicBool; #[cfg(any(test, feature = "test-util"))] use std::sync::atomic::{AtomicUsize, Ordering}; +#[cfg(any(test, feature = "test-util"))] use std::time::Duration; +#[cfg(test)] +use tokio::io::AsyncReadExt; use tokio::task::JoinSet; const MULTIPART_LIST_IO_CONCURRENCY: usize = 16; @@ -2998,7 +3053,7 @@ fn resolve_complete_etag(opts: &ObjectOptions, uploaded_parts: &[CompletePart]) mod tests { use super::*; use crate::config::storageclass::lookup_config_for_pools_without_env; - use crate::disk::{DiskAPI as _, ReadOptions}; + use crate::disk::ReadOptions; use crate::disk::{endpoint::Endpoint, format::FormatV3}; use crate::layout::endpoints::SetupType; use crate::services::notification_sys::install_remote_version_state_fleet_proof_for_test; @@ -3014,7 +3069,7 @@ mod tests { }; use crate::set_disk::ops::object::{PutObjectCommitBarrier, PutObjectCommitPause}; use crate::storage_api_contracts::namespace::NamespaceLocking as _; - use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + use crate::storage_api_contracts::object::ObjectIO as _; use rustfs_config::server_config::KVS; use rustfs_lock::{LockClient, client::local::LocalClient}; use serial_test::serial; @@ -4870,7 +4925,7 @@ mod tests { #[serial(metadata_cache_invalidation_probe)] async fn complete_multipart_generation_retires_cached_snapshot() { use crate::storage_api_contracts::multipart::MultipartOperations as _; - use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + use crate::storage_api_contracts::object::ObjectIO as _; let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; let bucket = "multipart-metadata-generation-bucket"; @@ -7369,9 +7424,7 @@ mod tests { mod crash_consistency { use super::*; use crate::crash_inject::{self, CrashPoint}; - use crate::storage_api_contracts::object::ObjectIO as _; use http::HeaderMap; - use tokio::io::AsyncReadExt as _; /// 1 MiB keeps every object off the 128 KiB inline fast path, so the /// commit moves real erasure shards through `rename_data`. diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index aa75a27e7..83604a719 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -19,11 +19,74 @@ //! bounds are unchanged, and the impls reach shared primitives through the //! SetDisks core (io_primitives) via inherent calls. -use super::super::*; +#[cfg(test)] +use super::super::MetadataCacheInvalidationProbe; +use super::super::{ + AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS, Arc, AsyncWrite, AtomicU64, BufReader, Bytes, CACHE_CONTROL, CONTENT_DISPOSITION, + CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, CompletePart, Cursor, DeleteAccounting, DeleteOptions, DeletedObject, + DiskError, DiskStore, EVENT_SET_DISK_COMMIT_TAIL_SLOW, EVENT_SET_DISK_PUT_OBJECT_STAGE_SUMMARY, EVENT_SET_DISK_WRITE, + EXPIRES, Error, EventArgs, EventName, FastLockGuard, FileInfo, FileInfoVersions, + GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING, + GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_INTERNAL_META, + GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT, + GET_STAGE_INLINE_PREPARE, GET_STAGE_LOCK_ACQUIRE, GET_STAGE_METADATA, GET_STAGE_OBJECT_INFO, GET_STAGE_PATH_DECISION, + GET_STAGE_READER_SETUP, GenericError, GetCodecStreamingDecision, GetDirectMemoryDecision, GetObjectReader, HTTPRangeSpec, + HashAlgorithm, HashMap, HashReader, HashSet, HeaderMap, HealChannelPriority, InstanceContext, Instant, LOG_COMPONENT_ECSTORE, + LOG_SUBSYSTEM_SET_DISK, OBJECT_OP_IGNORED_ERRS, ObjectApiError, ObjectInfo, ObjectKey, ObjectLockConfigSnapshot, + ObjectLockConfigState, ObjectOptions, ObjectReader, ObjectToDelete, OffsetDateTime, Ordering, Pin, PutObjReader, + RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, ReaderImpl, ReplicateDecision, ReplicationObjectBridge, Result, + SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS, SLASH_SEPARATOR, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE, + SUFFIX_RESTORE_OPERATION_ID, SetDisks, SmallWritePath, StorageError, TRANSITION_COMPLETE, UpdateMetadataOpts, Uuid, + WriteLayout, X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_RESTORE, + adaptive_duplex_buffer_size, build_get_object_info, build_inline_bitrot_readers, build_inline_bitrot_readers_from_refs, + can_try_inline_data_shards_direct, check_object_lock_delete, check_object_lock_for_deletion_with_state, + check_object_lock_retention_update, classify_get_codec_streaming_object_class, classify_put_write_path, + classify_storage_error, collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer, debug, + delete_file_info_version_id, disk, ensure_delete_commit_locks_held, error, finish_set_disk_read_lock, + get_codec_streaming_reader_gate, get_object_body_cache_hook, get_raw_etag, get_small_object_direct_memory_decision, + get_stage_timer_if_enabled, get_str, get_transitioned_object_reader_with_tier_manager, inline_erasure_shard_file_offset, + inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled, is_err_object_not_found, is_err_version_not_found, + is_explicit_null_version, is_lock_optimization_enabled, issue3031_diag_enabled, join_all, known_put_object_storage_size, + path_join_buf, put_restore_opts, record_compression_total_memory, record_get_codec_streaming_gate_decision, + record_get_direct_memory_decision, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path, + record_get_object_reader_path_observation, record_get_stage_duration_if_enabled, record_lock_acquire, + reduce_write_quorum_errs, release_materialized_read_lock, replication_write_may_pass_worm_gate, require_restore_operation_id, + resolve_delete_version_state, resolve_tiered_decommission_write_quorum_result, resolve_write_layout, + restore_commit_operation_id_from_metadata, restore_operation_id_from_metadata, send_event, + set_disk_delete_creates_delete_marker, should_force_delete_marker_for_missing_version, + should_persist_encryption_original_size, should_preserve_delete_replication_state, should_use_inline_fast_path, + take_prepared_get_object_metadata, to_object_err, try_read_inline_data_shards_direct, warn, +}; use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards}; +use crate::api::config::storageclass; +use crate::bucket::lifecycle::bucket_lifecycle_ops::LifecycleOps; use crate::bucket::utils::is_meta_bucketname; +use crate::bucket::versioning::VersioningApi; +use crate::disk::DiskAPI; +use crate::set_disk::coding; +use crate::set_disk::core::io_primitives::GetCodecStreamingReaderBuildOutcome; +use crate::set_disk::mem; +use crate::set_disk::metadata_sys; use crate::set_disk::read::GetObjectDownstreamWriter; +use crate::set_disk::runtime_sources; +use crate::storage_api_contracts::multipart::MultipartOperations; +use crate::storage_api_contracts::object::ObjectIO; +use crate::storage_api_contracts::object::ObjectOperations; +use rustfs_lock::LockManager; +use rustfs_rio::EtagResolvable; +use rustfs_rio::HashReaderMut; +use rustfs_rio::TryGetIndex; +use rustfs_utils::http::HeaderExt; +use tokio::io::AsyncWriteExt; +#[cfg(all(test, feature = "test-util"))] +use super::super::GetObjectMetadataCacheEntry; +#[cfg(test)] +use super::super::GetObjectMetadataCacheKey; +#[cfg(test)] +use super::super::capacity_scope_from_disks; +#[cfg(all(test, feature = "test-util"))] +use super::super::get_lock_acquire_timeout; use crate::bucket::lifecycle::{ tier_delete_journal::{ enqueue_committed_tier_delete_journal_entry, persist_tier_delete_journal_entry, @@ -47,6 +110,20 @@ use crate::bucket::replication::{ }; use crate::data_usage::quota_object_size; use crate::diagnostics::get::GetObjectFailureReason; +#[cfg(test)] +use crate::disk::DiskOption; +#[cfg(all(test, feature = "test-util"))] +use crate::disk::RUSTFS_META_MULTIPART_BUCKET; +#[cfg(test)] +use crate::disk::ReadOptions; +#[cfg(test)] +use crate::disk::STORAGE_FORMAT_FILE; +#[cfg(test)] +use crate::disk::endpoint::Endpoint; +#[cfg(test)] +use crate::disk::format::FormatV3; +#[cfg(test)] +use crate::disk::new_disk; use crate::disk::{DataDirDeleteStatus, OldCurrentSize}; use crate::error::is_err_invalid_upload_id; use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed}; @@ -54,15 +131,30 @@ use crate::object_api::{NamespaceLockFence, SCANNER_PUBLICATION_LEASE_FENCE_META use crate::services::notification_sys::RemoteVersionStateFleetProofToken; use crate::services::tier::tier::{TierConfigMgr, TierOperationLease}; use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal}; +#[cfg(test)] +use crate::storage_api_contracts::namespace::NamespaceLocking; +#[cfg(test)] +use crate::storage_api_contracts::object::HTTPPreconditions; use crate::store::ECStore; use crate::store::utils::clean_metadata; use futures::FutureExt as _; use http::HeaderValue; +#[cfg(test)] +use rustfs_filemeta::FileMeta; +#[cfg(all(test, feature = "test-util"))] +use rustfs_filemeta::ObjectPartInfo; use rustfs_utils::path::decode_dir_object; +#[cfg(test)] +use rustfs_utils::path::encode_dir_object; use std::future::Future; -use std::sync::OnceLock; use std::task::{Context, Poll}; +#[cfg(any(test, feature = "test-util"))] +use std::time::Duration; +#[cfg(test)] +use tokio::io::AsyncReadExt; use tokio::io::{AsyncRead, ReadBuf}; +#[cfg(all(test, feature = "test-util"))] +use tokio::sync::RwLock; use tokio_util::sync::CancellationToken; const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json"; @@ -1066,11 +1158,6 @@ mod restore_metadata_update_tests { } } -#[cfg(test)] -mod delete_replication_transport_tests { - use super::*; -} - fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result { coding::Erasure::try_new_with_options(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size, uses_legacy) .map_err(Error::from) @@ -1893,7 +1980,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { ) .await? { - core::io_primitives::GetCodecStreamingReaderBuildOutcome::Reader(stream) => { + GetCodecStreamingReaderBuildOutcome::Reader(stream) => { record_get_codec_streaming_gate_decision( codec_streaming_gate.object_class, GetCodecStreamingDecision::Use, @@ -1907,7 +1994,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { reader.body_source = body_source; return Ok(finish_set_disk_read_lock(reader, read_lock_guard.take(), bucket, object)); } - core::io_primitives::GetCodecStreamingReaderBuildOutcome::Fallback(reason) => { + GetCodecStreamingReaderBuildOutcome::Fallback(reason) => { record_get_codec_streaming_gate_decision( codec_streaming_gate.object_class, GetCodecStreamingDecision::Fallback(reason), @@ -5066,7 +5153,7 @@ pub(in crate::set_disk::ops) fn object_transaction_fencing_requested() -> bool { #[cfg(not(test))] fn object_transaction_fencing_requested_cached() -> bool { - static REQUESTED: OnceLock = OnceLock::new(); + static REQUESTED: std::sync::OnceLock = std::sync::OnceLock::new(); *REQUESTED.get_or_init(load_object_transaction_fencing_requested) } @@ -8584,7 +8671,6 @@ mod replication_lww_tests { use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; use super::*; - use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; use rustfs_utils::http::headers::{ AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING, }; @@ -9212,8 +9298,7 @@ mod inline_put_commit_path_tests { use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; use super::*; use crate::config::storageclass::lookup_config_for_pools_without_env; - use crate::disk::{DiskAPI as _, ReadOptions}; - use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + use crate::disk::ReadOptions; use rustfs_config::server_config::KVS; use serial_test::serial; use tokio::io::AsyncReadExt; @@ -9606,7 +9691,6 @@ mod get_object_downstream_close_accounting_tests { }; use crate::disk::RUSTFS_META_BUCKET; use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions}; - use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; use crate::test_metrics::CapturingRecorder; use std::time::Duration; @@ -9892,8 +9976,7 @@ mod get_object_downstream_close_accounting_tests { mod metadata_mutation_generation_tests { use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; use super::*; - use crate::disk::{DiskAPI as _, ReadOptions}; - use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; + use crate::disk::ReadOptions; async fn put_and_prime( set_disks: &Arc, @@ -10153,15 +10236,12 @@ mod transition_commit_failure_tests { use super::hermetic_set_disks_support::hermetic_set_disks; use super::*; use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions}; - use crate::disk::DiskAPI as _; use crate::services::tier::test_util::{MockWarmBackend, register_mock_tier}; use crate::services::tier::tier::TierConfigMgr; - use crate::storage_api_contracts::multipart::MultipartOperations as _; - use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; use http::HeaderMap; use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status}; use s3s::dto::RestoreRequest; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::io::AsyncReadExt; pub(super) fn restore_operation_id_metadata(operation_id: Uuid) -> HashMap { let mut metadata = HashMap::new(); @@ -11907,11 +11987,9 @@ mod transition_upload_integrity_tests { use super::transition_commit_failure_tests::{restore_metadata, restore_operation_id_metadata}; use super::*; use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions}; - use crate::disk::DiskAPI as _; use crate::layout::endpoints::SetupType; use crate::services::tier::test_util::register_mock_tier; use crate::set_disk::replication::RestoreFinalizeBarrier; - use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; use http::HeaderMap; use rustfs_filemeta::RestoreStatusOps as _; use rustfs_lock::client::local::LocalClient; @@ -13665,9 +13743,7 @@ mod transition_source_identity_matrix_tests { use super::hermetic_set_disks_support::hermetic_set_disks; use super::*; use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions}; - use crate::disk::DiskAPI as _; use crate::services::tier::test_util::register_mock_tier; - use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; #[test] fn transition_source_identity_treats_nil_version_as_null_source() { @@ -13913,7 +13989,7 @@ mod heterogeneous_pool_put_tests { }; use super::*; use crate::config::storageclass::lookup_config_for_pools_without_env; - use crate::disk::{DiskAPI as _, ReadOptions}; + use crate::disk::ReadOptions; use crate::services::notification_sys::install_remote_version_state_fleet_proof_for_test; use rustfs_config::server_config::KVS; use serial_test::serial; @@ -14337,7 +14413,6 @@ mod put_object_tmp_cleanup_tests { use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; use super::*; - use crate::disk::DiskAPI as _; use crate::set_disk::core::io_primitives::{ ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier, rename_fault_injection, }; @@ -15954,7 +16029,7 @@ mod put_object_tags_early_stop_regression_tests { use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; use super::*; - use crate::disk::{DiskAPI as _, ReadOptions}; + use crate::disk::ReadOptions; #[tokio::test] async fn put_object_tags_writes_all_online_disks_under_early_stop() { @@ -16041,9 +16116,7 @@ mod put_object_tags_early_stop_regression_tests { mod object_tagging_namespace_lock_tests { use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; use super::*; - use crate::disk::{DiskAPI as _, ReadOptions}; - use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; - use tokio::io::AsyncReadExt as _; + use crate::disk::ReadOptions; #[derive(Clone, Copy, Debug)] enum CompetingMutation { @@ -16315,7 +16388,6 @@ mod delete_objects_lock_gating_tests { use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks; use super::hermetic_set_disks_support::hermetic_set_disks_with_lockers_and_ctx; use super::*; - use crate::disk::DiskAPI as _; use serial_test::serial; async fn put_plain_object(set_disks: &Arc, bucket: &str, object: &str) { diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index ce3f47016..794be00d9 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -12,7 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::*; +use super::{ + Arc, Bytes, DiskError, DiskStore, ErasureCache, Error, FileInfo, GetCodecStreamingFallbackReason, GetObjectFileInfo, + GetObjectMetadataCacheEntry, GetObjectMetadataCacheGeneration, GetObjectMetadataCacheKey, GetObjectReadPolicy, HashAlgorithm, + LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectOptions, RUSTFS_META_BUCKET, + ReadOptions, Result, SetDisks, StorageError, adaptive_duplex_buffer_size, build_get_codec_streaming_decode_engine, + build_inline_bitrot_readers_from_refs, collect_inline_data_shard_fileinfos_by_index, debug, error, + get_codec_streaming_metrics_path, get_codec_streaming_multipart_max_parts, get_object_read_policy, + is_codec_streaming_multipart_enabled, is_multipart_reader_setup_prefetch_enabled, object_fits_single_block, + reduce_read_quorum_errs, to_object_err, try_read_inline_data_shards_direct, warn, +}; use crate::diagnostics::get::{ GET_DIRECT_MEMORY_SUBPATH_DISK_DATA_BLOCKS, GET_DIRECT_MEMORY_SUBPATH_INLINE_BUFFERED, GET_METADATA_CACHE_DECISION_HIT, GET_METADATA_CACHE_DECISION_MISS, GET_METADATA_CACHE_DECISION_REJECT, GET_METADATA_CACHE_DECISION_SKIP, @@ -22,43 +31,152 @@ use crate::diagnostics::get::{ GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_CHECKSUMS, GET_METADATA_CACHE_REASON_PART_NUMBER, GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_STALE_PUBLICATION, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, - GET_METADATA_CACHE_REASON_VERSIONED, GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, - GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR, - GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, - GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, - GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, - GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR, - GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID, - GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY, - GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, - GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, - GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, - GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, - GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, mark_get_object_downstream_closed, - record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled, -}; -use crate::erasure::coding::BitrotReader; -use crate::io_support::bitrot::{ - BitrotReaderStageMetrics, DeferredReaderStripeHandle, create_bitrot_reader_with_stage_metrics, create_deferred_bitrot_reader, - object_mmap_read_enabled, + GET_METADATA_CACHE_REASON_VERSIONED, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META, + GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP, + GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, GET_STAGE_READER_TASK_BITROT_READER_INIT, + GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, GetObjectFailureReason, classify_disk_error, + get_stage_timer_if_enabled, mark_get_object_downstream_closed, record_get_object_pipeline_failure, + record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled, }; +use crate::disk::DiskAPI; +use crate::io_support::bitrot::{BitrotReaderStageMetrics, DeferredReaderStripeHandle, object_mmap_read_enabled}; +use crate::set_disk::coding; +use crate::set_disk::runtime_sources; use crate::set_disk::shard_source::ShardReadCost; -use futures::stream::{FuturesUnordered, StreamExt}; -use metrics::counter; use std::{ - collections::{HashMap, VecDeque}, future::Future, io::IoSlice, pin::Pin, - sync::OnceLock, task::{Context, Poll}, time::{Duration, Instant}, }; use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; -use tokio::sync::RwLock; -use tokio::task::JoinSet; +#[cfg(test)] +use super::DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES; +#[cfg(test)] +use super::DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE; +#[cfg(test)] +use super::ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED; +#[cfg(test)] +use super::ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE; +#[cfg(test)] +use super::ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE; +#[cfg(test)] +use super::ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE; +#[cfg(test)] +use super::ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE; +#[cfg(test)] +use super::ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED; +#[cfg(test)] +use super::ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE; +#[cfg(test)] +use super::ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE; +#[cfg(test)] +use super::ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS; +#[cfg(test)] +use super::ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT; +#[cfg(test)] +use super::ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT; +#[cfg(test)] +use super::ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE; +#[cfg(test)] +use super::ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE; +#[cfg(test)] +use super::ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT; +#[cfg(test)] +use super::ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE; +#[cfg(test)] +use super::ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE; +#[cfg(test)] +use super::ENV_RUSTFS_GET_MULTIPART_READER_SETUP_PREFETCH; +#[cfg(test)] +use super::ENV_RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES; +#[cfg(test)] +use super::GET_OBJECT_METADATA_CACHE_TTL; +#[cfg(test)] +use super::GetCodecStreamingConfig; +#[cfg(test)] +use super::GetCodecStreamingDecision; +#[cfg(test)] +use super::GetCodecStreamingEngine; +#[cfg(test)] +use super::GetCodecStreamingGate; +#[cfg(test)] +use super::GetCodecStreamingObjectClass; +#[cfg(test)] +use super::GetCodecStreamingRollout; +#[cfg(test)] +use super::classify_get_codec_streaming_object_class; use super::core::io_primitives::*; +#[cfg(test)] +use super::get_codec_streaming_config_cached_core; +#[cfg(test)] +use super::get_codec_streaming_engine; +#[cfg(test)] +use super::get_codec_streaming_reader_gate; +#[cfg(test)] +use super::get_object_metadata_cache_max_entries; +#[cfg(test)] +use super::is_get_metadata_data_read_early_stop_enabled; +#[cfg(test)] +use super::is_get_metadata_early_stop_bounded_fanout_enabled; +#[cfg(test)] +use super::is_get_metadata_early_stop_enabled; +#[cfg(test)] +use super::is_version_early_stop_enabled; +#[cfg(test)] +use super::load_get_codec_streaming_config; +#[cfg(test)] +use super::with_get_object_read_policy; +#[cfg(test)] +use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; +#[cfg(test)] +use crate::diagnostics::get::GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE; +#[cfg(test)] +use crate::diagnostics::get::GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE; +#[cfg(test)] +use crate::diagnostics::get::{ + GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, + GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, + GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, + GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, + GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, + GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, + GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, +}; +#[cfg(test)] +use crate::disk::ReadMultipleResp; +#[cfg(test)] +use crate::disk::format::FormatV3; +#[cfg(test)] +use crate::erasure::codec::bridge::CodecStreamingDecodeEngine; +#[cfg(test)] +use crate::erasure::codec::bridge::GET_CODEC_STREAMING_ENGINE_RUSTFS; +#[cfg(test)] +use crate::object_api::PutObjReader; +#[cfg(test)] +use crate::storage_api_contracts::object::ObjectIO; +#[cfg(test)] +use crate::storage_api_contracts::range::HTTPRangeSpec; +#[cfg(test)] +use rustfs_filemeta::ObjectPartInfo; +#[cfg(test)] +use rustfs_heal_contracts::heal_channel::HealAdmissionResult; +#[cfg(test)] +use rustfs_heal_contracts::heal_channel::HealChannelPriority; +#[cfg(test)] +use rustfs_utils::http::SUFFIX_COMPRESSION; +#[cfg(test)] +use rustfs_utils::http::insert_str; +#[cfg(test)] +use time::OffsetDateTime; +#[cfg(test)] +use tokio::sync::RwLock; +#[cfg(test)] +use tokio::time::timeout; +#[cfg(test)] +use uuid::Uuid; pub(super) struct GetObjectDownstreamWriter { inner: W, diff --git a/crates/ecstore/src/set_disk/replication.rs b/crates/ecstore/src/set_disk/replication.rs index 5b38c8211..c1c0eeaf7 100644 --- a/crates/ecstore/src/set_disk/replication.rs +++ b/crates/ecstore/src/set_disk/replication.rs @@ -12,11 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::*; +use super::{ + Error, FileInfo, NamespaceLockFence, ObjectInfo, ObjectOptions, OffsetDateTime, Result, SetDisks, StorageError, + UpdateMetadataOpts, Uuid, X_AMZ_RESTORE, get_raw_etag, restore_operation_id_from_metadata, +}; use crate::bucket::lifecycle::lifecycle; use rustfs_filemeta::RestoreStatusOps; use rustfs_utils::http::headers::{AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE}; use s3s::dto::{RestoreStatus, Timestamp}; +#[cfg(all(test, feature = "test-util"))] +use std::sync::Arc; #[cfg(all(test, feature = "test-util"))] struct RestoreFinalizeBarrierState { diff --git a/crates/ecstore/src/set_disk/transition_matrix_tests.rs b/crates/ecstore/src/set_disk/transition_matrix_tests.rs index a137d55a6..e43f83f8c 100644 --- a/crates/ecstore/src/set_disk/transition_matrix_tests.rs +++ b/crates/ecstore/src/set_disk/transition_matrix_tests.rs @@ -12,10 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::*; +use super::{ + Arc, Duration, GetObjectMetadataCacheKey, HeaderMap, MakeBucketOptions, ObjectOptions, OffsetDateTime, PutObjReader, + SetDisks, Uuid, +}; use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time}; use crate::ecstore_validation_blackbox::make_local_set_disks; use crate::services::tier::test_util::register_mock_tier; +use crate::storage_api_contracts::bucket::BucketOperations; use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _}; use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status}; use tokio::io::AsyncReadExt; diff --git a/scripts/ecstore-module-lint-register.txt b/scripts/ecstore-module-lint-register.txt index 1033461f3..ccb10b437 100644 --- a/scripts/ecstore-module-lint-register.txt +++ b/scripts/ecstore-module-lint-register.txt @@ -110,4 +110,3 @@ crates/ecstore/src/services/tier/warm_backend_s3.rs|unused_variables crates/ecstore/src/services/tier/warm_backend_tencent.rs|clippy::all crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_must_use crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_variables -crates/ecstore/src/set_disk/mod.rs|unused_variables