mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-29 08:27:06 +00:00
refactor(ecstore): retire set_disk lint blankets via explicit imports (#6697)
refactor(ecstore): retire the set_disk lint blankets by making the prelude explicit
backlog#1823 step 1 / backlog#2029 road 2. Removes the last two module-level lint blankets in ecstore: set_disk/mod.rs #![allow(unused_imports)] and #![allow(unused_variables)], restoring both lints for the whole 40K-line subtree, and deletes the register line for the unused_variables blanket in the same diff (the guard from #6155 is a bidirectional exact match).
The unused_imports blanket existed because 14 submodules consumed mod.rs as a glob prelude (use super::* / use super::super::*), and rustc does not track consumption through glob re-exports. Each glob is now an explicit use super::{...} list, keeping mod.rs as the single import hub while making every import lint-checkable. Names consumed only by test or test-util units carry #[cfg(test)] / #[cfg(all(test, feature = "test-util"))] / #[cfg(any(test, feature = "test-util"))] gates matching their consumers; storage-api traits are routed through the storage_api_contracts facade per the architecture guard.
The sweep then deleted the genuinely dead imports the blanket was hiding (chrono::Utc, glob::Pattern, futures::task::AtomicWaker, rustfs_lock LocalLock, AsyncBatchProcessor, rand::Rng, std::future::Future among others in mod.rs, plus stale scoped imports and one empty test module shell across the subtree). One unused_variables finding surfaced: flush_read_version_coalescer_pending's lane_key is read only by the #[cfg(test)] counter block, handled with the cfg(not(test)) let _ pattern established in #6158.
Verification: cargo check zero warnings versus the 9cf276ed2 baseline on five lanes (default lib / --tests / rio-v2 --tests / test-util --tests / test-util,rio-v2 --tests; the --tests lane keeps the same three pre-existing core/pools.rs and store/object.rs dead-code warnings main already has); clippy --lib --tests -D warnings clean with test-util,rio-v2; cargo nextest run 4567 passed; make pre-commit exit 0.
This commit is contained in:
@@ -22,20 +22,47 @@
|
|||||||
//! byte-identical to the pre-move sources; only the module header
|
//! byte-identical to the pre-move sources; only the module header
|
||||||
//! (`use super::*;` -> `use super::super::*;`) and item visibility change.
|
//! (`use super::*;` -> `use super::super::*;`) and item visibility change.
|
||||||
|
|
||||||
use super::super::*;
|
#[cfg(test)]
|
||||||
|
use super::super::ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::super::ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::super::ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::super::get_metadata_slowtail_fault_delay;
|
||||||
|
use super::super::{
|
||||||
|
Bytes, CHECK_PART_DISK_NOT_FOUND, DeleteOptions, DiskError, DiskStore, EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
|
||||||
|
EVENT_SET_DISK_WRITE, Error, FileInfo, FileMeta, FileMetaShallowVersion, GetCodecStreamingFallbackReason,
|
||||||
|
GetObjectMetadataCacheEntry, HTTPPreconditions, HashAlgorithm, HealAdmissionResult, HealChannelPriority, HealRequestSource,
|
||||||
|
LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, MultipartWriteQuorumContext, OBJECT_OP_IGNORED_ERRS, ObjectOptions,
|
||||||
|
ObjectPartInfo, OffsetDateTime, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RawFileInfo, ReadMultipleReq,
|
||||||
|
ReadMultipleResp, ReadOptions, Result, SLASH_SEPARATOR, STORAGE_FORMAT_FILE, SetDisks, SnapshotLeaseToken, StorageError,
|
||||||
|
UpdateMetadataOpts, Uuid, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct,
|
||||||
|
capacity_scope_from_disks, coding, collect_inline_data_shard_fileinfos_by_index_or_reason, current_dirty_generation, debug,
|
||||||
|
disk, file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info, inline_erasure_shard_file_offset,
|
||||||
|
inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found, is_get_metadata_data_read_early_stop_enabled,
|
||||||
|
is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled, is_object_dangling,
|
||||||
|
is_version_early_stop_enabled, issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure,
|
||||||
|
merge_file_meta_versions, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs, reduce_write_quorum_errs,
|
||||||
|
send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn,
|
||||||
|
};
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD;
|
||||||
use crate::diagnostics::get::{
|
use crate::diagnostics::get::{
|
||||||
GET_DIRECT_MEMORY_SUBPATH_DISK_DATA_BLOCKS, GET_DIRECT_MEMORY_SUBPATH_INLINE_BUFFERED, GET_METADATA_CACHE_DECISION_HIT,
|
|
||||||
GET_METADATA_CACHE_DECISION_MISS, GET_METADATA_CACHE_DECISION_REJECT, GET_METADATA_CACHE_DECISION_SKIP,
|
|
||||||
GET_METADATA_CACHE_REASON_DATA_MOVEMENT, GET_METADATA_CACHE_REASON_DELETE_MARKER, GET_METADATA_CACHE_REASON_DIST_ERASURE,
|
|
||||||
GET_METADATA_CACHE_REASON_INCL_FREE_VERSIONS, GET_METADATA_CACHE_REASON_INSUFFICIENT_CACHED_QUORUM,
|
|
||||||
GET_METADATA_CACHE_REASON_META_BUCKET, GET_METADATA_CACHE_REASON_NO_LOCK, GET_METADATA_CACHE_REASON_NOT_FOUND_OR_EXPIRED,
|
|
||||||
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_NUMBER,
|
|
||||||
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID,
|
|
||||||
GET_METADATA_CACHE_REASON_VERSION_SUSPENDED, GET_METADATA_CACHE_REASON_VERSIONED,
|
|
||||||
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY,
|
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY,
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY,
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH,
|
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD,
|
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE,
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE,
|
||||||
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED,
|
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED,
|
||||||
@@ -45,16 +72,26 @@ use crate::diagnostics::get::{
|
|||||||
GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
|
GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
|
||||||
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
|
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
|
||||||
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
|
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
|
||||||
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY,
|
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META,
|
||||||
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE,
|
GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE,
|
||||||
GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
|
GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN,
|
||||||
GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
|
GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled, record_get_stage_duration_if_enabled,
|
||||||
GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
|
|
||||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
|
||||||
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
|
||||||
};
|
};
|
||||||
use crate::disk::disk_store::{DiskStoreRenameDataExt, get_drive_metadata_timeout};
|
#[cfg(test)]
|
||||||
|
use crate::disk::CHECK_PART_FILE_NOT_FOUND;
|
||||||
|
use crate::disk::DiskAPI;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::DiskOption;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::RUSTFS_META_TMP_BUCKET;
|
||||||
|
use crate::disk::disk_store::get_drive_metadata_timeout;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::endpoint::Endpoint;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::format::FormatV3;
|
||||||
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
|
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::new_disk;
|
||||||
use crate::disk::{
|
use crate::disk::{
|
||||||
BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk,
|
BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk,
|
||||||
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
|
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
|
||||||
@@ -65,9 +102,11 @@ use crate::io_support::bitrot::ShardReader;
|
|||||||
use crate::io_support::bitrot::{
|
use crate::io_support::bitrot::{
|
||||||
BitrotReaderStageMetrics, DeferredReaderStripeHandle, adjust_shard_read_params,
|
BitrotReaderStageMetrics, DeferredReaderStripeHandle, adjust_shard_read_params,
|
||||||
create_bitrot_reader_from_bytes_with_stage_metrics, create_deferred_bitrot_reader_with_stripe_handle,
|
create_bitrot_reader_from_bytes_with_stage_metrics, create_deferred_bitrot_reader_with_stripe_handle,
|
||||||
object_mmap_read_enabled, object_mmap_read_max_length,
|
object_mmap_read_max_length,
|
||||||
};
|
};
|
||||||
|
use crate::set_disk::runtime_sources;
|
||||||
use crate::set_disk::shard_source::ShardReadCost;
|
use crate::set_disk::shard_source::ShardReadCost;
|
||||||
|
use crate::storage_api_contracts::object::ObjectOperations;
|
||||||
use futures::FutureExt as _;
|
use futures::FutureExt as _;
|
||||||
use futures::stream::{FuturesUnordered, StreamExt};
|
use futures::stream::{FuturesUnordered, StreamExt};
|
||||||
use metrics::counter;
|
use metrics::counter;
|
||||||
@@ -281,6 +320,9 @@ async fn flush_read_version_coalescer_pending(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Only the #[cfg(test)] counter-recording block below reads this.
|
||||||
|
#[cfg(not(test))]
|
||||||
|
let _ = lane_key;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
{
|
{
|
||||||
let mut observed_paths = HashSet::new();
|
let mut observed_paths = HashSet::new();
|
||||||
@@ -6762,7 +6804,7 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use std::io::Cursor;
|
use std::io::Cursor;
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
use tokio::io::AsyncReadExt;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn write_precondition_lookup_errors_fail_closed_unless_absence_is_known() {
|
fn write_precondition_lookup_errors_fail_closed_unless_absence_is_known() {
|
||||||
|
|||||||
@@ -23,7 +23,7 @@
|
|||||||
//! This module only establishes the borrow handle. It moves no trait impl and
|
//! This module only establishes the borrow handle. It moves no trait impl and
|
||||||
//! changes no runtime behavior.
|
//! changes no runtime behavior.
|
||||||
|
|
||||||
use super::*;
|
use super::{Arc, DiskStore, Endpoint, FormatV3, LockClient, RwLock, SetDisks};
|
||||||
|
|
||||||
/// Lightweight, `Copy` handle borrowing the shared [`SetDisks`] core state.
|
/// Lightweight, `Copy` handle borrowing the shared [`SetDisks`] core state.
|
||||||
///
|
///
|
||||||
|
|||||||
@@ -12,8 +12,19 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use super::*;
|
use super::{
|
||||||
|
Bytes, DATA_MOVEMENT_MULTIPART_PREFIX, DiskError, DiskStore, FileInfo, HashMap, HashSet, OBJECT_OP_IGNORED_ERRS, ObjProps,
|
||||||
|
OffsetDateTime, SetDisks, Sha256, TRANSITION_COMPLETE, Uuid, debug, disk, error, file_info_is_valid_for_metadata, hex,
|
||||||
|
reduce_read_quorum_errs, warn,
|
||||||
|
};
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::DiskOption;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::endpoint::Endpoint;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::new_disk;
|
||||||
use rustfs_utils::http;
|
use rustfs_utils::http;
|
||||||
|
use sha2::Digest;
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
struct FileInfoIdentityGroup {
|
struct FileInfoIdentityGroup {
|
||||||
|
|||||||
@@ -38,23 +38,15 @@
|
|||||||
//! read primitives it drives.
|
//! read primitives it drives.
|
||||||
//! - `metadata.rs`, `replication.rs`, `shard_source.rs` — supporting helpers.
|
//! - `metadata.rs`, `replication.rs`, `shard_source.rs` — supporting helpers.
|
||||||
|
|
||||||
// #730: SetDisks still hosts staged read/heal/write migration helpers.
|
|
||||||
#![allow(unused_imports)]
|
|
||||||
#![allow(unused_variables)]
|
|
||||||
|
|
||||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||||
use crate::bucket::metadata_sys;
|
use crate::bucket::metadata_sys;
|
||||||
use crate::bucket::metadata_sys::ObjectLockConfigState;
|
use crate::bucket::metadata_sys::ObjectLockConfigState;
|
||||||
use crate::bucket::object_lock::objectlock_sys::{
|
use crate::bucket::object_lock::objectlock_sys::{
|
||||||
check_object_lock_for_deletion_with_default_retention, check_object_lock_for_deletion_with_state,
|
check_object_lock_for_deletion_with_state, check_retention_for_modification, replication_write_may_pass_worm_gate,
|
||||||
check_retention_for_modification, replication_write_may_pass_worm_gate,
|
|
||||||
};
|
};
|
||||||
use crate::bucket::replication::{
|
#[cfg(test)]
|
||||||
ReplicateDecision, ReplicationObjectBridge, ReplicationState, ReplicationStatusType, VersionPurgeStatusType,
|
use crate::bucket::replication::ReplicationState;
|
||||||
replication_state_to_filemeta,
|
use crate::bucket::replication::{ReplicateDecision, ReplicationObjectBridge, ReplicationStatusType, VersionPurgeStatusType};
|
||||||
};
|
|
||||||
use crate::bucket::versioning::VersioningApi;
|
|
||||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
|
||||||
use crate::cluster::rpc::heal_bucket_local_on_disks;
|
use crate::cluster::rpc::heal_bucket_local_on_disks;
|
||||||
use crate::data_usage::record_compression_total_memory;
|
use crate::data_usage::record_compression_total_memory;
|
||||||
use crate::diagnostics::get::{
|
use crate::diagnostics::get::{
|
||||||
@@ -73,9 +65,11 @@ use crate::disk::error_reduce::{
|
|||||||
BUCKET_OP_IGNORED_ERRS, OBJECT_OP_IGNORED_ERRS, build_write_quorum_failure_summary, count_errs, reduce_read_quorum_errs,
|
BUCKET_OP_IGNORED_ERRS, OBJECT_OP_IGNORED_ERRS, build_write_quorum_failure_summary, count_errs, reduce_read_quorum_errs,
|
||||||
reduce_write_quorum_errs,
|
reduce_write_quorum_errs,
|
||||||
};
|
};
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::has_part_err;
|
||||||
use crate::disk::{
|
use crate::disk::{
|
||||||
self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
|
self, CHECK_PART_DISK_NOT_FOUND, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
|
||||||
conv_part_err_to_int, has_part_err,
|
conv_part_err_to_int,
|
||||||
};
|
};
|
||||||
use crate::disk::{STORAGE_FORMAT_FILE, count_part_not_success};
|
use crate::disk::{STORAGE_FORMAT_FILE, count_part_not_success};
|
||||||
use crate::erasure::codec::bridge::{
|
use crate::erasure::codec::bridge::{
|
||||||
@@ -90,15 +84,14 @@ use crate::object_api::get_object_body_cache_hook;
|
|||||||
use crate::object_api::object_api_utils::get_raw_etag;
|
use crate::object_api::object_api_utils::get_raw_etag;
|
||||||
use crate::runtime::instance::{InstanceContext, bootstrap_ctx};
|
use crate::runtime::instance::{InstanceContext, bootstrap_ctx};
|
||||||
use crate::runtime::sources as runtime_sources;
|
use crate::runtime::sources as runtime_sources;
|
||||||
use crate::services::batch_processor::AsyncBatchProcessor;
|
#[cfg(test)]
|
||||||
|
use crate::storage_api_contracts::multipart::MultipartOperations;
|
||||||
use crate::storage_api_contracts::{
|
use crate::storage_api_contracts::{
|
||||||
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
|
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
|
||||||
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
|
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
|
||||||
multipart::{
|
multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
|
||||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations as _, MultipartUploadResult, PartInfo,
|
|
||||||
},
|
|
||||||
namespace::NamespaceLocking as _,
|
namespace::NamespaceLocking as _,
|
||||||
object::{DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
|
object::{DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectToDelete},
|
||||||
range::HTTPRangeSpec,
|
range::HTTPRangeSpec,
|
||||||
};
|
};
|
||||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||||
@@ -109,7 +102,7 @@ use crate::{
|
|||||||
disk::{
|
disk::{
|
||||||
CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, FileInfoVersions,
|
CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, FileInfoVersions,
|
||||||
RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
||||||
SnapshotLeaseToken, UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3, new_disk,
|
SnapshotLeaseToken, UpdateMetadataOpts, endpoint::Endpoint, error::DiskError, format::FormatV3,
|
||||||
},
|
},
|
||||||
error::{StorageError, to_object_err},
|
error::{StorageError, to_object_err},
|
||||||
object_api::{GetObjectReader, NamespaceLockFence, ObjectInfo, ObjectLockConfigSnapshot, PutObjReader},
|
object_api::{GetObjectReader, NamespaceLockFence, ObjectInfo, ObjectLockConfigSnapshot, PutObjReader},
|
||||||
@@ -122,18 +115,14 @@ use crate::{
|
|||||||
};
|
};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use bytesize::ByteSize;
|
use bytesize::ByteSize;
|
||||||
use chrono::Utc;
|
|
||||||
use futures::future::join_all;
|
use futures::future::join_all;
|
||||||
use futures::task::AtomicWaker;
|
|
||||||
use glob::Pattern;
|
|
||||||
use http::HeaderMap;
|
use http::HeaderMap;
|
||||||
use md5::{Digest as Md5Digest, Md5};
|
use md5::{Digest as Md5Digest, Md5};
|
||||||
use rand::{Rng, seq::SliceRandom};
|
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use rustfs_config::MI_B;
|
use rustfs_config::MI_B;
|
||||||
use rustfs_filemeta::{
|
use rustfs_filemeta::{
|
||||||
FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ObjectPartInfo,
|
FileInfo, FileMeta, FileMetaShallowVersion, MetaCacheEntries, MetaCacheEntry, ObjectPartInfo, RawFileInfo,
|
||||||
RawFileInfo, file_info_from_raw, merge_file_meta_versions,
|
merge_file_meta_versions,
|
||||||
};
|
};
|
||||||
use rustfs_heal_contracts::heal_channel::{
|
use rustfs_heal_contracts::heal_channel::{
|
||||||
DriveState, HealAdmissionResult, HealChannelPriority, HealItemType, HealOpts, HealRequestSource, HealScanMode,
|
DriveState, HealAdmissionResult, HealChannelPriority, HealItemType, HealOpts, HealRequestSource, HealScanMode,
|
||||||
@@ -144,9 +133,10 @@ use rustfs_io_metrics::{
|
|||||||
record_object_lock_diag_slow_acquire, record_object_lock_diag_slow_hold,
|
record_object_lock_diag_slow_acquire, record_object_lock_diag_slow_hold,
|
||||||
};
|
};
|
||||||
use rustfs_lock::LockClient;
|
use rustfs_lock::LockClient;
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_lock::LockManager;
|
||||||
use rustfs_lock::fast_lock::types::LockResult;
|
use rustfs_lock::fast_lock::types::LockResult;
|
||||||
use rustfs_lock::local_lock::LocalLock;
|
use rustfs_lock::{FastLockGuard, NamespaceLock, NamespaceLockGuard, NamespaceLockWrapper, ObjectKey};
|
||||||
use rustfs_lock::{FastLockGuard, LockManager, NamespaceLock, NamespaceLockGuard, NamespaceLockWrapper, ObjectKey};
|
|
||||||
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos};
|
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos};
|
||||||
use rustfs_object_capacity::capacity_scope::{
|
use rustfs_object_capacity::capacity_scope::{
|
||||||
CapacityScope, CapacityScopeDisk, current_dirty_generation, record_capacity_scope, record_global_dirty_scope,
|
CapacityScope, CapacityScopeDisk, current_dirty_generation, record_capacity_scope, record_global_dirty_scope,
|
||||||
@@ -158,7 +148,7 @@ use rustfs_utils::http::SSEC_ALGORITHM_HEADER;
|
|||||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||||
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
|
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
|
||||||
use rustfs_utils::http::headers::{
|
use rustfs_utils::http::headers::{
|
||||||
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES, HeaderExt as _,
|
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, EXPIRES,
|
||||||
};
|
};
|
||||||
use rustfs_utils::http::{
|
use rustfs_utils::http::{
|
||||||
SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE,
|
SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE,
|
||||||
@@ -171,30 +161,29 @@ use rustfs_utils::{
|
|||||||
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf},
|
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf},
|
||||||
};
|
};
|
||||||
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_RESTORE};
|
use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_RESTORE};
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::Sha256;
|
||||||
use std::future::Future;
|
|
||||||
use std::hash::{BuildHasher, Hash, Hasher};
|
use std::hash::{BuildHasher, Hash, Hasher};
|
||||||
use std::mem::{self};
|
use std::mem::{self};
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::sync::{Arc, OnceLock};
|
use std::sync::{Arc, OnceLock};
|
||||||
use std::task::{Context, Poll};
|
use std::task::{Context, Poll};
|
||||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||||
use std::{
|
use std::{
|
||||||
collections::{HashMap, HashSet},
|
collections::{HashMap, HashSet},
|
||||||
io::{Cursor, Write},
|
io::Cursor,
|
||||||
path::Path,
|
path::Path,
|
||||||
time::Duration,
|
time::Duration,
|
||||||
};
|
};
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
|
#[cfg(test)]
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
use tokio::sync::mpsc::Sender;
|
||||||
|
#[cfg(test)]
|
||||||
|
use tokio::time::timeout;
|
||||||
use tokio::{
|
use tokio::{
|
||||||
io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader, ReadBuf},
|
io::{AsyncRead, AsyncWrite, BufReader, ReadBuf},
|
||||||
sync::{RwLock, broadcast},
|
sync::RwLock,
|
||||||
};
|
|
||||||
use tokio::{
|
|
||||||
select,
|
|
||||||
sync::mpsc::{self, Sender},
|
|
||||||
time::{interval, timeout},
|
|
||||||
};
|
};
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
use tracing::error;
|
use tracing::error;
|
||||||
@@ -347,7 +336,7 @@ const ENV_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: &str = "RUSTFS_MULTIP
|
|||||||
const DEFAULT_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: usize = 128 * 1024 * 1024;
|
const DEFAULT_RUSTFS_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: usize = 128 * 1024 * 1024;
|
||||||
static CACHED_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: OnceLock<usize> = OnceLock::new();
|
static CACHED_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES: OnceLock<usize> = OnceLock::new();
|
||||||
|
|
||||||
use crate::io_support::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
|
use crate::io_support::rio::HashReader;
|
||||||
|
|
||||||
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
|
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
|
||||||
pub const MAX_PARTS_COUNT: usize = 10000;
|
pub const MAX_PARTS_COUNT: usize = 10000;
|
||||||
@@ -872,7 +861,7 @@ pub(crate) use ops::object::DeleteObjectCommitBarrier;
|
|||||||
#[cfg(feature = "test-util")]
|
#[cfg(feature = "test-util")]
|
||||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||||
pub(crate) use ops::object::body_cache_plaintext_len;
|
pub(crate) use ops::object::body_cache_plaintext_len;
|
||||||
#[cfg(test)]
|
#[cfg(all(test, feature = "test-util"))]
|
||||||
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
|
pub(crate) use ops::object::cleanup_rejected_transition_upload_durably;
|
||||||
#[cfg(any(test, feature = "test-util"))]
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
pub use ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||||
@@ -1029,8 +1018,7 @@ mod prepared_get_object_metadata_tests {
|
|||||||
use crate::ecstore_validation_blackbox::make_local_set_disks;
|
use crate::ecstore_validation_blackbox::make_local_set_disks;
|
||||||
use crate::object_api::{BLOCK_SIZE_V2, PutObjReader};
|
use crate::object_api::{BLOCK_SIZE_V2, PutObjReader};
|
||||||
use crate::set_disk::core::io_primitives::{bounded_metadata_fanout_order, disk_call_counters, rename_fanout_barrier};
|
use crate::set_disk::core::io_primitives::{bounded_metadata_fanout_order, disk_call_counters, rename_fanout_barrier};
|
||||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
|
||||||
use crate::test_metrics::CapturingRecorder;
|
use crate::test_metrics::CapturingRecorder;
|
||||||
use http::HeaderMap;
|
use http::HeaderMap;
|
||||||
use tokio::io::AsyncReadExt;
|
use tokio::io::AsyncReadExt;
|
||||||
@@ -5971,7 +5959,7 @@ mod tests {
|
|||||||
use crate::set_disk::core::io_primitives::rename_fanout_barrier;
|
use crate::set_disk::core::io_primitives::rename_fanout_barrier;
|
||||||
use crate::storage_api_contracts::{
|
use crate::storage_api_contracts::{
|
||||||
heal::HealOperations as _, lifecycle::TransitionedObject, list::ListOperations as _, multipart::CompletePart,
|
heal::HealOperations as _, lifecycle::TransitionedObject, list::ListOperations as _, multipart::CompletePart,
|
||||||
namespace::NamespaceLocking as _, object::ObjectIO as _, object::ObjectOperations as _,
|
object::ObjectOperations as _,
|
||||||
};
|
};
|
||||||
use crate::store::init_format::save_format_file;
|
use crate::store::init_format::save_format_file;
|
||||||
use crate::store::list_objects::ListPathOptions;
|
use crate::store::list_objects::ListPathOptions;
|
||||||
|
|||||||
@@ -12,7 +12,16 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use super::super::*;
|
use super::super::{
|
||||||
|
Cursor, DiskStore, EVENT_SET_DISK_WRITE, Error, FileInfo, HashAlgorithm, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK,
|
||||||
|
Result, join_all, warn,
|
||||||
|
};
|
||||||
|
use crate::disk::DiskAPI;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::RUSTFS_META_TMP_BUCKET;
|
||||||
|
use crate::set_disk::coding;
|
||||||
|
#[cfg(test)]
|
||||||
|
use bytes::Bytes;
|
||||||
|
|
||||||
/// Null out any disk whose shard writer failed (or was never created) so its
|
/// Null out any disk whose shard writer failed (or was never created) so its
|
||||||
/// truncated/absent shard is not committed by the final rename, and return the
|
/// truncated/absent shard is not committed by the final rename, and return the
|
||||||
@@ -132,7 +141,6 @@ pub(in crate::set_disk::ops) async fn verify_written_bitrot_shards(
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::super::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity;
|
use super::super::object::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity;
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::disk::DiskAPI as _;
|
|
||||||
|
|
||||||
async fn encode_streaming_shard(data: &[u8], shard_size: usize) -> Bytes {
|
async fn encode_streaming_shard(data: &[u8], shard_size: usize) -> Bytes {
|
||||||
let mut writer = coding::BitrotWriter::new(Cursor::new(Vec::new()), shard_size, HashAlgorithm::HighwayHash256S);
|
let mut writer = coding::BitrotWriter::new(Cursor::new(Vec::new()), shard_size, HashAlgorithm::HighwayHash256S);
|
||||||
|
|||||||
@@ -19,7 +19,12 @@
|
|||||||
//! `for SetDisks`, so its associated-type bounds are unchanged and runtime
|
//! `for SetDisks`, so its associated-type bounds are unchanged and runtime
|
||||||
//! behavior is the same.
|
//! behavior is the same.
|
||||||
|
|
||||||
use super::super::*;
|
use super::super::{
|
||||||
|
BUCKET_OP_IGNORED_ERRS, BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DiskError, Error, HashMap,
|
||||||
|
MakeBucketOptions, Result, SetDisks, is_reserved_or_invalid_bucket, join_all, reduce_write_quorum_errs,
|
||||||
|
};
|
||||||
|
use crate::api::bucket::metadata_sys;
|
||||||
|
use crate::disk::DiskAPI;
|
||||||
|
|
||||||
impl SetDisks {
|
impl SetDisks {
|
||||||
pub(crate) async fn list_bucket_for_scanner(&self, _opts: &BucketOptions) -> Result<(Vec<BucketInfo>, bool)> {
|
pub(crate) async fn list_bucket_for_scanner(&self, _opts: &BucketOptions) -> Result<(Vec<BucketInfo>, bool)> {
|
||||||
|
|||||||
@@ -12,9 +12,18 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use super::super::*;
|
use super::super::{
|
||||||
|
Bytes, CHECK_PART_FILE_CORRUPT, CHECK_PART_SUCCESS, DeleteOptions, DiskError, DiskStore, DriveState, EVENT_SET_DISK_HEAL,
|
||||||
|
Error, FileInfo, HashAlgorithm, HashMap, HealDriveInfo, HealItemType, HealOpts, HealResultItem, HealScanMode, Infos,
|
||||||
|
LOG_SUBSYSTEM_SET_DISK, ObjectInfo, ObjectOptions, ObjectPartInfo, Path, RUSTFS_META_TMP_BUCKET, ReadOptions, Result,
|
||||||
|
SLASH_SEPARATOR, SetDisks, StorageError, Uuid, coding, count_errs, count_part_not_success, create_bitrot_reader,
|
||||||
|
create_bitrot_writer, debug, disk, disks_with_all_parts, encode_dir_object, error, file_info_is_valid_for_metadata,
|
||||||
|
formats_match_reference_slots, get_format_erasure_in_quorum, get_lock_acquire_timeout, has_suffix,
|
||||||
|
heal_bucket_local_on_disks, is_object_dir_dangling, join_all, load_format_erasure_all, path_join_buf, save_format_file,
|
||||||
|
should_heal_object_on_disk, stat_all_dirs, to_object_err, warn,
|
||||||
|
};
|
||||||
use crate::disk::DataDirDeleteStatus;
|
use crate::disk::DataDirDeleteStatus;
|
||||||
use crate::disk::disk_store::DiskStoreRenameDataExt;
|
use crate::disk::DiskAPI;
|
||||||
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
|
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
|
||||||
use crate::io_support::bitrot::object_mmap_read_enabled;
|
use crate::io_support::bitrot::object_mmap_read_enabled;
|
||||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||||
|
|||||||
@@ -22,8 +22,14 @@
|
|||||||
//! every `(object, version)` present on ANY disk, feeding each to the existing
|
//! every `(object, version)` present on ANY disk, feeding each to the existing
|
||||||
//! per-version `SetDisks::heal_object`.
|
//! per-version `SetDisks::heal_object`.
|
||||||
|
|
||||||
use super::super::*;
|
use super::super::{
|
||||||
|
Arc, CancellationToken, DiskError, ListPathRawOptions, MetaCacheEntries, MetaCacheEntry, SetDisks, debug, disk, list_path_raw,
|
||||||
|
};
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::DiskAPI;
|
||||||
use crate::object_api::ObjectInfo;
|
use crate::object_api::ObjectInfo;
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_filemeta::FileMeta;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::sync::Mutex;
|
use std::sync::Mutex;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||||
|
|||||||
@@ -22,7 +22,11 @@
|
|||||||
//! runtime behavior is unchanged.
|
//! runtime behavior is unchanged.
|
||||||
|
|
||||||
use super::super::ctx::SetDisksCtx;
|
use super::super::ctx::SetDisksCtx;
|
||||||
use super::super::*;
|
use super::super::{
|
||||||
|
Arc, CancellationToken, DeleteOptions, DiskError, DiskStore, Error, ListObjectVersionsInfo, ListObjectsV2Info,
|
||||||
|
OBJECT_OP_IGNORED_ERRS, ObjectInfoOrErr, Result, Sender, SetDisks, WalkOptions, debug, join_all, reduce_write_quorum_errs,
|
||||||
|
};
|
||||||
|
use crate::disk::DiskAPI;
|
||||||
|
|
||||||
impl SetDisks {
|
impl SetDisks {
|
||||||
#[tracing::instrument(skip(self))]
|
#[tracing::instrument(skip(self))]
|
||||||
|
|||||||
@@ -19,9 +19,19 @@
|
|||||||
//! here; the contract stays implemented `for SetDisks`, so its associated-type
|
//! here; the contract stays implemented `for SetDisks`, so its associated-type
|
||||||
//! bounds are unchanged and helper access is via inherent calls.
|
//! bounds are unchanged and helper access is via inherent calls.
|
||||||
|
|
||||||
use super::super::*;
|
use super::super::{
|
||||||
|
Arc, DiskError, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, Endpoint, Error, FormatV3, HealChannelPriority, LockResult,
|
||||||
|
NamespaceLock, NamespaceLockWrapper, ObjectKey, Result, SetDisks, StorageError, debug, disk, info, load_format_erasure,
|
||||||
|
send_heal_disk, warn,
|
||||||
|
};
|
||||||
|
use crate::disk::DiskAPI;
|
||||||
use crate::disk::health_state::DriveMembershipSnapshot;
|
use crate::disk::health_state::DriveMembershipSnapshot;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::new_disk;
|
||||||
use crate::runtime::sources as runtime_sources;
|
use crate::runtime::sources as runtime_sources;
|
||||||
|
use rand::prelude::SliceRandom;
|
||||||
|
#[cfg(test)]
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
|
impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
|
||||||
|
|||||||
@@ -20,24 +20,79 @@
|
|||||||
//! contract stays implemented `for SetDisks`, so its associated-type bounds are
|
//! contract stays implemented `for SetDisks`, so its associated-type bounds are
|
||||||
//! unchanged; method bodies are moved verbatim and runtime behavior is the same.
|
//! unchanged; method bodies are moved verbatim and runtime behavior is the same.
|
||||||
|
|
||||||
use super::super::*;
|
#[cfg(test)]
|
||||||
|
use super::super::GetObjectMetadataCacheKey;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::super::MetadataCacheInvalidationProbe;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::super::capacity_scope_from_disks;
|
||||||
|
use super::super::{
|
||||||
|
AMZ_STORAGE_CLASS, Arc, Bytes, CompletePart, Cursor, DiskError, DiskStore, EVENT_SET_DISK_MULTIPART, Error, FileInfo,
|
||||||
|
GLOBAL_MIN_PART_SIZE, HashAlgorithm, HashMap, HashReader, HashSet, HealChannelPriority, Instant, LOG_COMPONENT_ECSTORE,
|
||||||
|
LOG_SUBSYSTEM_SET_DISK, ListMultipartsInfo, ListPartsInfo, MAX_PARTS_COUNT, MULTIPART_WRITE_QUORUM_RENAME_PART,
|
||||||
|
MULTIPART_WRITE_QUORUM_UPLOAD_METADATA, MULTIPART_WRITE_QUORUM_WRITER_SETUP, MultipartInfo, MultipartUploadResult,
|
||||||
|
MultipartWriteQuorumContext, NamespaceLockFence, OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectLockDiagGuard, ObjectOptions,
|
||||||
|
ObjectPartInfo, OffsetDateTime, PartInfo, PutObjReader, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
|
||||||
|
RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, Result, SLASH_SEPARATOR, SUFFIX_ACTUAL_OBJECT_SIZE_CAP,
|
||||||
|
SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC,
|
||||||
|
SUFFIX_RESTORE_OPERATION_ID, SetDisks, SmallWritePath, StorageError, Uuid, WriteLayout,
|
||||||
|
check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding, complete_multipart_part_error,
|
||||||
|
complete_multipart_part_error_result, complete_part_checksum, completed_multipart_object_part, contains_key_str,
|
||||||
|
create_bitrot_writer, debug, disk, error, get_complete_multipart_md5, get_header_map, get_str, insert_str,
|
||||||
|
is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size, log_multipart_write_quorum_failure,
|
||||||
|
parts_after_marker, path_join_buf, record_compression_total_memory, reduce_read_quorum_errs, reduce_write_quorum_errs,
|
||||||
|
remove_header_map, resolve_write_layout, restore_commit_operation_id_from_metadata, should_persist_encryption_original_size,
|
||||||
|
strip_internal_multipart_metadata, to_object_err, warn,
|
||||||
|
};
|
||||||
use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
|
use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::object::old_data_cleanup_receipt_path;
|
||||||
use super::object::{
|
use super::object::{
|
||||||
assign_object_transaction_epoch, object_transaction_fencing_fleet_proof, object_transaction_fencing_fleet_proof_matches,
|
assign_object_transaction_epoch, object_transaction_fencing_fleet_proof, object_transaction_fencing_fleet_proof_matches,
|
||||||
object_transaction_fencing_requested, old_data_cleanup_receipt_path, read_object_transaction_epoch_fence,
|
object_transaction_fencing_requested, read_object_transaction_epoch_fence, verify_object_transaction_epoch_fence,
|
||||||
verify_object_transaction_epoch_fence,
|
|
||||||
};
|
};
|
||||||
|
use crate::api::config::storageclass;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::bucket::metadata_sys::ObjectLockConfigState;
|
||||||
use crate::bucket::quota::reservation;
|
use crate::bucket::quota::reservation;
|
||||||
use crate::crash_inject::{self, CrashPoint};
|
use crate::crash_inject::{self, CrashPoint};
|
||||||
|
use crate::disk::DiskAPI;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::DiskOption;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::STORAGE_FORMAT_FILE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::new_disk;
|
||||||
use crate::multipart_listing::paginate_multipart_listing;
|
use crate::multipart_listing::paginate_multipart_listing;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::object_api::ObjectLockConfigSnapshot;
|
||||||
use crate::set_disk::core::io_primitives::finish_rename_tail_heal;
|
use crate::set_disk::core::io_primitives::finish_rename_tail_heal;
|
||||||
|
use crate::set_disk::mem;
|
||||||
|
use crate::set_disk::metadata_sys;
|
||||||
|
use crate::set_disk::runtime_sources;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::storage_api_contracts::multipart::MultipartOperations;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::storage_api_contracts::object::HTTPPreconditions;
|
||||||
|
use crate::storage_api_contracts::object::ObjectOperations;
|
||||||
use futures::{StreamExt, stream};
|
use futures::{StreamExt, stream};
|
||||||
|
#[cfg(test)]
|
||||||
|
use http::HeaderMap;
|
||||||
|
use rustfs_rio::EtagResolvable;
|
||||||
|
use rustfs_rio::TryGetIndex;
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_utils::http::SSEC_ALGORITHM_HEADER;
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_utils::http::SUFFIX_COMPRESSION;
|
||||||
use std::future::Future;
|
use std::future::Future;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use std::sync::atomic::AtomicBool;
|
use std::sync::atomic::AtomicBool;
|
||||||
#[cfg(any(test, feature = "test-util"))]
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
#[cfg(test)]
|
||||||
|
use tokio::io::AsyncReadExt;
|
||||||
use tokio::task::JoinSet;
|
use tokio::task::JoinSet;
|
||||||
|
|
||||||
const MULTIPART_LIST_IO_CONCURRENCY: usize = 16;
|
const MULTIPART_LIST_IO_CONCURRENCY: usize = 16;
|
||||||
@@ -2998,7 +3053,7 @@ fn resolve_complete_etag(opts: &ObjectOptions, uploaded_parts: &[CompletePart])
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::config::storageclass::lookup_config_for_pools_without_env;
|
use crate::config::storageclass::lookup_config_for_pools_without_env;
|
||||||
use crate::disk::{DiskAPI as _, ReadOptions};
|
use crate::disk::ReadOptions;
|
||||||
use crate::disk::{endpoint::Endpoint, format::FormatV3};
|
use crate::disk::{endpoint::Endpoint, format::FormatV3};
|
||||||
use crate::layout::endpoints::SetupType;
|
use crate::layout::endpoints::SetupType;
|
||||||
use crate::services::notification_sys::install_remote_version_state_fleet_proof_for_test;
|
use crate::services::notification_sys::install_remote_version_state_fleet_proof_for_test;
|
||||||
@@ -3014,7 +3069,7 @@ mod tests {
|
|||||||
};
|
};
|
||||||
use crate::set_disk::ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
use crate::set_disk::ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||||
use rustfs_config::server_config::KVS;
|
use rustfs_config::server_config::KVS;
|
||||||
use rustfs_lock::{LockClient, client::local::LocalClient};
|
use rustfs_lock::{LockClient, client::local::LocalClient};
|
||||||
use serial_test::serial;
|
use serial_test::serial;
|
||||||
@@ -4870,7 +4925,7 @@ mod tests {
|
|||||||
#[serial(metadata_cache_invalidation_probe)]
|
#[serial(metadata_cache_invalidation_probe)]
|
||||||
async fn complete_multipart_generation_retires_cached_snapshot() {
|
async fn complete_multipart_generation_retires_cached_snapshot() {
|
||||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||||
|
|
||||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||||
let bucket = "multipart-metadata-generation-bucket";
|
let bucket = "multipart-metadata-generation-bucket";
|
||||||
@@ -7369,9 +7424,7 @@ mod tests {
|
|||||||
mod crash_consistency {
|
mod crash_consistency {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::crash_inject::{self, CrashPoint};
|
use crate::crash_inject::{self, CrashPoint};
|
||||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
|
||||||
use http::HeaderMap;
|
use http::HeaderMap;
|
||||||
use tokio::io::AsyncReadExt as _;
|
|
||||||
|
|
||||||
/// 1 MiB keeps every object off the 128 KiB inline fast path, so the
|
/// 1 MiB keeps every object off the 128 KiB inline fast path, so the
|
||||||
/// commit moves real erasure shards through `rename_data`.
|
/// commit moves real erasure shards through `rename_data`.
|
||||||
|
|||||||
@@ -19,11 +19,74 @@
|
|||||||
//! bounds are unchanged, and the impls reach shared primitives through the
|
//! bounds are unchanged, and the impls reach shared primitives through the
|
||||||
//! SetDisks core (io_primitives) via inherent calls.
|
//! SetDisks core (io_primitives) via inherent calls.
|
||||||
|
|
||||||
use super::super::*;
|
#[cfg(test)]
|
||||||
|
use super::super::MetadataCacheInvalidationProbe;
|
||||||
|
use super::super::{
|
||||||
|
AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS, Arc, AsyncWrite, AtomicU64, BufReader, Bytes, CACHE_CONTROL, CONTENT_DISPOSITION,
|
||||||
|
CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, CompletePart, Cursor, DeleteAccounting, DeleteOptions, DeletedObject,
|
||||||
|
DiskError, DiskStore, EVENT_SET_DISK_COMMIT_TAIL_SLOW, EVENT_SET_DISK_PUT_OBJECT_STAGE_SUMMARY, EVENT_SET_DISK_WRITE,
|
||||||
|
EXPIRES, Error, EventArgs, EventName, FastLockGuard, FileInfo, FileInfoVersions,
|
||||||
|
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, GET_OBJECT_PATH_BODY_CACHE, GET_OBJECT_PATH_CODEC_STREAMING,
|
||||||
|
GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_EMPTY, GET_OBJECT_PATH_INLINE_DIRECT, GET_OBJECT_PATH_INTERNAL_META,
|
||||||
|
GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_REMOTE_TRANSITION, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_EMIT,
|
||||||
|
GET_STAGE_INLINE_PREPARE, GET_STAGE_LOCK_ACQUIRE, GET_STAGE_METADATA, GET_STAGE_OBJECT_INFO, GET_STAGE_PATH_DECISION,
|
||||||
|
GET_STAGE_READER_SETUP, GenericError, GetCodecStreamingDecision, GetDirectMemoryDecision, GetObjectReader, HTTPRangeSpec,
|
||||||
|
HashAlgorithm, HashMap, HashReader, HashSet, HeaderMap, HealChannelPriority, InstanceContext, Instant, LOG_COMPONENT_ECSTORE,
|
||||||
|
LOG_SUBSYSTEM_SET_DISK, OBJECT_OP_IGNORED_ERRS, ObjectApiError, ObjectInfo, ObjectKey, ObjectLockConfigSnapshot,
|
||||||
|
ObjectLockConfigState, ObjectOptions, ObjectReader, ObjectToDelete, OffsetDateTime, Ordering, Pin, PutObjReader,
|
||||||
|
RUSTFS_META_BUCKET, RUSTFS_META_TMP_BUCKET, ReaderImpl, ReplicateDecision, ReplicationObjectBridge, Result,
|
||||||
|
SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS, SLASH_SEPARATOR, SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_COMPRESSION_SIZE,
|
||||||
|
SUFFIX_RESTORE_OPERATION_ID, SetDisks, SmallWritePath, StorageError, TRANSITION_COMPLETE, UpdateMetadataOpts, Uuid,
|
||||||
|
WriteLayout, X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, X_AMZ_RESTORE,
|
||||||
|
adaptive_duplex_buffer_size, build_get_object_info, build_inline_bitrot_readers, build_inline_bitrot_readers_from_refs,
|
||||||
|
can_try_inline_data_shards_direct, check_object_lock_delete, check_object_lock_for_deletion_with_state,
|
||||||
|
check_object_lock_retention_update, classify_get_codec_streaming_object_class, classify_put_write_path,
|
||||||
|
classify_storage_error, collect_inline_data_shard_fileinfos_by_index, contains_key_str, create_bitrot_writer, debug,
|
||||||
|
delete_file_info_version_id, disk, ensure_delete_commit_locks_held, error, finish_set_disk_read_lock,
|
||||||
|
get_codec_streaming_reader_gate, get_object_body_cache_hook, get_raw_etag, get_small_object_direct_memory_decision,
|
||||||
|
get_stage_timer_if_enabled, get_str, get_transitioned_object_reader_with_tier_manager, inline_erasure_shard_file_offset,
|
||||||
|
inline_erasure_shard_size, insert_str, is_deadlock_detection_enabled, is_err_object_not_found, is_err_version_not_found,
|
||||||
|
is_explicit_null_version, is_lock_optimization_enabled, issue3031_diag_enabled, join_all, known_put_object_storage_size,
|
||||||
|
path_join_buf, put_restore_opts, record_compression_total_memory, record_get_codec_streaming_gate_decision,
|
||||||
|
record_get_direct_memory_decision, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path,
|
||||||
|
record_get_object_reader_path_observation, record_get_stage_duration_if_enabled, record_lock_acquire,
|
||||||
|
reduce_write_quorum_errs, release_materialized_read_lock, replication_write_may_pass_worm_gate, require_restore_operation_id,
|
||||||
|
resolve_delete_version_state, resolve_tiered_decommission_write_quorum_result, resolve_write_layout,
|
||||||
|
restore_commit_operation_id_from_metadata, restore_operation_id_from_metadata, send_event,
|
||||||
|
set_disk_delete_creates_delete_marker, should_force_delete_marker_for_missing_version,
|
||||||
|
should_persist_encryption_original_size, should_preserve_delete_replication_state, should_use_inline_fast_path,
|
||||||
|
take_prepared_get_object_metadata, to_object_err, try_read_inline_data_shards_direct, warn,
|
||||||
|
};
|
||||||
use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
|
use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
|
||||||
|
use crate::api::config::storageclass;
|
||||||
|
use crate::bucket::lifecycle::bucket_lifecycle_ops::LifecycleOps;
|
||||||
use crate::bucket::utils::is_meta_bucketname;
|
use crate::bucket::utils::is_meta_bucketname;
|
||||||
|
use crate::bucket::versioning::VersioningApi;
|
||||||
|
use crate::disk::DiskAPI;
|
||||||
|
use crate::set_disk::coding;
|
||||||
|
use crate::set_disk::core::io_primitives::GetCodecStreamingReaderBuildOutcome;
|
||||||
|
use crate::set_disk::mem;
|
||||||
|
use crate::set_disk::metadata_sys;
|
||||||
use crate::set_disk::read::GetObjectDownstreamWriter;
|
use crate::set_disk::read::GetObjectDownstreamWriter;
|
||||||
|
use crate::set_disk::runtime_sources;
|
||||||
|
use crate::storage_api_contracts::multipart::MultipartOperations;
|
||||||
|
use crate::storage_api_contracts::object::ObjectIO;
|
||||||
|
use crate::storage_api_contracts::object::ObjectOperations;
|
||||||
|
use rustfs_lock::LockManager;
|
||||||
|
use rustfs_rio::EtagResolvable;
|
||||||
|
use rustfs_rio::HashReaderMut;
|
||||||
|
use rustfs_rio::TryGetIndex;
|
||||||
|
use rustfs_utils::http::HeaderExt;
|
||||||
|
use tokio::io::AsyncWriteExt;
|
||||||
|
|
||||||
|
#[cfg(all(test, feature = "test-util"))]
|
||||||
|
use super::super::GetObjectMetadataCacheEntry;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::super::GetObjectMetadataCacheKey;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::super::capacity_scope_from_disks;
|
||||||
|
#[cfg(all(test, feature = "test-util"))]
|
||||||
|
use super::super::get_lock_acquire_timeout;
|
||||||
use crate::bucket::lifecycle::{
|
use crate::bucket::lifecycle::{
|
||||||
tier_delete_journal::{
|
tier_delete_journal::{
|
||||||
enqueue_committed_tier_delete_journal_entry, persist_tier_delete_journal_entry,
|
enqueue_committed_tier_delete_journal_entry, persist_tier_delete_journal_entry,
|
||||||
@@ -47,6 +110,20 @@ use crate::bucket::replication::{
|
|||||||
};
|
};
|
||||||
use crate::data_usage::quota_object_size;
|
use crate::data_usage::quota_object_size;
|
||||||
use crate::diagnostics::get::GetObjectFailureReason;
|
use crate::diagnostics::get::GetObjectFailureReason;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::DiskOption;
|
||||||
|
#[cfg(all(test, feature = "test-util"))]
|
||||||
|
use crate::disk::RUSTFS_META_MULTIPART_BUCKET;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::ReadOptions;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::STORAGE_FORMAT_FILE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::endpoint::Endpoint;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::format::FormatV3;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::new_disk;
|
||||||
use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
|
use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
|
||||||
use crate::error::is_err_invalid_upload_id;
|
use crate::error::is_err_invalid_upload_id;
|
||||||
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
|
use crate::object_api::{GetObjectBodySource, get_object_body_cache_hook_suppressed};
|
||||||
@@ -54,15 +131,30 @@ use crate::object_api::{NamespaceLockFence, SCANNER_PUBLICATION_LEASE_FENCE_META
|
|||||||
use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
|
use crate::services::notification_sys::RemoteVersionStateFleetProofToken;
|
||||||
use crate::services::tier::tier::{TierConfigMgr, TierOperationLease};
|
use crate::services::tier::tier::{TierConfigMgr, TierOperationLease};
|
||||||
use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal};
|
use crate::set_disk::core::io_primitives::{RenameTailCleanup, finish_rename_tail_heal};
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::storage_api_contracts::namespace::NamespaceLocking;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::storage_api_contracts::object::HTTPPreconditions;
|
||||||
use crate::store::ECStore;
|
use crate::store::ECStore;
|
||||||
use crate::store::utils::clean_metadata;
|
use crate::store::utils::clean_metadata;
|
||||||
use futures::FutureExt as _;
|
use futures::FutureExt as _;
|
||||||
use http::HeaderValue;
|
use http::HeaderValue;
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_filemeta::FileMeta;
|
||||||
|
#[cfg(all(test, feature = "test-util"))]
|
||||||
|
use rustfs_filemeta::ObjectPartInfo;
|
||||||
use rustfs_utils::path::decode_dir_object;
|
use rustfs_utils::path::decode_dir_object;
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_utils::path::encode_dir_object;
|
||||||
use std::future::Future;
|
use std::future::Future;
|
||||||
use std::sync::OnceLock;
|
|
||||||
use std::task::{Context, Poll};
|
use std::task::{Context, Poll};
|
||||||
|
#[cfg(any(test, feature = "test-util"))]
|
||||||
|
use std::time::Duration;
|
||||||
|
#[cfg(test)]
|
||||||
|
use tokio::io::AsyncReadExt;
|
||||||
use tokio::io::{AsyncRead, ReadBuf};
|
use tokio::io::{AsyncRead, ReadBuf};
|
||||||
|
#[cfg(all(test, feature = "test-util"))]
|
||||||
|
use tokio::sync::RwLock;
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json";
|
const OLD_DATA_CLEANUP_RECEIPT_FILE: &str = ".rustfs-old-data-cleanup-receipt.json";
|
||||||
@@ -1066,11 +1158,6 @@ mod restore_metadata_update_tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod delete_replication_transport_tests {
|
|
||||||
use super::*;
|
|
||||||
}
|
|
||||||
|
|
||||||
fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Erasure> {
|
fn erasure_from_file_info(fi: &FileInfo, uses_legacy: bool) -> Result<coding::Erasure> {
|
||||||
coding::Erasure::try_new_with_options(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size, uses_legacy)
|
coding::Erasure::try_new_with_options(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size, uses_legacy)
|
||||||
.map_err(Error::from)
|
.map_err(Error::from)
|
||||||
@@ -1893,7 +1980,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
|||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
{
|
{
|
||||||
core::io_primitives::GetCodecStreamingReaderBuildOutcome::Reader(stream) => {
|
GetCodecStreamingReaderBuildOutcome::Reader(stream) => {
|
||||||
record_get_codec_streaming_gate_decision(
|
record_get_codec_streaming_gate_decision(
|
||||||
codec_streaming_gate.object_class,
|
codec_streaming_gate.object_class,
|
||||||
GetCodecStreamingDecision::Use,
|
GetCodecStreamingDecision::Use,
|
||||||
@@ -1907,7 +1994,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
|||||||
reader.body_source = body_source;
|
reader.body_source = body_source;
|
||||||
return Ok(finish_set_disk_read_lock(reader, read_lock_guard.take(), bucket, object));
|
return Ok(finish_set_disk_read_lock(reader, read_lock_guard.take(), bucket, object));
|
||||||
}
|
}
|
||||||
core::io_primitives::GetCodecStreamingReaderBuildOutcome::Fallback(reason) => {
|
GetCodecStreamingReaderBuildOutcome::Fallback(reason) => {
|
||||||
record_get_codec_streaming_gate_decision(
|
record_get_codec_streaming_gate_decision(
|
||||||
codec_streaming_gate.object_class,
|
codec_streaming_gate.object_class,
|
||||||
GetCodecStreamingDecision::Fallback(reason),
|
GetCodecStreamingDecision::Fallback(reason),
|
||||||
@@ -5066,7 +5153,7 @@ pub(in crate::set_disk::ops) fn object_transaction_fencing_requested() -> bool {
|
|||||||
|
|
||||||
#[cfg(not(test))]
|
#[cfg(not(test))]
|
||||||
fn object_transaction_fencing_requested_cached() -> bool {
|
fn object_transaction_fencing_requested_cached() -> bool {
|
||||||
static REQUESTED: OnceLock<bool> = OnceLock::new();
|
static REQUESTED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
|
||||||
*REQUESTED.get_or_init(load_object_transaction_fencing_requested)
|
*REQUESTED.get_or_init(load_object_transaction_fencing_requested)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -8584,7 +8671,6 @@ mod replication_lww_tests {
|
|||||||
|
|
||||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
|
||||||
use rustfs_utils::http::headers::{
|
use rustfs_utils::http::headers::{
|
||||||
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
|
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, AMZ_OBJECT_TAGGING,
|
||||||
};
|
};
|
||||||
@@ -9212,8 +9298,7 @@ mod inline_put_commit_path_tests {
|
|||||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::config::storageclass::lookup_config_for_pools_without_env;
|
use crate::config::storageclass::lookup_config_for_pools_without_env;
|
||||||
use crate::disk::{DiskAPI as _, ReadOptions};
|
use crate::disk::ReadOptions;
|
||||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
|
||||||
use rustfs_config::server_config::KVS;
|
use rustfs_config::server_config::KVS;
|
||||||
use serial_test::serial;
|
use serial_test::serial;
|
||||||
use tokio::io::AsyncReadExt;
|
use tokio::io::AsyncReadExt;
|
||||||
@@ -9606,7 +9691,6 @@ mod get_object_downstream_close_accounting_tests {
|
|||||||
};
|
};
|
||||||
use crate::disk::RUSTFS_META_BUCKET;
|
use crate::disk::RUSTFS_META_BUCKET;
|
||||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
|
||||||
use crate::test_metrics::CapturingRecorder;
|
use crate::test_metrics::CapturingRecorder;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
@@ -9892,8 +9976,7 @@ mod get_object_downstream_close_accounting_tests {
|
|||||||
mod metadata_mutation_generation_tests {
|
mod metadata_mutation_generation_tests {
|
||||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::disk::{DiskAPI as _, ReadOptions};
|
use crate::disk::ReadOptions;
|
||||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
|
||||||
|
|
||||||
async fn put_and_prime(
|
async fn put_and_prime(
|
||||||
set_disks: &Arc<SetDisks>,
|
set_disks: &Arc<SetDisks>,
|
||||||
@@ -10153,15 +10236,12 @@ mod transition_commit_failure_tests {
|
|||||||
use super::hermetic_set_disks_support::hermetic_set_disks;
|
use super::hermetic_set_disks_support::hermetic_set_disks;
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions};
|
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions};
|
||||||
use crate::disk::DiskAPI as _;
|
|
||||||
use crate::services::tier::test_util::{MockWarmBackend, register_mock_tier};
|
use crate::services::tier::test_util::{MockWarmBackend, register_mock_tier};
|
||||||
use crate::services::tier::tier::TierConfigMgr;
|
use crate::services::tier::tier::TierConfigMgr;
|
||||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
|
||||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
|
||||||
use http::HeaderMap;
|
use http::HeaderMap;
|
||||||
use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
|
use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
|
||||||
use s3s::dto::RestoreRequest;
|
use s3s::dto::RestoreRequest;
|
||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
use tokio::io::AsyncReadExt;
|
||||||
|
|
||||||
pub(super) fn restore_operation_id_metadata(operation_id: Uuid) -> HashMap<String, String> {
|
pub(super) fn restore_operation_id_metadata(operation_id: Uuid) -> HashMap<String, String> {
|
||||||
let mut metadata = HashMap::new();
|
let mut metadata = HashMap::new();
|
||||||
@@ -11907,11 +11987,9 @@ mod transition_upload_integrity_tests {
|
|||||||
use super::transition_commit_failure_tests::{restore_metadata, restore_operation_id_metadata};
|
use super::transition_commit_failure_tests::{restore_metadata, restore_operation_id_metadata};
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions};
|
use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions};
|
||||||
use crate::disk::DiskAPI as _;
|
|
||||||
use crate::layout::endpoints::SetupType;
|
use crate::layout::endpoints::SetupType;
|
||||||
use crate::services::tier::test_util::register_mock_tier;
|
use crate::services::tier::test_util::register_mock_tier;
|
||||||
use crate::set_disk::replication::RestoreFinalizeBarrier;
|
use crate::set_disk::replication::RestoreFinalizeBarrier;
|
||||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
|
||||||
use http::HeaderMap;
|
use http::HeaderMap;
|
||||||
use rustfs_filemeta::RestoreStatusOps as _;
|
use rustfs_filemeta::RestoreStatusOps as _;
|
||||||
use rustfs_lock::client::local::LocalClient;
|
use rustfs_lock::client::local::LocalClient;
|
||||||
@@ -13665,9 +13743,7 @@ mod transition_source_identity_matrix_tests {
|
|||||||
use super::hermetic_set_disks_support::hermetic_set_disks;
|
use super::hermetic_set_disks_support::hermetic_set_disks;
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions};
|
use crate::bucket::lifecycle::lifecycle::{TRANSITION_PENDING, TransitionOptions};
|
||||||
use crate::disk::DiskAPI as _;
|
|
||||||
use crate::services::tier::test_util::register_mock_tier;
|
use crate::services::tier::test_util::register_mock_tier;
|
||||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn transition_source_identity_treats_nil_version_as_null_source() {
|
fn transition_source_identity_treats_nil_version_as_null_source() {
|
||||||
@@ -13913,7 +13989,7 @@ mod heterogeneous_pool_put_tests {
|
|||||||
};
|
};
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::config::storageclass::lookup_config_for_pools_without_env;
|
use crate::config::storageclass::lookup_config_for_pools_without_env;
|
||||||
use crate::disk::{DiskAPI as _, ReadOptions};
|
use crate::disk::ReadOptions;
|
||||||
use crate::services::notification_sys::install_remote_version_state_fleet_proof_for_test;
|
use crate::services::notification_sys::install_remote_version_state_fleet_proof_for_test;
|
||||||
use rustfs_config::server_config::KVS;
|
use rustfs_config::server_config::KVS;
|
||||||
use serial_test::serial;
|
use serial_test::serial;
|
||||||
@@ -14337,7 +14413,6 @@ mod put_object_tmp_cleanup_tests {
|
|||||||
|
|
||||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::disk::DiskAPI as _;
|
|
||||||
use crate::set_disk::core::io_primitives::{
|
use crate::set_disk::core::io_primitives::{
|
||||||
ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier, rename_fault_injection,
|
ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier, rename_fault_injection,
|
||||||
};
|
};
|
||||||
@@ -15954,7 +16029,7 @@ mod put_object_tags_early_stop_regression_tests {
|
|||||||
|
|
||||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::disk::{DiskAPI as _, ReadOptions};
|
use crate::disk::ReadOptions;
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn put_object_tags_writes_all_online_disks_under_early_stop() {
|
async fn put_object_tags_writes_all_online_disks_under_early_stop() {
|
||||||
@@ -16041,9 +16116,7 @@ mod put_object_tags_early_stop_regression_tests {
|
|||||||
mod object_tagging_namespace_lock_tests {
|
mod object_tagging_namespace_lock_tests {
|
||||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::disk::{DiskAPI as _, ReadOptions};
|
use crate::disk::ReadOptions;
|
||||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
|
||||||
use tokio::io::AsyncReadExt as _;
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug)]
|
#[derive(Clone, Copy, Debug)]
|
||||||
enum CompetingMutation {
|
enum CompetingMutation {
|
||||||
@@ -16315,7 +16388,6 @@ mod delete_objects_lock_gating_tests {
|
|||||||
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
|
||||||
use super::hermetic_set_disks_support::hermetic_set_disks_with_lockers_and_ctx;
|
use super::hermetic_set_disks_support::hermetic_set_disks_with_lockers_and_ctx;
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::disk::DiskAPI as _;
|
|
||||||
use serial_test::serial;
|
use serial_test::serial;
|
||||||
|
|
||||||
async fn put_plain_object(set_disks: &Arc<SetDisks>, bucket: &str, object: &str) {
|
async fn put_plain_object(set_disks: &Arc<SetDisks>, bucket: &str, object: &str) {
|
||||||
|
|||||||
@@ -12,7 +12,16 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use super::*;
|
use super::{
|
||||||
|
Arc, Bytes, DiskError, DiskStore, ErasureCache, Error, FileInfo, GetCodecStreamingFallbackReason, GetObjectFileInfo,
|
||||||
|
GetObjectMetadataCacheEntry, GetObjectMetadataCacheGeneration, GetObjectMetadataCacheKey, GetObjectReadPolicy, HashAlgorithm,
|
||||||
|
LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectOptions, RUSTFS_META_BUCKET,
|
||||||
|
ReadOptions, Result, SetDisks, StorageError, adaptive_duplex_buffer_size, build_get_codec_streaming_decode_engine,
|
||||||
|
build_inline_bitrot_readers_from_refs, collect_inline_data_shard_fileinfos_by_index, debug, error,
|
||||||
|
get_codec_streaming_metrics_path, get_codec_streaming_multipart_max_parts, get_object_read_policy,
|
||||||
|
is_codec_streaming_multipart_enabled, is_multipart_reader_setup_prefetch_enabled, object_fits_single_block,
|
||||||
|
reduce_read_quorum_errs, to_object_err, try_read_inline_data_shards_direct, warn,
|
||||||
|
};
|
||||||
use crate::diagnostics::get::{
|
use crate::diagnostics::get::{
|
||||||
GET_DIRECT_MEMORY_SUBPATH_DISK_DATA_BLOCKS, GET_DIRECT_MEMORY_SUBPATH_INLINE_BUFFERED, GET_METADATA_CACHE_DECISION_HIT,
|
GET_DIRECT_MEMORY_SUBPATH_DISK_DATA_BLOCKS, GET_DIRECT_MEMORY_SUBPATH_INLINE_BUFFERED, GET_METADATA_CACHE_DECISION_HIT,
|
||||||
GET_METADATA_CACHE_DECISION_MISS, GET_METADATA_CACHE_DECISION_REJECT, GET_METADATA_CACHE_DECISION_SKIP,
|
GET_METADATA_CACHE_DECISION_MISS, GET_METADATA_CACHE_DECISION_REJECT, GET_METADATA_CACHE_DECISION_SKIP,
|
||||||
@@ -22,43 +31,152 @@ use crate::diagnostics::get::{
|
|||||||
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_CHECKSUMS, GET_METADATA_CACHE_REASON_PART_NUMBER,
|
GET_METADATA_CACHE_REASON_NOT_READ_DATA, GET_METADATA_CACHE_REASON_PART_CHECKSUMS, GET_METADATA_CACHE_REASON_PART_NUMBER,
|
||||||
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_STALE_PUBLICATION,
|
GET_METADATA_CACHE_REASON_RAW_DATA_MOVEMENT_READ, GET_METADATA_CACHE_REASON_STALE_PUBLICATION,
|
||||||
GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, GET_METADATA_CACHE_REASON_VERSION_SUSPENDED,
|
GET_METADATA_CACHE_REASON_USABLE, GET_METADATA_CACHE_REASON_VERSION_ID, GET_METADATA_CACHE_REASON_VERSION_SUSPENDED,
|
||||||
GET_METADATA_CACHE_REASON_VERSIONED, GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA,
|
GET_METADATA_CACHE_REASON_VERSIONED, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META,
|
||||||
GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR,
|
GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE, GET_STAGE_METADATA_CACHE_LOOKUP,
|
||||||
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND,
|
GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP, GET_STAGE_READER_TASK_BITROT_READER_INIT,
|
||||||
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
|
GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, GetObjectFailureReason, classify_disk_error,
|
||||||
GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
|
get_stage_timer_if_enabled, mark_get_object_downstream_closed, record_get_object_pipeline_failure,
|
||||||
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
|
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
||||||
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
|
|
||||||
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_CODEC_STREAMING, GET_OBJECT_PATH_DIRECT_MEMORY,
|
|
||||||
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_OBJECT_PATH_SET_DISK, GET_STAGE_DECODE,
|
|
||||||
GET_STAGE_METADATA_CACHE_LOOKUP, GET_STAGE_METADATA_RESOLVE, GET_STAGE_RANGE, GET_STAGE_READER_SETUP,
|
|
||||||
GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM,
|
|
||||||
GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION,
|
|
||||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, mark_get_object_downstream_closed,
|
|
||||||
record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
|
||||||
};
|
|
||||||
use crate::erasure::coding::BitrotReader;
|
|
||||||
use crate::io_support::bitrot::{
|
|
||||||
BitrotReaderStageMetrics, DeferredReaderStripeHandle, create_bitrot_reader_with_stage_metrics, create_deferred_bitrot_reader,
|
|
||||||
object_mmap_read_enabled,
|
|
||||||
};
|
};
|
||||||
|
use crate::disk::DiskAPI;
|
||||||
|
use crate::io_support::bitrot::{BitrotReaderStageMetrics, DeferredReaderStripeHandle, object_mmap_read_enabled};
|
||||||
|
use crate::set_disk::coding;
|
||||||
|
use crate::set_disk::runtime_sources;
|
||||||
use crate::set_disk::shard_source::ShardReadCost;
|
use crate::set_disk::shard_source::ShardReadCost;
|
||||||
use futures::stream::{FuturesUnordered, StreamExt};
|
|
||||||
use metrics::counter;
|
|
||||||
use std::{
|
use std::{
|
||||||
collections::{HashMap, VecDeque},
|
|
||||||
future::Future,
|
future::Future,
|
||||||
io::IoSlice,
|
io::IoSlice,
|
||||||
pin::Pin,
|
pin::Pin,
|
||||||
sync::OnceLock,
|
|
||||||
task::{Context, Poll},
|
task::{Context, Poll},
|
||||||
time::{Duration, Instant},
|
time::{Duration, Instant},
|
||||||
};
|
};
|
||||||
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
|
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
|
||||||
use tokio::sync::RwLock;
|
|
||||||
use tokio::task::JoinSet;
|
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::DEFAULT_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::DEFAULT_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_MAX_SIZE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_MAX_PARTS;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT_PCT;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_CODEC_STREAMING_RUSTFS_MIN_SIZE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_MULTIPART_READER_SETUP_PREFETCH;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::ENV_RUSTFS_GET_OBJECT_METADATA_CACHE_MAX_ENTRIES;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::GET_OBJECT_METADATA_CACHE_TTL;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::GetCodecStreamingConfig;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::GetCodecStreamingDecision;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::GetCodecStreamingEngine;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::GetCodecStreamingGate;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::GetCodecStreamingObjectClass;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::GetCodecStreamingRollout;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::classify_get_codec_streaming_object_class;
|
||||||
use super::core::io_primitives::*;
|
use super::core::io_primitives::*;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::get_codec_streaming_config_cached_core;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::get_codec_streaming_engine;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::get_codec_streaming_reader_gate;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::get_object_metadata_cache_max_entries;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::is_get_metadata_data_read_early_stop_enabled;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::is_get_metadata_early_stop_bounded_fanout_enabled;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::is_get_metadata_early_stop_enabled;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::is_version_early_stop_enabled;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::load_get_codec_streaming_config;
|
||||||
|
#[cfg(test)]
|
||||||
|
use super::with_get_object_read_policy;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::diagnostics::get::GET_OBJECT_PATH_CODEC_STREAMING_LEGACY_ENGINE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::diagnostics::get::GET_OBJECT_PATH_CODEC_STREAMING_RUSTFS_ENGINE;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::diagnostics::get::{
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
|
||||||
|
GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND,
|
||||||
|
GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT,
|
||||||
|
GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND,
|
||||||
|
};
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::ReadMultipleResp;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::disk::format::FormatV3;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::erasure::codec::bridge::CodecStreamingDecodeEngine;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::erasure::codec::bridge::GET_CODEC_STREAMING_ENGINE_RUSTFS;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::object_api::PutObjReader;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::storage_api_contracts::object::ObjectIO;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_filemeta::ObjectPartInfo;
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_heal_contracts::heal_channel::HealAdmissionResult;
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_heal_contracts::heal_channel::HealChannelPriority;
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_utils::http::SUFFIX_COMPRESSION;
|
||||||
|
#[cfg(test)]
|
||||||
|
use rustfs_utils::http::insert_str;
|
||||||
|
#[cfg(test)]
|
||||||
|
use time::OffsetDateTime;
|
||||||
|
#[cfg(test)]
|
||||||
|
use tokio::sync::RwLock;
|
||||||
|
#[cfg(test)]
|
||||||
|
use tokio::time::timeout;
|
||||||
|
#[cfg(test)]
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
pub(super) struct GetObjectDownstreamWriter<W> {
|
pub(super) struct GetObjectDownstreamWriter<W> {
|
||||||
inner: W,
|
inner: W,
|
||||||
|
|||||||
@@ -12,11 +12,16 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use super::*;
|
use super::{
|
||||||
|
Error, FileInfo, NamespaceLockFence, ObjectInfo, ObjectOptions, OffsetDateTime, Result, SetDisks, StorageError,
|
||||||
|
UpdateMetadataOpts, Uuid, X_AMZ_RESTORE, get_raw_etag, restore_operation_id_from_metadata,
|
||||||
|
};
|
||||||
use crate::bucket::lifecycle::lifecycle;
|
use crate::bucket::lifecycle::lifecycle;
|
||||||
use rustfs_filemeta::RestoreStatusOps;
|
use rustfs_filemeta::RestoreStatusOps;
|
||||||
use rustfs_utils::http::headers::{AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE};
|
use rustfs_utils::http::headers::{AMZ_RESTORE_EXPIRY_DAYS, AMZ_RESTORE_REQUEST_DATE};
|
||||||
use s3s::dto::{RestoreStatus, Timestamp};
|
use s3s::dto::{RestoreStatus, Timestamp};
|
||||||
|
#[cfg(all(test, feature = "test-util"))]
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
#[cfg(all(test, feature = "test-util"))]
|
#[cfg(all(test, feature = "test-util"))]
|
||||||
struct RestoreFinalizeBarrierState {
|
struct RestoreFinalizeBarrierState {
|
||||||
|
|||||||
@@ -12,10 +12,14 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use super::*;
|
use super::{
|
||||||
|
Arc, Duration, GetObjectMetadataCacheKey, HeaderMap, MakeBucketOptions, ObjectOptions, OffsetDateTime, PutObjReader,
|
||||||
|
SetDisks, Uuid,
|
||||||
|
};
|
||||||
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time};
|
use crate::bucket::lifecycle::lifecycle::{TRANSITION_COMPLETE, TRANSITION_PENDING, TransitionOptions, expected_expiry_time};
|
||||||
use crate::ecstore_validation_blackbox::make_local_set_disks;
|
use crate::ecstore_validation_blackbox::make_local_set_disks;
|
||||||
use crate::services::tier::test_util::register_mock_tier;
|
use crate::services::tier::test_util::register_mock_tier;
|
||||||
|
use crate::storage_api_contracts::bucket::BucketOperations;
|
||||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||||
use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
|
use rustfs_filemeta::{RestoreStatusOps as _, parse_restore_obj_status};
|
||||||
use tokio::io::AsyncReadExt;
|
use tokio::io::AsyncReadExt;
|
||||||
|
|||||||
@@ -110,4 +110,3 @@ crates/ecstore/src/services/tier/warm_backend_s3.rs|unused_variables
|
|||||||
crates/ecstore/src/services/tier/warm_backend_tencent.rs|clippy::all
|
crates/ecstore/src/services/tier/warm_backend_tencent.rs|clippy::all
|
||||||
crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_must_use
|
crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_must_use
|
||||||
crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_variables
|
crates/ecstore/src/services/tier/warm_backend_tencent.rs|unused_variables
|
||||||
crates/ecstore/src/set_disk/mod.rs|unused_variables
|
|
||||||
|
|||||||
Reference in New Issue
Block a user