mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-01 16:50:45 +00:00
0e5fe8c658
* feat(scanner): share node admission with background heal * fix(scanner): drain candidates after listing completion * fix(scanner): yield cache leases for shared node admission * test(scanner): attest adaptive admission in process receipts * fix(concurrency): declare Tokio time dependency * feat(concurrency): retain bounded physical resource subsets * fix(scanner): retain resource ownership in native filesystem jobs * feat(concurrency): acquire physical groups before node work * feat(scanner): gate background reads by physical device * feat(concurrency): combine preallocated resource selections * feat(scanner): share physical topology across disk plans * fix(storage): retain only native disk resource subsets * feat(storage): admit auxiliary physical resources without waiting * feat(heal): admit physical resources before object repair * fix(storage): retain resources during native format probes * fix(storage): retain resources through direct writer I/O * fix(storage): retain resources through buffered writer I/O * fix(storage): retain physical resources through streamed reads * fix(storage): retain resources through native filesystem jobs * fix(storage): retain physical ownership during bitrot reads * fix(ecstore): retain resources during positioned reads * fix(ecstore): retain staged metadata write ownership * fix(ecstore): retain native io_uring read ownership * fix(ecstore): enforce receiver-side background admission * fix(ecstore): retain resources through whole-file native I/O * fix(ecstore): retain physical ownership during durable rename sync * fix(storage): account for retained background stream buffers * fix(storage): retain memory charges for background read results * fix(storage): account for native io_uring read allocations * fix(storage): retain memory charges for direct-write buffers * fix(ecstore): account for heal and RPC reader buffers * test(scanner): verify samples and clean concurrent collectors * fix(storage): retain inline and HTTP writer memory reservations * perf(rio): reuse reserved HTTP writer storage * fix(rpc): reserve background scanner pipe storage * perf(rpc): reuse reserved response reader storage * feat(scanner): measure cache copy allocation requests * fix(rpc): budget namespace scanner responses * fix(scanner): retain the enabled supervisor task handle * fix(scanner): budget namespace frame read staging * fix(scanner): account namespace envelope decode storage * fix(heal): retain and join the MRF consumer on shutdown * test(heal): isolate process-global MRF consumer fixture * fix(heal): join manager workers during shutdown * test(heal): verify shutdown at native read boundaries * fix(heal): drain channel commands before stopping MRF * fix(heal): admit resume scans and retain replacement workers * build: pin profiling sources and trust fixed ownership dependencies * fix(heal): yield node admission while accepted native work drains * fix(startup): retain shutdown work and preserve deadline failures * fix(heal): admit inventory probes and retain replacement work * fix(shutdown): preserve maintenance and frontend join failures * fix(shutdown): retain HTTP child tasks through real retirement * fix(shutdown): join actual TLS reload work before retirement * test(multipart): preserve failures before fault barriers * fix: keep physical admission inside repository guards Route the multipart lock diagnostics filter through the e2e storage compatibility module, reuse the parent module's s3s body aliases in the namespace memory test, and share one Reed-Solomon verify error mapping across the reserved-buffer and default paths. * fix: reconcile physical admission with main's new call sites Main added a source-admission metadata write layer, a CAS write fence, fixture writers and scanner handoff tests against the old borrowed-slice and node-permit shapes. Pass owned Bytes through the new metadata layer, answer the scanner handoff with ScannerNodePermit, gate the test-only disk lookup re-export, and clear the lint findings that stable 1.99 reports for the shutdown and admission code without suppressions. * test(concurrency): name reversed attachment groups clearly * fix(ecstore): box disk operation futures below admission layers Every LocalDiskWrapper operation reaches the LocalDisk future through the health, admission and timeout layers (track_cleanup_mutation, the track_disk_health family, the io-permit and native-resource admission split, and track_admitted_disk_operation). Each layer embedded that future, and unoptimized builds copy it into every awaiting frame, so one local delete cost about 1.3 MiB of stack and 16 tests overflowed libtest's default 2 MiB stack. Box the operation future once where track_admitted_disk_operation creates it, through a non-async helper so the temporary never stays in a polled frame. The layers above now hold only a pointer; the cost is one allocation per local disk operation. * fix(heal): box object heal futures below admission The admission wrappers took the object heal future by value, so unoptimized builds copied it into three awaiting frames, about 650 KiB per heal. Create it through a non-async helper instead, leaving only a pointer in those frames. * test(ecstore): count the I/O backend's root FD owner
97 lines
3.8 KiB
Rust
97 lines
3.8 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#[cfg(test)]
|
|
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
|
|
#[cfg(test)]
|
|
pub(crate) const OBJECT_LOCK_TRACE_FILTER: &str = "info,rustfs_ecstore::object_lock_diag=trace";
|
|
#[cfg(test)]
|
|
pub(crate) use rustfs_ecstore::api::disk::{
|
|
DiskOption, Endpoint, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, VolumeInfo, WalkDirOptions, new_disk,
|
|
};
|
|
pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
|
#[cfg(test)]
|
|
pub(crate) use rustfs_ecstore::api::rpc::{
|
|
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
|
verify_tonic_boot_epoch_response,
|
|
};
|
|
#[cfg(test)]
|
|
pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
|
|
|
|
#[cfg(test)]
|
|
pub(crate) mod node_interact {
|
|
pub(crate) use super::{
|
|
TonicInterceptor, VolumeInfo, WalkDirOptions, gen_tonic_signature_interceptor, node_service_time_out_client,
|
|
};
|
|
}
|
|
|
|
pub(crate) mod grpc_lock {
|
|
pub(crate) use super::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
|
|
}
|
|
|
|
/// Signing/transport surface used by the cross-process internode RPC signature
|
|
/// acceptance tests (backlog#1327). The signing helpers are what let a test mint
|
|
/// the one legitimately signed request an on-path attacker is assumed to have
|
|
/// captured; every attack in that suite then only *reuses* those bytes.
|
|
#[cfg(test)]
|
|
pub(crate) mod internode_rpc_signature {
|
|
pub(crate) use super::{
|
|
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
|
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
|
|
};
|
|
}
|
|
|
|
#[cfg(test)]
|
|
pub(crate) mod replication_extension {
|
|
pub(crate) use super::BucketTargetSys;
|
|
}
|
|
|
|
#[cfg(test)]
|
|
pub(crate) mod upgrade_compatibility {
|
|
pub(crate) use super::{DiskOption, Endpoint, RUSTFS_META_MULTIPART_BUCKET, new_disk};
|
|
}
|
|
|
|
#[cfg(test)]
|
|
pub(crate) mod multipart_rpc_delay {
|
|
pub(crate) use super::RUSTFS_META_MULTIPART_BUCKET;
|
|
}
|
|
|
|
#[cfg(test)]
|
|
pub(crate) mod multipart_fence_acceptance {
|
|
pub(crate) use super::RUSTFS_META_MULTIPART_BUCKET;
|
|
|
|
/// Targeted lock diagnostics for the metadata-delay quorum case.
|
|
pub(crate) const LOCK_DIAGNOSTICS_FILTER: &str = "warn,rustfs_lock::distributed_lock=debug,rustfs_ecstore::object_lock_diag=trace,rustfs_ecstore::cluster::rpc::remote_locker=info";
|
|
}
|
|
|
|
#[cfg(test)]
|
|
pub(crate) mod multipart_source_rpc {
|
|
pub(crate) use rustfs_ecstore::api::disk::{
|
|
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
|
|
MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceSnapshot,
|
|
PublicationIdentity, SourceGeneration, SourceGeometry, SourceLineage,
|
|
};
|
|
}
|
|
|
|
#[cfg(test)]
|
|
pub(crate) mod scanner_peer_bucket {
|
|
use rustfs_ecstore::api::disk as ecstore_disk;
|
|
use rustfs_ecstore::api::erasure as ecstore_erasure;
|
|
use rustfs_ecstore::api::rpc as ecstore_rpc;
|
|
|
|
pub(crate) use ecstore_disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
|
pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
|
|
pub(crate) use ecstore_rpc::PeerRestClient;
|
|
}
|