gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-10-14 08:30:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(integrity): add inventory, audit, and protected migration (#8065)

Browse Source 
* feat(integrity): add inventory, audit, and protected migration

* refactor(admin): use gateway facade for integrity handlers

* fix(integrity): sort fingerprint metadata explicitly
This commit is contained in:
cxymds
2026-09-22 19:23:41 +08:00
committed by GitHub
parent d0ce2f758b
commit 122abfaae2
21 changed files with 1968 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/set_disk/mod.rs
+2
View File
@@ -13698,6 +13698,7 @@ mod tests {
true,
false,
false,
false,
GET_OBJECT_PATH_LEGACY_DUPLEX,
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART,
metrics_size_bucket,
@@ -13811,6 +13812,7 @@ mod tests {
true,
false,
false,
false,
GET_OBJECT_PATH_LEGACY_DUPLEX,
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART,
metrics_size_bucket,
crates/ecstore/src/set_disk/ops/object.rs
+7
View File
@@ -2776,6 +2776,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
self.set_index,
self.pool_index,
opts.skip_verify_bitrot,
opts.suppress_read_repair,
true,
true,
GET_OBJECT_PATH_LEGACY_DUPLEX,
@@ -2805,6 +2806,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
self.set_index,
self.pool_index,
opts.skip_verify_bitrot,
opts.suppress_read_repair,
true,
false,
GET_OBJECT_PATH_LEGACY_DUPLEX,
@@ -2893,6 +2895,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
self.set_index,
self.pool_index,
opts.skip_verify_bitrot,
opts.suppress_read_repair,
true,
false,
GET_OBJECT_PATH_DIRECT_MEMORY,
@@ -3063,6 +3066,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
let set_index = self.set_index;
let pool_index = self.pool_index;
let skip_verify = opts.skip_verify_bitrot;
let suppress_read_repair = opts.suppress_read_repair;
// The producer runs in a separate Tokio task, so carry the caller's
// read policy across the task boundary explicitly. Tokio task-local
// values are not inherited by spawned tasks.
@@ -3093,6 +3097,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
set_index,
pool_index,
skip_verify,
suppress_read_repair,
false,
false,
GET_OBJECT_PATH_LEGACY_DUPLEX,
@@ -9493,6 +9498,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let set_index = self.set_index;
let pool_index = self.pool_index;
let skip_verify = opts.skip_verify_bitrot;
let suppress_read_repair = opts.suppress_read_repair;
let metrics_size_bucket = rustfs_io_metrics::get_object_size_bucket(cloned_fi.size);
let erasure_cache = Arc::clone(&self.erasure_cache);
let producer = async move {
@@ -9510,6 +9516,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
set_index,
pool_index,
skip_verify,
suppress_read_repair,
false,
false,
GET_OBJECT_PATH_LEGACY_DUPLEX,
crates/ecstore/src/set_disk/read.rs
+95 -16
View File
@@ -579,7 +579,7 @@ impl SetDisks {
}
}
metadata_fanout_diagnostics.record_quorum_candidate_latency(metadata_metrics_path, fileinfo_selection_quorum);
if errs.iter().any(|err| err.is_some()) {
if !opts.suppress_read_repair && errs.iter().any(|err| err.is_some()) {
let version_id = resolved_read_repair_version_id(&fi, opts.version_id.as_deref());
submit_read_repair_heal(
&fi.volume,
@@ -848,6 +848,7 @@ impl SetDisks {
set_index: usize,
pool_index: usize,
skip_verify_bitrot: bool,
suppress_read_repair: bool,
prefer_data_blocks_first_reader_setup: bool,
require_reconstruction_surplus: bool,
metrics_path: &'static str,
@@ -1192,7 +1193,7 @@ impl SetDisks {
"Shard availability check"
);
if missing_shards > 0 && available_shards >= erasure.data_shards {
if !suppress_read_repair && missing_shards > 0 && available_shards >= erasure.data_shards {
// We have missing shards but enough to read - trigger background heal
debug!(
bucket,
@@ -1315,20 +1316,22 @@ impl SetDisks {
// is bound to the read-repair reservation, so only the
// first sighting within the dedup TTL books a journal
// record instead of one per retried read.
submit_read_repair_heal_with_submitter(
ReadRepairHealSubmission {
bucket,
object,
version_id: version_id.as_deref(),
pool_index,
set_index,
part_number: Some(part_number),
reason: "decode_error",
mrf_intent: Some((rustfs_common::mrf_channel::MrfKind::DecodeFailure, fi.version_id)),
},
send_read_repair_heal_request,
)
.await;
if !suppress_read_repair {
submit_read_repair_heal_with_submitter(
ReadRepairHealSubmission {
bucket,
object,
version_id: version_id.as_deref(),
pool_index,
set_index,
part_number: Some(part_number),
reason: "decode_error",
mrf_intent: Some((rustfs_common::mrf_channel::MrfKind::DecodeFailure, fi.version_id)),
},
send_read_repair_heal_request,
)
.await;
}
has_err = false;
}
}
@@ -2515,6 +2518,7 @@ mod metadata_cache_tests {
false,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
@@ -2547,6 +2551,7 @@ mod metadata_cache_tests {
false,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
@@ -2572,6 +2577,7 @@ mod metadata_cache_tests {
false,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
@@ -2595,6 +2601,7 @@ mod metadata_cache_tests {
false,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
@@ -2620,6 +2627,7 @@ mod metadata_cache_tests {
false,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
@@ -2659,6 +2667,7 @@ mod metadata_cache_tests {
false,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"empty",
@@ -2693,6 +2702,7 @@ mod metadata_cache_tests {
false,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"plain",
"small",
@@ -2992,6 +3002,74 @@ mod metadata_cache_tests {
);
}
#[test]
fn missing_metadata_read_repair_is_suppressed_only_for_read_only_requests() {
use crate::object_api::{PutObjReader, ShardIntegrityWriteMode, WriteCompletion};
use crate::storage_api_contracts::object::ObjectIO;
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime");
let recorder = crate::test_metrics::CapturingRecorder::default();
metrics::with_local_recorder(&recorder, || {
runtime.block_on(async {
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
let (dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, ctx).await;
let bucket = format!("read-only-{}", Uuid::new_v4());
let object = "missing-metadata";
for disk in set.disks.read().await.iter().flatten() {
disk.make_volume(&bucket).await.expect("bucket volume");
}
let written = set
.put_object(
&bucket,
object,
&mut PutObjReader::from_vec(vec![7; 1024]),
&ObjectOptions {
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Legacy),
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("fully committed fixture");
let missing = dirs[0].path().join(&bucket).join(object).join("xl.meta");
tokio::fs::remove_file(&missing).await.expect("remove one metadata replica");
// Intercept the production submitter at its dedup admission boundary.
// This avoids initializing the process-global heal channel or allowing
// a background worker to repair the physical fault under test.
let version = written.version_id.map(|id| id.to_string());
let reservation = reserve_read_repair_heal(&bucket, object, version.as_deref(), 0, 0)
.await
.expect("unique object reservation");
for (suppress_read_repair, submissions) in [(true, 0), (false, 1), (true, 1)] {
set.get_object_fileinfo(
&bucket,
object,
&ObjectOptions {
include_part_checksums: true,
suppress_read_repair,
..Default::default()
},
false,
false,
)
.await
.expect("remaining metadata replicas retain read quorum");
assert_eq!(
recorder.counter_value("rustfs_heal_read_repair_dedup_total", &[("reason", "duplicate")]),
submissions,
"only ordinary reads must reach read-repair admission"
);
assert!(!missing.exists(), "the read-only probe must not restore metadata");
}
release_read_repair_heal_reservation(&reservation).await;
});
});
}
#[tokio::test]
async fn read_repair_heal_dedupes_same_object_version() {
let bucket = format!("bucket-{}", Uuid::new_v4());
@@ -5055,6 +5133,7 @@ mod tests {
false,
false,
false,
false,
GET_OBJECT_PATH_SET_DISK,
"test-object-class",
"test-size-bucket",