gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-29 16:37:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: migrate consumers off rustfs-common heal/scanner shims (#6623)

Browse Source 
* refactor(ecstore): import heal/scanner contracts crates directly (backlog#1843)

* refactor(heal): import heal/scanner contracts crates directly (backlog#1843)

* refactor(lifecycle): import heal/scanner contracts crates directly (backlog#1843)

* refactor(obs): import heal/scanner contracts crates directly (backlog#1843)

* refactor(protos): import heal/scanner contracts crates directly (backlog#1843)

* refactor(scanner): import heal/scanner contracts crates directly (backlog#1843)

* refactor(rustfs): import heal/scanner contracts crates directly (backlog#1843)
This commit is contained in:
Zhengchao An
2026-08-26 21:13:18 +08:00
committed by GitHub
parent 2ada8a5cfb
commit a96dd7d289
72 changed files with 263 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/data_usage_define.rs
+1 -1
View File
@@ -23,7 +23,6 @@ use std::{
use http::HeaderMap;
use metrics::{counter, describe_counter, describe_histogram, histogram};
use rustfs_common::heal_channel::HealScanMode;
#[cfg(test)]
use rustfs_config::ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS;
pub use rustfs_data_usage::{
@@ -33,6 +32,7 @@ pub use rustfs_data_usage::{
SizeReconciliationScope, SizeSummary, TierAccountingProof, TierStats, UNKNOWN_TIER, UNKNOWN_TIER_DIAGNOSTIC_BYTE_CAP,
UNKNOWN_TIER_DIAGNOSTIC_ENTRY_CAP, UnknownTierStats, hash_path, prefix_usage_in_cache,
};
use rustfs_heal_contracts::heal_channel::HealScanMode;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use tokio::time::{Duration, Instant, sleep, timeout};
use tracing::{debug, warn};
crates/scanner/src/lib.rs
+1 -1
View File
@@ -79,7 +79,7 @@ pub use remote_scanner::{
remote_scanner_request_matches_envelope, serve_remote_scanner_request, validate_remote_scanner_request_fence,
};
pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
pub use rustfs_common::last_minute;
pub use rustfs_scanner_contracts::last_minute;
pub use scanner::{
ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, init_data_scanner,
reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_topology_digest,
crates/scanner/src/remote_scanner/stream.rs
+2 -2
View File
@@ -26,9 +26,9 @@ use crate::{
scanner_publication_admission_for_epoch, scanner_publication_epoch,
};
use hmac::{Hmac, KeyInit, Mac};
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{Metric, Metrics};
use rustfs_credentials::try_get_rpc_token;
use rustfs_heal_contracts::heal_channel::HealScanMode;
use rustfs_scanner_contracts::metrics::{Metric, Metrics};
use rustfs_utils::path::path_join_buf;
use serde::{Deserialize, Serialize};
use sha2::Sha256;
crates/scanner/src/scanner.rs
+6 -6
View File
@@ -40,12 +40,6 @@ use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGua
use crate::{ScannerConfigObjectDelete, ScannerObjectIO, ScannerObjectOptions};
use bytes::Bytes;
use chrono::{DateTime, Utc};
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource,
emit_scan_cycle_complete, emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded,
global_metrics,
};
use rustfs_config::ScannerSpeed;
#[cfg(test)]
use rustfs_config::{
@@ -54,7 +48,13 @@ use rustfs_config::{
};
use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
use rustfs_data_usage::observed_data_usage_is_newer;
use rustfs_heal_contracts::heal_channel::HealScanMode;
use rustfs_lock::{NamespaceLockGuard, error::LockError};
use rustfs_scanner_contracts::metrics::{
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource,
emit_scan_cycle_complete, emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded,
global_metrics,
};
use serde::{Deserialize, Serialize};
use sha2::{Digest as _, Sha256};
use tokio::sync::{Notify, mpsc};
crates/scanner/src/scanner_folder.rs
+8 -8
View File
@@ -41,19 +41,19 @@ use crate::storage_api::owner::{
#[cfg(test)]
use crate::storage_api::owner::{EcstoreExpirationStatus as ExpirationStatus, EcstoreLifecycleRule as LifecycleRule};
use metrics::{counter, describe_counter};
use rustfs_common::heal_channel::{
HEAL_DELETE_DANGLING, HealAdmissionDropReason, HealAdmissionResult, HealChannelPriority, HealChannelRequest,
HealRequestSource, HealScanMode, send_heal_request_with_admission,
};
use rustfs_common::metrics::{
CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource,
UpdateCurrentPathFn, current_path_updater, global_metrics,
};
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit, trace_subscriber_count};
use rustfs_filemeta::{
MAX_META_CACHE_HEAL_CANDIDATES, MAX_META_CACHE_HEAL_TRUNCATED_OBJECTS, MetaCacheEntries, MetaCacheEntry,
MetaCacheHealCandidateKind,
};
use rustfs_heal_contracts::heal_channel::{
HEAL_DELETE_DANGLING, HealAdmissionDropReason, HealAdmissionResult, HealChannelPriority, HealChannelRequest,
HealRequestSource, HealScanMode, send_heal_request_with_admission,
};
use rustfs_scanner_contracts::metrics::{
CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource,
UpdateCurrentPathFn, current_path_updater, global_metrics,
};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use time::OffsetDateTime;
use tokio::select;
crates/scanner/src/scanner_folder/tests.rs
+4 -4
View File
@@ -745,7 +745,7 @@ async fn test_scanner_heal_admission_accounting_maps_normal_scan_to_heal() {
&metrics,
HealScanMode::Normal,
Ok(HealAdmissionResult::Dropped(
rustfs_common::heal_channel::HealAdmissionDropReason::QueueFull,
rustfs_heal_contracts::heal_channel::HealAdmissionDropReason::QueueFull,
)),
);
record_scanner_heal_admission(&metrics, HealScanMode::Normal, Err(()));
@@ -1687,7 +1687,7 @@ fn test_describe_heal_admission_formats_unadmitted_results() {
assert_eq!(describe_heal_admission(HealAdmissionResult::Full), "queue_full");
assert_eq!(
describe_heal_admission(HealAdmissionResult::Dropped(
rustfs_common::heal_channel::HealAdmissionDropReason::QueueFull
rustfs_heal_contracts::heal_channel::HealAdmissionDropReason::QueueFull
)),
"dropped:queue_full"
);
@@ -1879,10 +1879,10 @@ async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
let healed_versions = Arc::new(Mutex::new(Vec::<Option<String>>::new()));
let healed_versions_clone = healed_versions.clone();
let mut heal_rx =
rustfs_common::heal_channel::init_heal_channel().expect("heal channel should initialize once for scanner tests");
rustfs_heal_contracts::heal_channel::init_heal_channel().expect("heal channel should initialize once for scanner tests");
let _heal_responder = tokio::spawn(async move {
while let Some(command) = heal_rx.recv().await {
if let rustfs_common::heal_channel::HealChannelCommand::Start {
if let rustfs_heal_contracts::heal_channel::HealChannelCommand::Start {
request, response_tx, ..
} = command
{
crates/scanner/src/scanner_io.rs
+4 -2
View File
@@ -24,13 +24,15 @@ use crate::{
use futures::future::join_all;
use metrics::counter;
use rand::seq::SliceRandom as _;
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics};
#[cfg(test)]
use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS};
use rustfs_data_usage::{BucketTargetUsageInfo, BucketUsageInfo};
use rustfs_filemeta::FileMeta;
use rustfs_heal_contracts::heal_channel::HealScanMode;
use rustfs_lock::{LockError, NamespaceLockGuard};
use rustfs_scanner_contracts::metrics::{
Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics,
};
use rustfs_utils::path::path_join_buf;
use s3s::dto::{
BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration, VersioningConfiguration,
crates/scanner/src/scanner_io/guards.rs
+3 -3
View File
@@ -147,13 +147,13 @@ impl Drop for DiskBucketScanActiveGuard {
pub(super) struct BucketDriveFailureGuard {
failed: bool,
source: rustfs_common::metrics::ScannerWorkSource,
source: rustfs_scanner_contracts::metrics::ScannerWorkSource,
bucket: String,
drive: String,
}
impl BucketDriveFailureGuard {
pub(super) fn new(source: rustfs_common::metrics::ScannerWorkSource, bucket: &str, drive: &str) -> Self {
pub(super) fn new(source: rustfs_scanner_contracts::metrics::ScannerWorkSource, bucket: &str, drive: &str) -> Self {
Self {
failed: true,
source,
@@ -285,7 +285,7 @@ pub(super) fn scanner_task_join_error(stage: &str, err: tokio::task::JoinError)
#[cfg(test)]
mod tests {
use super::*;
use rustfs_common::metrics::{ScannerWorkSource, global_metrics};
use rustfs_scanner_contracts::metrics::{ScannerWorkSource, global_metrics};
#[test]
fn bucket_drive_failure_guard_retires_active_scan_on_drop() {
crates/scanner/src/scanner_io/io_disk.rs
+2 -2
View File
@@ -161,8 +161,8 @@ impl ScannerIODisk for Disk {
let bucket = cache.info.name.clone();
let disk_path = self.path().to_string_lossy().to_string();
let source = match scan_mode {
HealScanMode::Deep => rustfs_common::metrics::ScannerWorkSource::Bitrot,
HealScanMode::Normal | HealScanMode::Unknown => rustfs_common::metrics::ScannerWorkSource::Usage,
HealScanMode::Deep => rustfs_scanner_contracts::metrics::ScannerWorkSource::Bitrot,
HealScanMode::Normal | HealScanMode::Unknown => rustfs_scanner_contracts::metrics::ScannerWorkSource::Usage,
};
global_metrics().record_scan_bucket_drive_start(source, &bucket, &disk_path);
let mut failure_guard = BucketDriveFailureGuard::new(source, &bucket, &disk_path);
crates/scanner/src/sleeper.rs
+1 -1
View File
@@ -16,11 +16,11 @@ use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use std::sync::{Arc, LazyLock, RwLock};
use std::time::Instant;
use rustfs_common::metrics::global_metrics;
use rustfs_config::{
DEFAULT_SCANNER_IDLE_MODE, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED,
ENV_SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed,
};
use rustfs_scanner_contracts::metrics::global_metrics;
use tokio::time::Duration;
const MIN_SLEEP: Duration = Duration::from_millis(1);