feat(scanner): coordinate usage and workload boundaries (#7093)

* test(scanner): wire usage and heal rebuild gates

* docs(scanner): define usage authority protocol

* docs(heal): clarify scanner and ecstore boundaries

* refactor(scanner): split metrics from contracts

* feat(scanner): use shared workload snapshots

* fix(ecstore): recheck capacity before decommission drain
This commit is contained in:
houseme
2026-09-03 17:02:43 +08:00
committed by GitHub
parent 3ab7a1921f
commit 0e6ee3bf62
57 changed files with 632 additions and 134 deletions
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=21f2a5484bb10bed1f11bd6397fe80f20d60a870e4ed5016a1f3acbdb4e6f735 sha256-darwin=bb8c16cd63a94ff5e5e400d891ccc980710529dd5cb13a5a0c955c6c313bccd9
sha256-linux=29fd001e70d0114b59d65dd2a516c1f96a5fea1d6bb8fb315550a27f94d7b39b sha256-linux=e9a8d64e73f627c4d26c236dbbba690c9ee03a9e26d42a4244515b4439365535
+1 -1
View File
@@ -1 +1 @@
sha256=d90fa941cb737fe6fcf39ca13d45440183e77a7e047880c025af7e6bfe1d0517 sha256=a2542dc86bbff56b2177efc621785c56fa7e8d813b209b7d935e1e41a9f0ad15
+18 -7
View File
@@ -407,10 +407,15 @@ test-group = 'ecstore-serial-flaky'
# harness_self}_test` stays in e2e-full and the fault / concurrency / # harness_self}_test` stays in e2e-full and the fault / concurrency /
# real-source modules stay in e2e-nightly; this is an allowlist, not a module # real-source modules stay in e2e-nightly; this is an allowlist, not a module
# clause, so a new ODM test never lands here silently. # clause, so a new ODM test never lands here silently.
#
# Scanner authoritative usage publication (backlog#2213): data_usage_test is
# the PR-lane e2e coverage for scanner usage snapshots consumed by quota and
# admin surfaces. It uses the same single-node, random-port, isolated-temp-dir
# fixture as the existing smoke modules.
[profile.e2e-smoke] [profile.e2e-smoke]
default-filter = """ default-filter = """
package(e2e_test) & ( package(e2e_test) & (
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/) test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/) | test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
| test(/^reliant::lifecycle::/) | test(/^reliant::lifecycle::/)
| test(/^reliant::tiering::/) | test(/^reliant::tiering::/)
@@ -488,12 +493,16 @@ path = "junit.xml"
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# e2e-nightly profile — destructive multi-process cluster fault domains # e2e-nightly profile — destructive multi-process cluster fault domains
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# These seven modules are deliberately outside e2e-full's merge budget. Each # These eight modules are deliberately outside e2e-full's merge budget. Each
# starts a real multi-process or multi-disk topology and exercises node/disk # starts a real multi-process or multi-disk topology and exercises node/disk
# loss, quorum, cleanup, notification fan-in, or admin-timeout behavior. The # loss, quorum, cleanup, notification fan-in, or admin-timeout behavior. The
# consolidated nightly workflow runs them serially to avoid resource # consolidated nightly workflow runs them serially to avoid resource
# starvation; failures are never retried. # starvation; failures are never retried.
# #
# heal_erasure_disk_rebuild_test also runs in e2e-full so core heal rebuild
# regressions are caught no later than the merge/main lane. It remains here for
# nightly serial coverage with the other cluster fault domains.
#
# On-demand migration (backlog#2158 ODM-11) joins by the second clause: the # On-demand migration (backlog#2158 ODM-11) joins by the second clause: the
# fault matrix waits out the 30 s circuit-breaker window, the concurrency # fault matrix waits out the 30 s circuit-breaker window, the concurrency
# matrix drives 100-deep bursts, and the real-source cases start a second # matrix drives 100-deep bursts, and the real-source cases start a second
@@ -542,11 +551,13 @@ path = "junit.xml"
# The filter is "the whole e2e_test crate MINUS the sets owned by other lanes": # The filter is "the whole e2e_test crate MINUS the sets owned by other lanes":
# * protocols:: — FTPS/SFTP/WebDAV, run from the dedicated protocol profile # * protocols:: — FTPS/SFTP/WebDAV, run from the dedicated protocol profile
# with one worker because the suite owns fixed ports. # with one worker because the suite owns fixed ports.
# * the 7 cluster suites that spin up a RustFSTestClusterEnvironment # * cluster suites that spin up a RustFSTestClusterEnvironment
# (cluster_concurrency, cluster_multidrive_pool, stale_multipart_cleanup_cluster, # (cluster_concurrency, cluster_multidrive_pool, stale_multipart_cleanup_cluster,
# namespace_lock_quorum, heal_erasure_disk_rebuild, admin_timeout_regression, # namespace_lock_quorum, admin_timeout_regression, object_lambda) — too
# object_lambda) — too heavy for the merge budget; they run in the # heavy for the merge budget; they run in the e2e-nightly serial
# e2e-nightly serial cluster-fault lane. # cluster-fault lane. heal_erasure_disk_rebuild is intentionally not
# excluded here because backlog#2213 promotes core heal rebuild coverage to
# this merge/main lane while retaining nightly coverage.
# * replication_extension_test — repl-1 already splits it into the PR # * replication_extension_test — repl-1 already splits it into the PR
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (56 slow) lanes and reserves # `e2e-smoke` (20 fast) and `e2e-repl-nightly` (56 slow) lanes and reserves
# it for those, so e2e-full does not double-run it. # it for those, so e2e-full does not double-run it.
@@ -562,7 +573,7 @@ path = "junit.xml"
default-filter = """ default-filter = """
package(e2e_test) package(e2e_test)
& !test(/^protocols::/) & !test(/^protocols::/)
& !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/) & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
& !test(/^replication_extension_test::/) & !test(/^replication_extension_test::/)
& !test(/^replication_target_matrix_test::/) & !test(/^replication_target_matrix_test::/)
& !test(/^on_demand_migration::(concurrency_test|fault_test|real_source_test)::/) & !test(/^on_demand_migration::(concurrency_test|fault_test|real_source_test)::/)
+2 -2
View File
@@ -92,7 +92,7 @@ refactors.
| Domain | Current workspace crates | Responsibility | | Domain | Current workspace crates | Responsibility |
|--------|--------------------------|----------------| |--------|--------------------------|----------------|
| Foundation | `checksums`, `common`, `config`, `data-usage`, `heal-contracts`, `scanner-contracts`, `utils` | Shared configuration, data-usage models, heal/scanner domain contracts, utilities, and checksums. | | Foundation | `checksums`, `common`, `config`, `data-usage`, `heal-contracts`, `scanner-contracts`, `scanner-metrics`, `utils` | Shared configuration, data-usage models, heal/scanner domain contracts and telemetry types, utilities, and checksums. |
| I/O and storage | `concurrency`, `ecstore`, `filemeta`, `heal`, `io-core`, `io-metrics`, `lifecycle`, `lock`, `object-capacity`, `object-data-cache`, `replication`, `rio`, `rio-v2`, `s3-client`, `scanner`, `storage-api` | Erasure-coded object storage, metadata, recovery, lifecycle, replication, locking, cache, I/O pipelines, and the engine-side S3 client for remote tier/transition targets. | | I/O and storage | `concurrency`, `ecstore`, `filemeta`, `heal`, `io-core`, `io-metrics`, `lifecycle`, `lock`, `object-capacity`, `object-data-cache`, `replication`, `rio`, `rio-v2`, `s3-client`, `scanner`, `storage-api` | Erasure-coded object storage, metadata, recovery, lifecycle, replication, locking, cache, I/O pipelines, and the engine-side S3 client for remote tier/transition targets. |
| Security and identity | `credentials`, `crypto`, `iam`, `keystone`, `kms`, `policy`, `security-governance`, `signer`, `tls-runtime`, `trusted-proxies` | Credentials, authentication, authorization, encryption, key management, TLS, and security contracts. | | Security and identity | `credentials`, `crypto`, `iam`, `keystone`, `kms`, `policy`, `security-governance`, `signer`, `tls-runtime`, `trusted-proxies` | Credentials, authentication, authorization, encryption, key management, TLS, and security contracts. |
| Protocols and contracts | `extension-schema`, `madmin`, `protos`, `protocols`, `s3-ops`, `s3-types`, `s3select-api`, `s3select-query` | Admin, inter-node, S3, S3 Select, and optional protocol contracts. | | Protocols and contracts | `extension-schema`, `madmin`, `protos`, `protocols`, `s3-ops`, `s3-types`, `s3select-api`, `s3select-query` | Admin, inter-node, S3, S3 Select, and optional protocol contracts. |
@@ -135,7 +135,7 @@ default build (lifecycle:
`crates/ecstore/src/bucket/replication/replication_state.rs`) — a naming `crates/ecstore/src/bucket/replication/replication_state.rs`) — a naming
collision, not copies; renaming is tracked in rustfs/backlog#1847. collision, not copies; renaming is tracked in rustfs/backlog#1847.
- `LastMinuteLatency` has two deliberately different implementations: the - `LastMinuteLatency` has two deliberately different implementations: the
per-second bucketed accumulator in `crates/scanner-contracts/src/last_minute.rs` and per-second bucketed accumulator in `crates/scanner-metrics/src/last_minute.rs` and
the in-memory endpoint-health sample tracker in the in-memory endpoint-health sample tracker in
`crates/ecstore/src/bucket/bucket_target_sys.rs` (its doc comment explains `crates/ecstore/src/bucket/bucket_target_sys.rs` (its doc comment explains
why it stays local). why it stays local).
Generated
+10 -5
View File
@@ -9517,7 +9517,7 @@ dependencies = [
"rustfs-s3select-api", "rustfs-s3select-api",
"rustfs-s3select-query", "rustfs-s3select-query",
"rustfs-scanner", "rustfs-scanner",
"rustfs-scanner-contracts", "rustfs-scanner-metrics",
"rustfs-security-governance", "rustfs-security-governance",
"rustfs-signer", "rustfs-signer",
"rustfs-storage-api", "rustfs-storage-api",
@@ -9763,7 +9763,7 @@ dependencies = [
"rustfs-rio-v2", "rustfs-rio-v2",
"rustfs-s3-client", "rustfs-s3-client",
"rustfs-s3-types", "rustfs-s3-types",
"rustfs-scanner-contracts", "rustfs-scanner-metrics",
"rustfs-storage-api", "rustfs-storage-api",
"rustfs-uring", "rustfs-uring",
"rustfs-utils", "rustfs-utils",
@@ -10119,7 +10119,7 @@ dependencies = [
"proptest", "proptest",
"rustfs-config", "rustfs-config",
"rustfs-replication", "rustfs-replication",
"rustfs-scanner-contracts", "rustfs-scanner-metrics",
"rustfs-storage-api", "rustfs-storage-api",
"s3s", "s3s",
"serial_test", "serial_test",
@@ -10317,7 +10317,7 @@ dependencies = [
"rustfs-iam", "rustfs-iam",
"rustfs-io-metrics", "rustfs-io-metrics",
"rustfs-notify", "rustfs-notify",
"rustfs-scanner-contracts", "rustfs-scanner-metrics",
"rustfs-security-governance", "rustfs-security-governance",
"rustfs-storage-api", "rustfs-storage-api",
"rustfs-utils", "rustfs-utils",
@@ -10674,6 +10674,7 @@ dependencies = [
"rand 0.10.2", "rand 0.10.2",
"rmp-serde", "rmp-serde",
"rustfs-common", "rustfs-common",
"rustfs-concurrency",
"rustfs-config", "rustfs-config",
"rustfs-credentials", "rustfs-credentials",
"rustfs-data-usage", "rustfs-data-usage",
@@ -10682,7 +10683,7 @@ dependencies = [
"rustfs-heal-contracts", "rustfs-heal-contracts",
"rustfs-lock", "rustfs-lock",
"rustfs-s3-types", "rustfs-s3-types",
"rustfs-scanner-contracts", "rustfs-scanner-metrics",
"rustfs-storage-api", "rustfs-storage-api",
"rustfs-utils", "rustfs-utils",
"s3s", "s3s",
@@ -10704,6 +10705,10 @@ dependencies = [
[[package]] [[package]]
name = "rustfs-scanner-contracts" name = "rustfs-scanner-contracts"
version = "1.0.0-rc.5" version = "1.0.0-rc.5"
[[package]]
name = "rustfs-scanner-metrics"
version = "1.0.0-rc.5"
dependencies = [ dependencies = [
"chrono", "chrono",
"jiff", "jiff",
+3 -1
View File
@@ -52,7 +52,8 @@ members = [
"crates/s3select-api", # S3 Select API interface "crates/s3select-api", # S3 Select API interface
"crates/s3select-query", # S3 Select query engine "crates/s3select-query", # S3 Select query engine
"crates/scanner", # Scanner for data integrity checks and health monitoring "crates/scanner", # Scanner for data integrity checks and health monitoring
"crates/scanner-contracts", # Scanner metrics and cycle contracts "crates/scanner-contracts", # Scanner storage and wire contracts
"crates/scanner-metrics", # Scanner metrics and cycle telemetry
"crates/security-governance", # Security governance contracts "crates/security-governance", # Security governance contracts
"crates/extension-schema", # Extension schema contracts "crates/extension-schema", # Extension schema contracts
"crates/signer", # client signer "crates/signer", # client signer
@@ -94,6 +95,7 @@ rustfs = { path = "./rustfs", version = "1.0.0-rc.5" }
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.5" } rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.5" }
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.5" } rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.5" }
rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.5" } rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.5" }
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.0-rc.5" }
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.5" } rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.5" }
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.5" } rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.5" }
rustfs-common = { path = "crates/common", version = "1.0.0-rc.5" } rustfs-common = { path = "crates/common", version = "1.0.0-rc.5" }
+1 -1
View File
@@ -130,7 +130,7 @@ rustfs-concurrency.workspace = true
rustfs-credentials = { workspace = true } rustfs-credentials = { workspace = true }
rustfs-common.workspace = true rustfs-common.workspace = true
rustfs-heal-contracts.workspace = true rustfs-heal-contracts.workspace = true
rustfs-scanner-contracts.workspace = true rustfs-scanner-metrics.workspace = true
rustfs-policy.workspace = true rustfs-policy.workspace = true
rustfs-protos.workspace = true rustfs-protos.workspace = true
rustfs-replication.workspace = true rustfs-replication.workspace = true
@@ -17,7 +17,7 @@ use crate::bucket::lifecycle::lifecycle;
use crate::object_api::ObjectInfo; use crate::object_api::ObjectInfo;
use crate::services::event_notification::{EventArgs, send_event}; use crate::services::event_notification::{EventArgs, send_event};
use rustfs_s3_types::EventName; use rustfs_s3_types::EventName;
use rustfs_scanner_contracts::metrics::IlmAction; use rustfs_scanner_metrics::metrics::IlmAction;
const LIFECYCLE_EXPIRY_USER_AGENT: &str = "Internal: [ILM-Expiry]"; const LIFECYCLE_EXPIRY_USER_AGENT: &str = "Internal: [ILM-Expiry]";
const LIFECYCLE_TRANSITION_USER_AGENT: &str = "Internal: [ILM-Transition]"; const LIFECYCLE_TRANSITION_USER_AGENT: &str = "Internal: [ILM-Transition]";
@@ -84,7 +84,7 @@ use rustfs_data_usage::TierStats;
use rustfs_filemeta::{ use rustfs_filemeta::{
FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info, is_restored_object_on_disk, FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info, is_restored_object_on_disk,
}; };
use rustfs_scanner_contracts::metrics::{ use rustfs_scanner_metrics::metrics::{
IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics, IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
}; };
use rustfs_utils::{ use rustfs_utils::{
@@ -5596,7 +5596,7 @@ mod tests {
use rustfs_filemeta::{FileInfo, FileMeta}; use rustfs_filemeta::{FileInfo, FileMeta};
#[cfg(feature = "test-util")] #[cfg(feature = "test-util")]
use rustfs_s3_client::transition_api::ReaderImpl; use rustfs_s3_client::transition_api::ReaderImpl;
use rustfs_scanner_contracts::metrics::{IlmAction, global_metrics}; use rustfs_scanner_metrics::metrics::{IlmAction, global_metrics};
use s3s::dto::{ use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry, BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry,
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, OutputLocation, RestoreRequest, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, OutputLocation, RestoreRequest,
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use rustfs_scanner_contracts::metrics::IlmAction; use rustfs_scanner_metrics::metrics::IlmAction;
use crate::bucket::lifecycle::lifecycle::ObjectOpts; use crate::bucket::lifecycle::lifecycle::ObjectOpts;
use crate::bucket::replication::ReplicationLifecycleBridge; use crate::bucket::replication::ReplicationLifecycleBridge;
@@ -77,7 +77,7 @@ mod tests {
use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge}; use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
use crate::object_api::{ObjectInfo, ObjectOptions}; use crate::object_api::{ObjectInfo, ObjectOptions};
use crate::storage_api_contracts::object::ObjectToDelete; use crate::storage_api_contracts::object::ObjectToDelete;
use rustfs_scanner_contracts::metrics::IlmAction; use rustfs_scanner_metrics::metrics::IlmAction;
use s3s::dto::{ use s3s::dto::{
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication,
DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus,
+5 -5
View File
@@ -16,7 +16,7 @@ use super::{BucketQuota, QuotaCheckResult, QuotaError, QuotaOperation};
use crate::bucket::metadata_sys::{BucketMetadataSys, update, update_if_incarnation}; use crate::bucket::metadata_sys::{BucketMetadataSys, update, update_if_incarnation};
use crate::data_usage::get_bucket_usage_memory; use crate::data_usage::get_bucket_usage_memory;
use rustfs_config::QUOTA_CONFIG_FILE; use rustfs_config::QUOTA_CONFIG_FILE;
use rustfs_scanner_contracts::metrics::Metric; use rustfs_scanner_metrics::metrics::Metric;
use std::sync::Arc; use std::sync::Arc;
use std::time::Instant; use std::time::Instant;
use time::OffsetDateTime; use time::OffsetDateTime;
@@ -120,9 +120,9 @@ impl QuotaChecker {
let duration = start_time.elapsed(); let duration = start_time.elapsed();
// inc_time is now a plain fn (not async) — no .await needed. // inc_time is now a plain fn (not async) — no .await needed.
rustfs_scanner_contracts::metrics::Metrics::inc_time(Metric::QuotaCheck, duration); rustfs_scanner_metrics::metrics::Metrics::inc_time(Metric::QuotaCheck, duration);
if !allowed { if !allowed {
rustfs_scanner_contracts::metrics::Metrics::inc_time(Metric::QuotaViolation, duration); rustfs_scanner_metrics::metrics::Metrics::inc_time(Metric::QuotaViolation, duration);
} }
Ok(result) Ok(result)
@@ -185,7 +185,7 @@ impl QuotaChecker {
.await .await
.map_err(QuotaError::StorageError)?; .map_err(QuotaError::StorageError)?;
rustfs_scanner_contracts::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed()); rustfs_scanner_metrics::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
Ok(updated_at) Ok(updated_at)
} }
@@ -206,7 +206,7 @@ impl QuotaChecker {
} }
.map_err(QuotaError::StorageError)?; .map_err(QuotaError::StorageError)?;
rustfs_scanner_contracts::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed()); rustfs_scanner_metrics::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
Ok(updated_at) Ok(updated_at)
} }
+43 -1
View File
@@ -13504,7 +13504,9 @@ impl ECStore {
} }
return Ok(()); return Ok(());
} }
let result = self.decommission_in_background(rx.clone(), idx, entry_budget).await; let result = self
.decommission_in_background(rx.clone(), idx, generation, entry_budget)
.await;
if let Err(err) = &result if let Err(err) = &result
&& (is_decommission_capacity_blocked_error(err) || is_decommission_target_capacity_error(err)) && (is_decommission_capacity_blocked_error(err) || is_decommission_target_capacity_error(err))
@@ -13989,8 +13991,10 @@ impl ECStore {
self: &Arc<Self>, self: &Arc<Self>,
rx: CancellationToken, rx: CancellationToken,
idx: usize, idx: usize,
generation: OffsetDateTime,
entry_budget: Arc<Semaphore>, entry_budget: Arc<Semaphore>,
) -> Result<()> { ) -> Result<()> {
self.ensure_decommission_runtime_capacity_available(idx, generation).await?;
let pool = get_by_index(self.pools.as_slice(), idx, "load decommission background pool")?.clone(); let pool = get_by_index(self.pools.as_slice(), idx, "load decommission background pool")?.clone();
let pending = { let pending = {
@@ -17253,6 +17257,44 @@ mod tests {
); );
} }
#[tokio::test]
#[serial_test::serial]
async fn decommission_worker_rechecks_runtime_capacity_before_empty_background_completion() {
let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
let enough = vec![
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
];
set_decommission_capacity_info_overrides_for_test(store.id, vec![enough.clone()]);
store
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
.await
.expect("the initial reservation should be activated");
let shortage = vec![enough[0], DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)];
set_decommission_capacity_info_overrides_for_test(store.id, vec![shortage]);
let canceler = DecommissionCanceler::new(CancellationToken::new());
store.decommission_cancelers.write().await[0] = Some(canceler.clone());
store
.do_decommission_in_routine(canceler, 0, Arc::new(Semaphore::new(1)))
.await
.expect("runtime capacity shortage should pause the worker before background completion");
let local = store.pool_meta.read().await;
let info = local.pools[0]
.decommission
.as_ref()
.expect("the blocked decommission state should remain present");
assert!(!info.complete && !info.failed && !info.canceled);
assert!(info.capacity_blocked_reason.is_some());
assert!(
info.capacity_reservation
.as_ref()
.is_some_and(DecommissionCapacityReservation::active)
);
}
fn pool_meta_replica_test_meta(cmd_line: &str) -> PoolMeta { fn pool_meta_replica_test_meta(cmd_line: &str) -> PoolMeta {
PoolMeta { PoolMeta {
version: POOL_META_VERSION, version: POOL_META_VERSION,
+1 -1
View File
@@ -286,7 +286,7 @@ pub struct QuotaAdmission {
pub struct LifecycleDeleteAllRequest { pub struct LifecycleDeleteAllRequest {
pub(crate) version_id: Option<Uuid>, pub(crate) version_id: Option<Uuid>,
pub(crate) delete_marker: bool, pub(crate) delete_marker: bool,
pub(crate) action: rustfs_scanner_contracts::metrics::IlmAction, pub(crate) action: rustfs_scanner_metrics::metrics::IlmAction,
pub(crate) rule_id: String, pub(crate) rule_id: String,
pub(crate) phase: LifecycleDeleteAllPhase, pub(crate) phase: LifecycleDeleteAllPhase,
} }
+26 -26
View File
@@ -32,7 +32,7 @@ use rustfs_madmin::metrics::{
ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, ScannerSourceWorkSnapshot as MadminScannerSourceWorkSnapshot, ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, ScannerSourceWorkSnapshot as MadminScannerSourceWorkSnapshot,
ScannerUsageFreshnessSnapshot as MadminScannerUsageFreshnessSnapshot, TimedAction as MadminTimedAction, ScannerUsageFreshnessSnapshot as MadminScannerUsageFreshnessSnapshot, TimedAction as MadminTimedAction,
}; };
use rustfs_scanner_contracts::metrics::global_metrics; use rustfs_scanner_metrics::metrics::global_metrics;
use rustfs_utils::os::get_drive_stats; use rustfs_utils::os::get_drive_stats;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
@@ -82,7 +82,7 @@ fn unix_millis_to_jiff_timestamp(millis: u64, fallback: Timestamp) -> Timestamp
} }
} }
fn to_madmin_scanner_metrics(metrics: rustfs_scanner_contracts::metrics::ScannerMetricsReport) -> MadminScannerMetrics { fn to_madmin_scanner_metrics(metrics: rustfs_scanner_metrics::metrics::ScannerMetricsReport) -> MadminScannerMetrics {
MadminScannerMetrics { MadminScannerMetrics {
collected_at: metrics.collected_at, collected_at: metrics.collected_at,
current_cycle: metrics.current_cycle, current_cycle: metrics.current_cycle,
@@ -565,7 +565,7 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
mod test { mod test {
use super::*; use super::*;
use rustfs_io_metrics::internode_metrics::global_internode_metrics; use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_scanner_contracts::metrics::CurrentCycle; use rustfs_scanner_metrics::metrics::CurrentCycle;
use serial_test::serial; use serial_test::serial;
use std::time::Duration; use std::time::Duration;
@@ -619,7 +619,7 @@ mod test {
#[test] #[test]
fn scanner_metrics_mapping_preserves_partial_source_status() { fn scanner_metrics_mapping_preserves_partial_source_status() {
let current_started = Utc::now() - chrono::Duration::seconds(5); let current_started = Utc::now() - chrono::Duration::seconds(5);
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport { let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
current_cycle_active: true, current_cycle_active: true,
current_started: chrono_to_jiff_timestamp(current_started), current_started: chrono_to_jiff_timestamp(current_started),
last_cycle_partial_source: "usage".to_string(), last_cycle_partial_source: "usage".to_string(),
@@ -628,7 +628,7 @@ mod test {
cycle_recovery_required_total: 2, cycle_recovery_required_total: 2,
cycle_last_progress_age: 17, cycle_last_progress_age: 17,
leader_lease_without_progress: true, leader_lease_without_progress: true,
partial_cycles_by_source: vec![rustfs_scanner_contracts::metrics::ScannerSourceCycleSnapshot { partial_cycles_by_source: vec![rustfs_scanner_metrics::metrics::ScannerSourceCycleSnapshot {
source: "usage".to_string(), source: "usage".to_string(),
cycles: 2, cycles: 2,
}], }],
@@ -654,11 +654,11 @@ mod test {
#[tokio::test] #[tokio::test]
#[serial] #[serial]
async fn collect_local_metrics_preserves_scanner_cycle_started_time() { async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
let previous_init_time = *rustfs_scanner_contracts::GLOBAL_INIT_TIME.read().await; let previous_init_time = *rustfs_scanner_metrics::GLOBAL_INIT_TIME.read().await;
let previous_cycle = global_metrics().get_cycle().await; let previous_cycle = global_metrics().get_cycle().await;
let init_time = Utc::now() - chrono::Duration::hours(1); let init_time = Utc::now() - chrono::Duration::hours(1);
let cycle_started = Utc::now() - chrono::Duration::seconds(5); let cycle_started = Utc::now() - chrono::Duration::seconds(5);
*rustfs_scanner_contracts::GLOBAL_INIT_TIME.write().await = Some(init_time); *rustfs_scanner_metrics::GLOBAL_INIT_TIME.write().await = Some(init_time);
let cycle = CurrentCycle { let cycle = CurrentCycle {
current: 0, current: 0,
next: 1, next: 1,
@@ -673,7 +673,7 @@ mod test {
.finish_scan_cycle_work_with_cycle(cycle_start, previous_cycle.clone().unwrap_or_default()) .finish_scan_cycle_work_with_cycle(cycle_start, previous_cycle.clone().unwrap_or_default())
.await; .await;
global_metrics().set_cycle(previous_cycle).await; global_metrics().set_cycle(previous_cycle).await;
*rustfs_scanner_contracts::GLOBAL_INIT_TIME.write().await = previous_init_time; *rustfs_scanner_metrics::GLOBAL_INIT_TIME.write().await = previous_init_time;
let encoded = rmp_serde::to_vec_named(&realtime).expect("realtime metrics should encode"); let encoded = rmp_serde::to_vec_named(&realtime).expect("realtime metrics should encode");
let decoded: RealtimeMetrics = rmp_serde::from_slice(&encoded).expect("realtime metrics should decode"); let decoded: RealtimeMetrics = rmp_serde::from_slice(&encoded).expect("realtime metrics should decode");
@@ -686,8 +686,8 @@ mod test {
#[test] #[test]
fn scanner_metrics_mapping_preserves_pacing_pressure() { fn scanner_metrics_mapping_preserves_pacing_pressure() {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport { let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
pacing_pressure: rustfs_scanner_contracts::metrics::ScannerPacingPressureSnapshot { pacing_pressure: rustfs_scanner_metrics::metrics::ScannerPacingPressureSnapshot {
primary_pressure: "cycle_budget".to_string(), primary_pressure: "cycle_budget".to_string(),
current_queued_scans: 4, current_queued_scans: 4,
current_active_scans: 2, current_active_scans: 2,
@@ -712,12 +712,12 @@ mod test {
#[test] #[test]
fn scanner_metrics_mapping_preserves_lifecycle_transition_status() { fn scanner_metrics_mapping_preserves_lifecycle_transition_status() {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport { let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
current_cycle_lifecycle_expiry_actions: 2, current_cycle_lifecycle_expiry_actions: 2,
current_cycle_lifecycle_transition_actions: 3, current_cycle_lifecycle_transition_actions: 3,
last_cycle_lifecycle_expiry_actions: 5, last_cycle_lifecycle_expiry_actions: 5,
last_cycle_lifecycle_transition_actions: 7, last_cycle_lifecycle_transition_actions: 7,
lifecycle_expiry: rustfs_scanner_contracts::metrics::ScannerLifecycleExpirySnapshot { lifecycle_expiry: rustfs_scanner_metrics::metrics::ScannerLifecycleExpirySnapshot {
current_queue_capacity: 16, current_queue_capacity: 16,
current_queued: 5, current_queued: 5,
current_active: 2, current_active: 2,
@@ -729,7 +729,7 @@ mod test {
scanner_not_enqueued: 2, scanner_not_enqueued: 2,
delete_failed: 1, delete_failed: 1,
}, },
lifecycle_transition: rustfs_scanner_contracts::metrics::ScannerLifecycleTransitionSnapshot { lifecycle_transition: rustfs_scanner_metrics::metrics::ScannerLifecycleTransitionSnapshot {
current_queue_capacity: 16, current_queue_capacity: 16,
current_queued: 5, current_queued: 5,
current_active: 2, current_active: 2,
@@ -778,10 +778,10 @@ mod test {
#[test] #[test]
fn scanner_metrics_mapping_preserves_maintenance_control_status() { fn scanner_metrics_mapping_preserves_maintenance_control_status() {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport { let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
maintenance_control: rustfs_scanner_contracts::metrics::ScannerMaintenanceControlSnapshot { maintenance_control: rustfs_scanner_metrics::metrics::ScannerMaintenanceControlSnapshot {
primary_control: "blocked_source".to_string(), primary_control: "blocked_source".to_string(),
sources: vec![rustfs_scanner_contracts::metrics::ScannerMaintenanceSourceSnapshot { sources: vec![rustfs_scanner_metrics::metrics::ScannerMaintenanceSourceSnapshot {
source: "lifecycle".to_string(), source: "lifecycle".to_string(),
state: "blocked".to_string(), state: "blocked".to_string(),
reason: "missed_work".to_string(), reason: "missed_work".to_string(),
@@ -815,8 +815,8 @@ mod test {
#[test] #[test]
fn scanner_metrics_mapping_preserves_usage_freshness_status() { fn scanner_metrics_mapping_preserves_usage_freshness_status() {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport { let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
usage_freshness: rustfs_scanner_contracts::metrics::ScannerUsageFreshnessSnapshot { usage_freshness: rustfs_scanner_metrics::metrics::ScannerUsageFreshnessSnapshot {
dirty_pending_buckets: 3, dirty_pending_buckets: 3,
last_dirty_mark_unix_secs: 10, last_dirty_mark_unix_secs: 10,
last_dirty_clear_unix_secs: 11, last_dirty_clear_unix_secs: 11,
@@ -857,7 +857,7 @@ mod test {
#[test] #[test]
fn scanner_metrics_mapping_preserves_distributed_status_fields() { fn scanner_metrics_mapping_preserves_distributed_status_fields() {
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport { let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
active_scan_paths: 2, active_scan_paths: 2,
oldest_active_path_age_seconds: 45, oldest_active_path_age_seconds: 45,
active_paths: vec!["disk-a/bucket-a".to_string(), "disk-b/bucket-b".to_string()], active_paths: vec!["disk-a/bucket-a".to_string(), "disk-b/bucket-b".to_string()],
@@ -913,7 +913,7 @@ mod test {
cycle_max_directories: 38, cycle_max_directories: 38,
bitrot_cycle_enabled: true, bitrot_cycle_enabled: true,
bitrot_cycle_seconds: 39.0, bitrot_cycle_seconds: 39.0,
scan_checkpoint: Some(rustfs_scanner_contracts::metrics::ScannerCheckpointReport { scan_checkpoint: Some(rustfs_scanner_metrics::metrics::ScannerCheckpointReport {
version: 1, version: 1,
resume_after: "bucket-a/prefix-a".to_string(), resume_after: "bucket-a/prefix-a".to_string(),
reason: "directories".to_string(), reason: "directories".to_string(),
@@ -923,7 +923,7 @@ mod test {
scan_checkpoint_cleared: 41, scan_checkpoint_cleared: 41,
scan_checkpoint_ignored: 42, scan_checkpoint_ignored: 42,
scan_checkpoint_stale: 43, scan_checkpoint_stale: 43,
source_work: vec![rustfs_scanner_contracts::metrics::ScannerSourceWorkSnapshot { source_work: vec![rustfs_scanner_metrics::metrics::ScannerSourceWorkSnapshot {
source: "usage".to_string(), source: "usage".to_string(),
checked: 44, checked: 44,
queued: 45, queued: 45,
@@ -932,7 +932,7 @@ mod test {
skipped: 48, skipped: 48,
missed: 49, missed: 49,
}], }],
current_cycle_source_work: vec![rustfs_scanner_contracts::metrics::ScannerSourceWorkSnapshot { current_cycle_source_work: vec![rustfs_scanner_metrics::metrics::ScannerSourceWorkSnapshot {
source: "lifecycle".to_string(), source: "lifecycle".to_string(),
checked: 50, checked: 50,
queued: 51, queued: 51,
@@ -941,7 +941,7 @@ mod test {
skipped: 54, skipped: 54,
missed: 55, missed: 55,
}], }],
last_cycle_source_work: vec![rustfs_scanner_contracts::metrics::ScannerSourceWorkSnapshot { last_cycle_source_work: vec![rustfs_scanner_metrics::metrics::ScannerSourceWorkSnapshot {
source: "heal".to_string(), source: "heal".to_string(),
checked: 56, checked: 56,
queued: 57, queued: 57,
@@ -950,7 +950,7 @@ mod test {
skipped: 60, skipped: 60,
missed: 61, missed: 61,
}], }],
replication_repair: vec![rustfs_scanner_contracts::metrics::ScannerReplicationRepairSnapshot { replication_repair: vec![rustfs_scanner_metrics::metrics::ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(), source: "bucket_replication".to_string(),
kind: "object".to_string(), kind: "object".to_string(),
scanner_role: "repair_admission".to_string(), scanner_role: "repair_admission".to_string(),
@@ -962,7 +962,7 @@ mod test {
skipped: 66, skipped: 66,
missed: 67, missed: 67,
}], }],
current_cycle_replication_repair: vec![rustfs_scanner_contracts::metrics::ScannerReplicationRepairSnapshot { current_cycle_replication_repair: vec![rustfs_scanner_metrics::metrics::ScannerReplicationRepairSnapshot {
source: "bucket_replication".to_string(), source: "bucket_replication".to_string(),
kind: "delete_marker".to_string(), kind: "delete_marker".to_string(),
scanner_role: "repair_admission".to_string(), scanner_role: "repair_admission".to_string(),
@@ -974,7 +974,7 @@ mod test {
skipped: 72, skipped: 72,
missed: 73, missed: 73,
}], }],
last_cycle_replication_repair: vec![rustfs_scanner_contracts::metrics::ScannerReplicationRepairSnapshot { last_cycle_replication_repair: vec![rustfs_scanner_metrics::metrics::ScannerReplicationRepairSnapshot {
source: "site_replication".to_string(), source: "site_replication".to_string(),
kind: "active_resync".to_string(), kind: "active_resync".to_string(),
scanner_role: "boundary_signal".to_string(), scanner_role: "boundary_signal".to_string(),
+3 -3
View File
@@ -1168,7 +1168,7 @@ mod lifecycle_delete_all_plan_tests {
crate::object_api::LifecycleDeleteAllRequest { crate::object_api::LifecycleDeleteAllRequest {
version_id: Some(version_id), version_id: Some(version_id),
delete_marker: true, delete_marker: true,
action: rustfs_scanner_contracts::metrics::IlmAction::DelMarkerDeleteAllVersionsAction, action: rustfs_scanner_metrics::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
rule_id: "rule".to_string(), rule_id: "rule".to_string(),
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight, phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
} }
@@ -1361,7 +1361,7 @@ mod lifecycle_delete_all_plan_tests {
let request = crate::object_api::LifecycleDeleteAllRequest { let request = crate::object_api::LifecycleDeleteAllRequest {
version_id: None, version_id: None,
delete_marker: false, delete_marker: false,
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction, action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
rule_id: "rule".to_string(), rule_id: "rule".to_string(),
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight, phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
}; };
@@ -19363,7 +19363,7 @@ mod delete_objects_lock_gating_tests {
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest { lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
version_id: Some(trigger_version_id), version_id: Some(trigger_version_id),
delete_marker: false, delete_marker: false,
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction, action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
rule_id: "rule".to_string(), rule_id: "rule".to_string(),
phase: crate::object_api::LifecycleDeleteAllPhase::History, phase: crate::object_api::LifecycleDeleteAllPhase::History,
}), }),
+2 -2
View File
@@ -15934,7 +15934,7 @@ mod tests {
.expect("unknown transition metadata should be written"); .expect("unknown transition metadata should be written");
} }
let lifecycle_event = crate::bucket::lifecycle::lifecycle::Event { let lifecycle_event = crate::bucket::lifecycle::lifecycle::Event {
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction, action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
rule_id: "delete-all-versions".to_string(), rule_id: "delete-all-versions".to_string(),
..Default::default() ..Default::default()
}; };
@@ -16186,7 +16186,7 @@ mod tests {
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest { lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
version_id: original.version_id, version_id: original.version_id,
delete_marker: false, delete_marker: false,
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction, action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
rule_id: "rule".to_string(), rule_id: "rule".to_string(),
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight, phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
}), }),
+1 -1
View File
@@ -6448,7 +6448,7 @@ mod tests {
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest { lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
version_id: Some(version_id), version_id: Some(version_id),
delete_marker: false, delete_marker: false,
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction, action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
rule_id: "delete-all".to_string(), rule_id: "delete-all".to_string(),
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight, phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
}), }),
+2 -2
View File
@@ -1322,7 +1322,7 @@ mod tests {
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest { lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
version_id: Some(trigger_id), version_id: Some(trigger_id),
delete_marker: false, delete_marker: false,
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction, action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
rule_id: "rule".to_string(), rule_id: "rule".to_string(),
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight, phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
}), }),
@@ -1492,7 +1492,7 @@ mod tests {
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest { lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
version_id: Some(marker_id), version_id: Some(marker_id),
delete_marker: true, delete_marker: true,
action: rustfs_scanner_contracts::metrics::IlmAction::DelMarkerDeleteAllVersionsAction, action: rustfs_scanner_metrics::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
rule_id: "rule".to_string(), rule_id: "rule".to_string(),
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight, phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
}), }),
+1 -1
View File
@@ -53,7 +53,7 @@ hotpath-cpu = [
hotpath.workspace = true hotpath.workspace = true
async-trait.workspace = true async-trait.workspace = true
metrics.workspace = true metrics.workspace = true
rustfs-scanner-contracts.workspace = true rustfs-scanner-metrics.workspace = true
rustfs-config = { workspace = true, features = ["constants"] } rustfs-config = { workspace = true, features = ["constants"] }
rustfs-replication.workspace = true rustfs-replication.workspace = true
rustfs-storage-api.workspace = true rustfs-storage-api.workspace = true
+1 -1
View File
@@ -62,7 +62,7 @@ const ERR_LIFECYCLE_EXPIRED_OBJECT_DELETE_MARKER_WITH_TAGS: &str =
const ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION: &str = "Rule must have at least one of Expiration, Transition, NoncurrentVersionExpiration, NoncurrentVersionTransition, or DelMarkerExpiration"; const ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION: &str = "Rule must have at least one of Expiration, Transition, NoncurrentVersionExpiration, NoncurrentVersionTransition, or DelMarkerExpiration";
const ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT: &str = "Legacy Prefix and Filter cannot both be present in a lifecycle rule. Use Filter.Prefix instead of the top-level Prefix element."; const ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT: &str = "Legacy Prefix and Filter cannot both be present in a lifecycle rule. Use Filter.Prefix instead of the top-level Prefix element.";
pub use rustfs_scanner_contracts::metrics::IlmAction; pub use rustfs_scanner_metrics::metrics::IlmAction;
#[async_trait::async_trait] #[async_trait::async_trait]
pub trait RuleValidate { pub trait RuleValidate {
+2 -2
View File
@@ -19,7 +19,7 @@ use time::OffsetDateTime;
use tracing::info; use tracing::info;
use rustfs_replication::ReplicationStatusType; use rustfs_replication::ReplicationStatusType;
use rustfs_scanner_contracts::metrics::IlmAction; use rustfs_scanner_metrics::metrics::IlmAction;
use crate::object_lock; use crate::object_lock;
use crate::{Event, Lifecycle, ObjectOpts}; use crate::{Event, Lifecycle, ObjectOpts};
@@ -197,7 +197,7 @@ mod tests {
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use rustfs_scanner_contracts::metrics::IlmAction; use rustfs_scanner_metrics::metrics::IlmAction;
use s3s::dto::{ use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule, BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule,
NoncurrentVersionExpiration, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, NoncurrentVersionExpiration, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule,
+1 -1
View File
@@ -21,4 +21,4 @@ mod tagging;
pub use core::*; pub use core::*;
pub use evaluator::Evaluator; pub use evaluator::Evaluator;
pub use rustfs_replication::{ReplicationStatusType, VersionPurgeStatusType}; pub use rustfs_replication::{ReplicationStatusType, VersionPurgeStatusType};
pub use rustfs_scanner_contracts::metrics::IlmAction; pub use rustfs_scanner_metrics::metrics::IlmAction;
+1 -1
View File
@@ -106,7 +106,7 @@ hotpath.workspace = true
rustfs-audit = { workspace = true } rustfs-audit = { workspace = true }
rustfs-common = { workspace = true } rustfs-common = { workspace = true }
rustfs-heal-contracts = { workspace = true } rustfs-heal-contracts = { workspace = true }
rustfs-scanner-contracts = { workspace = true } rustfs-scanner-metrics = { workspace = true }
rustfs-config = { workspace = true, features = ["observability"] } rustfs-config = { workspace = true, features = ["observability"] }
# NOTE: This dependency on rustfs-ecstore is a known architectural limitation. # NOTE: This dependency on rustfs-ecstore is a known architectural limitation.
# The obs crate imports types from ecstore for metrics collection. # The obs crate imports types from ecstore for metrics collection.
+1 -1
View File
@@ -493,7 +493,7 @@ mod tests {
use super::*; use super::*;
use crate::metrics::report::report_metrics; use crate::metrics::report::report_metrics;
use metrics_util::debugging::DebuggingRecorder; use metrics_util::debugging::DebuggingRecorder;
use rustfs_scanner_contracts::metrics::{Metric, Metrics}; use rustfs_scanner_metrics::metrics::{Metric, Metrics};
fn prometheus_counter_name(name: &str) -> String { fn prometheus_counter_name(name: &str) -> String {
if name.ends_with("_total") { if name.ends_with("_total") {
+4 -4
View File
@@ -45,7 +45,7 @@ use rustfs_io_metrics::{
ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, s3_op_metrics_snapshot, ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, s3_op_metrics_snapshot,
snapshot_process_resource_and_system, snapshot_process_resource_and_system_with, snapshot_process_resource_and_system, snapshot_process_resource_and_system_with,
}; };
use rustfs_scanner_contracts::metrics::{ use rustfs_scanner_metrics::metrics::{
ScannerActiveBucketDriveSnapshot, ScannerBucketDriveResultSnapshot, ScannerMetricsReport, ScannerSourceWorkSnapshot, ScannerActiveBucketDriveSnapshot, ScannerBucketDriveResultSnapshot, ScannerMetricsReport, ScannerSourceWorkSnapshot,
global_metrics, global_metrics,
}; };
@@ -1679,7 +1679,7 @@ pub async fn collect_compression_cluster_stats() -> Option<CompressionClusterSta
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use rustfs_scanner_contracts::metrics::ScannerSourceWorkSnapshot; use rustfs_scanner_metrics::metrics::ScannerSourceWorkSnapshot;
use std::io::{Read, Write}; use std::io::{Read, Write};
use std::net::{Shutdown, TcpListener, TcpStream}; use std::net::{Shutdown, TcpListener, TcpStream};
use std::thread; use std::thread;
@@ -2134,7 +2134,7 @@ mod tests {
#[test] #[test]
fn ilm_detail_stats_keep_expiry_and_transition_results_separate() { fn ilm_detail_stats_keep_expiry_and_transition_results_separate() {
let report = ScannerMetricsReport { let report = ScannerMetricsReport {
lifecycle_expiry: rustfs_scanner_contracts::metrics::ScannerLifecycleExpirySnapshot { lifecycle_expiry: rustfs_scanner_metrics::metrics::ScannerLifecycleExpirySnapshot {
current_queued: 2, current_queued: 2,
current_active: 1, current_active: 1,
scanner_queued: 10, scanner_queued: 10,
@@ -2142,7 +2142,7 @@ mod tests {
delete_failed: 4, delete_failed: 4,
..Default::default() ..Default::default()
}, },
lifecycle_transition: rustfs_scanner_contracts::metrics::ScannerLifecycleTransitionSnapshot { lifecycle_transition: rustfs_scanner_metrics::metrics::ScannerLifecycleTransitionSnapshot {
current_queued: 5, current_queued: 5,
current_active: 6, current_active: 6,
queue_full: 7, queue_full: 7,
+2 -12
View File
@@ -20,26 +20,16 @@ license.workspace = true
repository.workspace = true repository.workspace = true
rust-version.workspace = true rust-version.workspace = true
homepage.workspace = true homepage.workspace = true
description = "Scanner metrics and lifecycle-cycle contracts shared by the scanner, storage engine, and observability layers." description = "Scanner storage and wire contracts shared by the scanner and storage engine."
keywords = ["scanner", "contracts", "metrics", "rustfs", "Minio"] keywords = ["scanner", "contracts", "rustfs", "Minio"]
categories = ["web-programming", "development-tools", "data-structures"] categories = ["web-programming", "development-tools", "data-structures"]
[lints] [lints]
workspace = true workspace = true
[dependencies] [dependencies]
chrono = { workspace = true, features = ["serde"] }
jiff = { workspace = true, features = ["serde"] }
metrics = { workspace = true }
rmp-serde = { workspace = true }
rustfs-heal-contracts = { workspace = true }
serde = { workspace = true, features = ["derive"] }
tokio = { workspace = true, features = ["sync"] }
[dev-dependencies] [dev-dependencies]
serde_json = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
uuid = { workspace = true, features = ["v4"] }
[lib] [lib]
doctest = false doctest = false
-6
View File
@@ -11,9 +11,3 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
pub mod init_time;
pub mod last_minute;
pub mod metrics;
pub use init_time::{GLOBAL_INIT_TIME, get_global_init_time, set_global_init_time_now};
+45
View File
@@ -0,0 +1,45 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[package]
name = "rustfs-scanner-metrics"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
homepage.workspace = true
description = "Scanner metrics and lifecycle-cycle telemetry shared by the scanner, storage engine, and observability layers."
keywords = ["scanner", "metrics", "rustfs", "Minio"]
categories = ["web-programming", "development-tools", "data-structures"]
[lints]
workspace = true
[dependencies]
chrono = { workspace = true, features = ["serde"] }
jiff = { workspace = true, features = ["serde"] }
metrics = { workspace = true }
rmp-serde = { workspace = true }
rustfs-heal-contracts = { workspace = true }
serde = { workspace = true, features = ["derive"] }
tokio = { workspace = true, features = ["sync"] }
[dev-dependencies]
serde_json = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
uuid = { workspace = true, features = ["v4"] }
[lib]
doctest = false
+19
View File
@@ -0,0 +1,19 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod init_time;
pub mod last_minute;
pub mod metrics;
pub use init_time::{GLOBAL_INIT_TIME, get_global_init_time, set_global_init_time_now};
+2 -1
View File
@@ -73,8 +73,9 @@ hotpath-cpu = [
hotpath.workspace = true hotpath.workspace = true
rustfs-config = { workspace = true, features = ["server-config-model"] } rustfs-config = { workspace = true, features = ["server-config-model"] }
rustfs-common = { workspace = true } rustfs-common = { workspace = true }
rustfs-concurrency = { workspace = true }
rustfs-heal-contracts = { workspace = true } rustfs-heal-contracts = { workspace = true }
rustfs-scanner-contracts = { workspace = true } rustfs-scanner-metrics = { workspace = true }
rustfs-credentials = { workspace = true } rustfs-credentials = { workspace = true }
rustfs-utils = { workspace = true } rustfs-utils = { workspace = true }
tokio = { workspace = true, features = ["fs", "sync", "time", "macros", "rt-multi-thread"] } tokio = { workspace = true, features = ["fs", "sync", "time", "macros", "rt-multi-thread"] }
+3 -1
View File
@@ -70,6 +70,7 @@ mod scanner_heal_admission_baseline;
pub mod scanner_io; pub mod scanner_io;
pub mod sleeper; pub mod sleeper;
pub(crate) mod storage_api; pub(crate) mod storage_api;
mod workload_admission;
pub use data_usage_define::*; pub use data_usage_define::*;
pub use error::ScannerError; pub use error::ScannerError;
@@ -80,7 +81,7 @@ pub use remote_scanner::{
remote_scanner_request_matches_envelope, serve_remote_scanner_request, validate_remote_scanner_request_fence, remote_scanner_request_matches_envelope, serve_remote_scanner_request, validate_remote_scanner_request_fence,
}; };
pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config}; pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
pub use rustfs_scanner_contracts::last_minute; pub use rustfs_scanner_metrics::last_minute;
pub use scanner::{ pub use scanner::{
ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason, ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason,
ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, ScannerUsageStateResetResult, ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, ScannerUsageStateResetResult,
@@ -96,6 +97,7 @@ pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::atomic::{AtomicU64, Ordering};
pub use storage_api::ScannerReplicationConfig as ReplicationConfig; pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
pub use storage_api::scan::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION}; pub use storage_api::scan::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
pub use workload_admission::set_scanner_workload_admission_snapshot_provider;
static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0); static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
static SCANNER_RUNTIME_INSTANCES: AtomicU64 = AtomicU64::new(0); static SCANNER_RUNTIME_INSTANCES: AtomicU64 = AtomicU64::new(0);
+1 -1
View File
@@ -28,7 +28,7 @@ use crate::{
use hmac::{Hmac, KeyInit, Mac}; use hmac::{Hmac, KeyInit, Mac};
use rustfs_credentials::try_get_rpc_token; use rustfs_credentials::try_get_rpc_token;
use rustfs_heal_contracts::heal_channel::HealScanMode; use rustfs_heal_contracts::heal_channel::HealScanMode;
use rustfs_scanner_contracts::metrics::{Metric, Metrics}; use rustfs_scanner_metrics::metrics::{Metric, Metrics};
use rustfs_utils::path::path_join_buf; use rustfs_utils::path::path_join_buf;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use sha2::Sha256; use sha2::Sha256;
+1 -1
View File
@@ -52,7 +52,7 @@ use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELA
use rustfs_data_usage::observed_data_usage_is_newer; use rustfs_data_usage::observed_data_usage_is_newer;
use rustfs_heal_contracts::heal_channel::HealScanMode; use rustfs_heal_contracts::heal_channel::HealScanMode;
use rustfs_lock::{NamespaceLockGuard, error::LockError}; use rustfs_lock::{NamespaceLockGuard, error::LockError};
use rustfs_scanner_contracts::metrics::{ use rustfs_scanner_metrics::metrics::{
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource, CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource,
emit_scan_cycle_complete, emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, emit_scan_cycle_complete, emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded,
global_metrics, global_metrics,
+1 -1
View File
@@ -50,7 +50,7 @@ use rustfs_heal_contracts::heal_channel::{
HEAL_DELETE_DANGLING, HealAdmissionDropReason, HealAdmissionResult, HealChannelPriority, HealChannelRequest, HEAL_DELETE_DANGLING, HealAdmissionDropReason, HealAdmissionResult, HealChannelPriority, HealChannelRequest,
HealRequestSource, HealScanMode, send_heal_request_with_admission, HealRequestSource, HealScanMode, send_heal_request_with_admission,
}; };
use rustfs_scanner_contracts::metrics::{ use rustfs_scanner_metrics::metrics::{
CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource, CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource,
UpdateCurrentPathFn, current_path_updater, global_metrics, UpdateCurrentPathFn, current_path_updater, global_metrics,
}; };
+1 -1
View File
@@ -30,7 +30,7 @@ use rustfs_data_usage::{BucketTargetUsageInfo, BucketUsageInfo};
use rustfs_filemeta::FileMeta; use rustfs_filemeta::FileMeta;
use rustfs_heal_contracts::heal_channel::HealScanMode; use rustfs_heal_contracts::heal_channel::HealScanMode;
use rustfs_lock::{LockError, NamespaceLockGuard}; use rustfs_lock::{LockError, NamespaceLockGuard};
use rustfs_scanner_contracts::metrics::{ use rustfs_scanner_metrics::metrics::{
Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics, Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics,
}; };
use rustfs_utils::path::path_join_buf; use rustfs_utils::path::path_join_buf;
+4 -4
View File
@@ -147,13 +147,13 @@ impl Drop for DiskBucketScanActiveGuard {
pub(super) struct BucketDriveFailureGuard { pub(super) struct BucketDriveFailureGuard {
failed: bool, failed: bool,
source: rustfs_scanner_contracts::metrics::ScannerWorkSource, source: rustfs_scanner_metrics::metrics::ScannerWorkSource,
bucket: String, bucket: String,
drive: String, drive: String,
} }
impl BucketDriveFailureGuard { impl BucketDriveFailureGuard {
pub(super) fn new(source: rustfs_scanner_contracts::metrics::ScannerWorkSource, bucket: &str, drive: &str) -> Self { pub(super) fn new(source: rustfs_scanner_metrics::metrics::ScannerWorkSource, bucket: &str, drive: &str) -> Self {
Self { Self {
failed: true, failed: true,
source, source,
@@ -245,7 +245,7 @@ pub(super) fn scanner_concurrency_limit(configured: usize, available: usize) ->
return 0; return 0;
} }
if crate::current_foreground_read_activity() > 0 { if crate::workload_admission::foreground_workload_activity() > 0 {
return 1; return 1;
} }
@@ -285,7 +285,7 @@ pub(super) fn scanner_task_join_error(stage: &str, err: tokio::task::JoinError)
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use rustfs_scanner_contracts::metrics::{ScannerWorkSource, global_metrics}; use rustfs_scanner_metrics::metrics::{ScannerWorkSource, global_metrics};
use tokio::sync::oneshot; use tokio::sync::oneshot;
fn active_bucket_drive_count(source: ScannerWorkSource, bucket: &str, drive: &str) -> u64 { fn active_bucket_drive_count(source: ScannerWorkSource, bucket: &str, drive: &str) -> u64 {
+2 -2
View File
@@ -161,8 +161,8 @@ impl ScannerIODisk for Disk {
let bucket = cache.info.name.clone(); let bucket = cache.info.name.clone();
let disk_path = self.path().to_string_lossy().to_string(); let disk_path = self.path().to_string_lossy().to_string();
let source = match scan_mode { let source = match scan_mode {
HealScanMode::Deep => rustfs_scanner_contracts::metrics::ScannerWorkSource::Bitrot, HealScanMode::Deep => rustfs_scanner_metrics::metrics::ScannerWorkSource::Bitrot,
HealScanMode::Normal | HealScanMode::Unknown => rustfs_scanner_contracts::metrics::ScannerWorkSource::Usage, HealScanMode::Normal | HealScanMode::Unknown => rustfs_scanner_metrics::metrics::ScannerWorkSource::Usage,
}; };
global_metrics().record_scan_bucket_drive_start(source, &bucket, &disk_path); global_metrics().record_scan_bucket_drive_start(source, &bucket, &disk_path);
let mut failure_guard = BucketDriveFailureGuard::new(source, &bucket, &disk_path); let mut failure_guard = BucketDriveFailureGuard::new(source, &bucket, &disk_path);
+41
View File
@@ -26,12 +26,32 @@ use crate::{
ScannerPutObjReader, UNKNOWN_TIER, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, ScannerPutObjReader, UNKNOWN_TIER, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests,
init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path, init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
}; };
use rustfs_concurrency::{
AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadAdmissionSnapshotProvider,
WorkloadClass,
};
use rustfs_filemeta::FileInfo; use rustfs_filemeta::FileInfo;
use serial_test::serial; use serial_test::serial;
use std::sync::Arc;
use temp_env::with_var; use temp_env::with_var;
use time::OffsetDateTime; use time::OffsetDateTime;
use uuid::Uuid; use uuid::Uuid;
#[derive(Clone)]
struct FixedWorkloadProvider {
snapshot: WorkloadAdmissionRegistrySnapshot,
}
impl WorkloadAdmissionSnapshotProvider for FixedWorkloadProvider {
fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot {
self.snapshot.clone()
}
}
fn install_scanner_workload_provider(snapshot: WorkloadAdmissionRegistrySnapshot) {
crate::set_scanner_workload_admission_snapshot_provider(Arc::new(FixedWorkloadProvider { snapshot }));
}
fn bucket_info(name: &str) -> BucketInfo { fn bucket_info(name: &str) -> BucketInfo {
BucketInfo { BucketInfo {
name: name.to_string(), name: name.to_string(),
@@ -1079,6 +1099,7 @@ async fn bucket_cache_pending_heal_reaches_cycle_maintenance_state() {
#[serial] #[serial]
fn scanner_concurrency_limit_preserves_available_when_unconfigured() { fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
crate::reset_foreground_read_activity_for_test(); crate::reset_foreground_read_activity_for_test();
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
assert_eq!(scanner_concurrency_limit(0, 4), 4); assert_eq!(scanner_concurrency_limit(0, 4), 4);
} }
@@ -1086,6 +1107,7 @@ fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
#[serial] #[serial]
fn scanner_concurrency_limit_caps_to_configured_value() { fn scanner_concurrency_limit_caps_to_configured_value() {
crate::reset_foreground_read_activity_for_test(); crate::reset_foreground_read_activity_for_test();
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
assert_eq!(scanner_concurrency_limit(2, 4), 2); assert_eq!(scanner_concurrency_limit(2, 4), 2);
} }
@@ -1093,6 +1115,7 @@ fn scanner_concurrency_limit_caps_to_configured_value() {
#[serial] #[serial]
fn scanner_concurrency_limit_never_exceeds_available_work() { fn scanner_concurrency_limit_never_exceeds_available_work() {
crate::reset_foreground_read_activity_for_test(); crate::reset_foreground_read_activity_for_test();
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
assert_eq!(scanner_concurrency_limit(8, 4), 4); assert_eq!(scanner_concurrency_limit(8, 4), 4);
} }
@@ -1100,6 +1123,7 @@ fn scanner_concurrency_limit_never_exceeds_available_work() {
#[serial] #[serial]
fn scanner_concurrency_limit_handles_no_available_work() { fn scanner_concurrency_limit_handles_no_available_work() {
crate::reset_foreground_read_activity_for_test(); crate::reset_foreground_read_activity_for_test();
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
assert_eq!(scanner_concurrency_limit(2, 0), 0); assert_eq!(scanner_concurrency_limit(2, 0), 0);
} }
@@ -1107,16 +1131,33 @@ fn scanner_concurrency_limit_handles_no_available_work() {
#[serial] #[serial]
fn scanner_concurrency_limit_yields_to_foreground_reads() { fn scanner_concurrency_limit_yields_to_foreground_reads() {
crate::reset_foreground_read_activity_for_test(); crate::reset_foreground_read_activity_for_test();
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
crate::set_foreground_read_activity(8); crate::set_foreground_read_activity(8);
assert_eq!(scanner_concurrency_limit(0, 4), 1); assert_eq!(scanner_concurrency_limit(0, 4), 1);
assert_eq!(scanner_concurrency_limit(3, 4), 1); assert_eq!(scanner_concurrency_limit(3, 4), 1);
crate::reset_foreground_read_activity_for_test(); crate::reset_foreground_read_activity_for_test();
} }
#[test]
#[serial]
fn scanner_concurrency_limit_yields_to_shared_foreground_pressure() {
crate::reset_foreground_read_activity_for_test();
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
install_scanner_workload_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Open).with_counts(Some(2), None, Some(16)),
]));
assert_eq!(scanner_concurrency_limit(0, 4), 1);
assert_eq!(scanner_concurrency_limit(3, 4), 1);
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
}
#[test] #[test]
#[serial] #[serial]
fn scanner_concurrency_limit_yields_to_streaming_reads() { fn scanner_concurrency_limit_yields_to_streaming_reads() {
crate::reset_foreground_read_activity_for_test(); crate::reset_foreground_read_activity_for_test();
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
let _guard = crate::ForegroundReadGuard::new(); let _guard = crate::ForegroundReadGuard::new();
assert_eq!(scanner_concurrency_limit(0, 4), 1); assert_eq!(scanner_concurrency_limit(0, 4), 1);
+18 -16
View File
@@ -20,7 +20,7 @@ use rustfs_config::{
DEFAULT_SCANNER_IDLE_MODE, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED, DEFAULT_SCANNER_IDLE_MODE, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED,
ENV_SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed, ENV_SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed,
}; };
use rustfs_scanner_contracts::metrics::global_metrics; use rustfs_scanner_metrics::metrics::global_metrics;
use tokio::time::Duration; use tokio::time::Duration;
const MIN_SLEEP: Duration = Duration::from_millis(1); const MIN_SLEEP: Duration = Duration::from_millis(1);
@@ -29,8 +29,8 @@ const SCANNER_SPEED_FAST: u8 = 1;
const SCANNER_SPEED_DEFAULT: u8 = 2; const SCANNER_SPEED_DEFAULT: u8 = 2;
const SCANNER_SPEED_SLOW: u8 = 3; const SCANNER_SPEED_SLOW: u8 = 3;
const SCANNER_SPEED_SLOWEST: u8 = 4; const SCANNER_SPEED_SLOWEST: u8 = 4;
const FOREGROUND_READ_BACKOFF_PER_REQUEST_MS: u64 = 10; const FOREGROUND_WORKLOAD_BACKOFF_PER_REQUEST_MS: u64 = 10;
const FOREGROUND_READ_BACKOFF_MAX_MS: u64 = 250; const FOREGROUND_WORKLOAD_BACKOFF_MAX_MS: u64 = 250;
static SCANNER_DEFAULT_SPEED_PRESET: AtomicU8 = AtomicU8::new(SCANNER_SPEED_DEFAULT); static SCANNER_DEFAULT_SPEED_PRESET: AtomicU8 = AtomicU8::new(SCANNER_SPEED_DEFAULT);
@@ -78,15 +78,15 @@ pub(crate) fn scanner_yield_every_n_objects() -> u64 {
rustfs_utils::get_env_u64(ENV_SCANNER_YIELD_EVERY_N_OBJECTS, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS) rustfs_utils::get_env_u64(ENV_SCANNER_YIELD_EVERY_N_OBJECTS, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS)
} }
fn foreground_read_backoff_duration(active_reads: u64) -> Duration { fn foreground_workload_backoff_duration(active_foreground_workloads: u64) -> Duration {
if active_reads == 0 { if active_foreground_workloads == 0 {
return Duration::ZERO; return Duration::ZERO;
} }
Duration::from_millis( Duration::from_millis(
active_reads active_foreground_workloads
.saturating_mul(FOREGROUND_READ_BACKOFF_PER_REQUEST_MS) .saturating_mul(FOREGROUND_WORKLOAD_BACKOFF_PER_REQUEST_MS)
.min(FOREGROUND_READ_BACKOFF_MAX_MS), .min(FOREGROUND_WORKLOAD_BACKOFF_MAX_MS),
) )
} }
@@ -147,14 +147,15 @@ impl DynamicSleeper {
} }
let (factor, max_sleep) = self.read_params(); let (factor, max_sleep) = self.read_params();
if factor == 0.0 || max_sleep.is_zero() { if factor == 0.0 || max_sleep.is_zero() {
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity()); let foreground_sleep =
foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
if !foreground_sleep.is_zero() { if !foreground_sleep.is_zero() {
tokio::time::sleep(foreground_sleep).await; tokio::time::sleep(foreground_sleep).await;
global_metrics().record_scanner_throttle_sleep(foreground_sleep); global_metrics().record_scanner_throttle_sleep(foreground_sleep);
} }
return; return;
} }
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity()); let foreground_sleep = foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
let sleep_dur = Duration::from_secs_f64(MIN_SLEEP.as_secs_f64() * factor) let sleep_dur = Duration::from_secs_f64(MIN_SLEEP.as_secs_f64() * factor)
.min(max_sleep) .min(max_sleep)
.max(foreground_sleep); .max(foreground_sleep);
@@ -235,7 +236,8 @@ impl SleepTimer {
} }
let (factor, max_sleep) = self.sleeper.read_params(); let (factor, max_sleep) = self.sleeper.read_params();
if factor == 0.0 || max_sleep.is_zero() { if factor == 0.0 || max_sleep.is_zero() {
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity()); let foreground_sleep =
foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
if !foreground_sleep.is_zero() { if !foreground_sleep.is_zero() {
tokio::time::sleep(foreground_sleep).await; tokio::time::sleep(foreground_sleep).await;
global_metrics().record_scanner_throttle_sleep(foreground_sleep); global_metrics().record_scanner_throttle_sleep(foreground_sleep);
@@ -243,7 +245,7 @@ impl SleepTimer {
return; return;
} }
let elapsed = self.start.elapsed(); let elapsed = self.start.elapsed();
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity()); let foreground_sleep = foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
let sleep_dur = Duration::from_secs_f64(elapsed.as_secs_f64() * factor) let sleep_dur = Duration::from_secs_f64(elapsed.as_secs_f64() * factor)
.max(MIN_SLEEP) .max(MIN_SLEEP)
.min(max_sleep) .min(max_sleep)
@@ -304,10 +306,10 @@ mod tests {
} }
#[test] #[test]
fn foreground_read_backoff_is_capped() { fn foreground_workload_backoff_is_capped() {
assert_eq!(foreground_read_backoff_duration(0), Duration::ZERO); assert_eq!(foreground_workload_backoff_duration(0), Duration::ZERO);
assert_eq!(foreground_read_backoff_duration(1), Duration::from_millis(10)); assert_eq!(foreground_workload_backoff_duration(1), Duration::from_millis(10));
assert_eq!(foreground_read_backoff_duration(80), Duration::from_millis(250)); assert_eq!(foreground_workload_backoff_duration(80), Duration::from_millis(250));
} }
#[test] #[test]
+165
View File
@@ -0,0 +1,165 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::{Arc, LazyLock, RwLock};
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshotProvider, WorkloadClass};
type WorkloadSnapshotProviderRef = Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>;
static SCANNER_WORKLOAD_ADMISSION_PROVIDER: LazyLock<RwLock<Option<WorkloadSnapshotProviderRef>>> =
LazyLock::new(|| RwLock::new(None));
pub fn set_scanner_workload_admission_snapshot_provider(provider: WorkloadSnapshotProviderRef) {
*SCANNER_WORKLOAD_ADMISSION_PROVIDER
.write()
.unwrap_or_else(|err| err.into_inner()) = Some(provider);
}
fn scanner_workload_admission_snapshot_provider() -> Option<WorkloadSnapshotProviderRef> {
SCANNER_WORKLOAD_ADMISSION_PROVIDER
.read()
.unwrap_or_else(|err| err.into_inner())
.clone()
}
#[cfg(test)]
pub(crate) fn clear_scanner_workload_admission_snapshot_provider_for_test() {
*SCANNER_WORKLOAD_ADMISSION_PROVIDER
.write()
.unwrap_or_else(|err| err.into_inner()) = None;
}
pub(crate) fn foreground_workload_activity() -> u64 {
let local_activity = crate::current_foreground_read_activity();
let Some(provider) = scanner_workload_admission_snapshot_provider() else {
return local_activity;
};
local_activity.max(foreground_activity_from_snapshot(&provider.workload_admission_snapshot()))
}
fn foreground_activity_from_snapshot(snapshot: &WorkloadAdmissionRegistrySnapshot) -> u64 {
[WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite]
.into_iter()
.filter_map(|class| snapshot.get(class))
.map(|entry| {
entry
.active
.or_else(|| {
matches!(entry.state, AdmissionState::Saturated).then(|| entry.limit.filter(|limit| *limit > 0).unwrap_or(1))
})
.map(usize_to_u64_saturated)
.unwrap_or(0)
})
.max()
.unwrap_or(0)
}
fn usize_to_u64_saturated(value: usize) -> u64 {
u64::try_from(value).unwrap_or(u64::MAX)
}
#[cfg(test)]
mod tests {
use super::*;
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshot};
use serial_test::serial;
#[derive(Clone)]
struct FixedWorkloadProvider {
snapshot: WorkloadAdmissionRegistrySnapshot,
}
impl WorkloadAdmissionSnapshotProvider for FixedWorkloadProvider {
fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot {
self.snapshot.clone()
}
}
fn install_provider(snapshot: WorkloadAdmissionRegistrySnapshot) {
set_scanner_workload_admission_snapshot_provider(Arc::new(FixedWorkloadProvider { snapshot }));
}
#[test]
#[serial]
fn foreground_workload_activity_falls_back_to_local_read_activity() {
clear_scanner_workload_admission_snapshot_provider_for_test();
crate::reset_foreground_read_activity_for_test();
crate::set_foreground_read_activity(3);
assert_eq!(foreground_workload_activity(), 3);
crate::reset_foreground_read_activity_for_test();
}
#[test]
#[serial]
fn foreground_workload_activity_uses_shared_provider_counts() {
clear_scanner_workload_admission_snapshot_provider_for_test();
crate::reset_foreground_read_activity_for_test();
install_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Open).with_counts(
Some(5),
None,
Some(8),
),
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Open).with_counts(
Some(2),
None,
Some(4),
),
]));
assert_eq!(foreground_workload_activity(), 5);
clear_scanner_workload_admission_snapshot_provider_for_test();
}
#[test]
#[serial]
fn foreground_workload_activity_treats_saturation_without_counts_as_pressure() {
clear_scanner_workload_admission_snapshot_provider_for_test();
crate::reset_foreground_read_activity_for_test();
install_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Saturated).with_counts(
None,
None,
Some(7),
),
]));
assert_eq!(foreground_workload_activity(), 7);
clear_scanner_workload_admission_snapshot_provider_for_test();
}
#[test]
#[serial]
fn foreground_workload_activity_treats_zero_limit_saturation_as_pressure() {
clear_scanner_workload_admission_snapshot_provider_for_test();
crate::reset_foreground_read_activity_for_test();
install_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Saturated).with_counts(
None,
None,
Some(0),
),
]));
assert_eq!(foreground_workload_activity(), 1);
clear_scanner_workload_admission_snapshot_provider_for_test();
}
}
+2
View File
@@ -42,6 +42,8 @@ Required headings and strings in these files are asserted by `scripts/check_arch
| [workload-admission-contracts.md](workload-admission-contracts.md) | adding a workload class or snapshot provider, or consuming admission state from a background job | | [workload-admission-contracts.md](workload-admission-contracts.md) | adding a workload class or snapshot provider, or consuming admission state from a background job |
| [background-controller-contract.md](background-controller-contract.md) | adding a status snapshot or reconcile surface for a background service, or being tempted to fold several services into a generic controller | | [background-controller-contract.md](background-controller-contract.md) | adding a status snapshot or reconcile surface for a background service, or being tempted to fold several services into a generic controller |
| [background-services-inventory.md](background-services-inventory.md) | you need one audited background service's desired source, current-status inputs, status surface, and declared side effects | | [background-services-inventory.md](background-services-inventory.md) | you need one audited background service's desired source, current-status inputs, status surface, and declared side effects |
| [scanner-usage-publication.md](scanner-usage-publication.md) | changing scanner data-usage cache publication, quota-visible usage snapshots, scanner cycle recovery, or the persisted scanner usage artifacts |
| [scanner-usage-authority-decision.md](scanner-usage-authority-decision.md) | deciding whether quota admission depends on scanner data usage, removing scanner publication layers, or designing a scanner storage boundary |
| [config-model-boundary-adr.md](config-model-boundary-adr.md) | touching the server-config model (`Config`, `KV`, `KVS`) or its persistence, or asking which crate owns which part of server configuration | | [config-model-boundary-adr.md](config-model-boundary-adr.md) | touching the server-config model (`Config`, `KV`, `KVS`) or its persistence, or asking which crate owns which part of server configuration |
| [admin-route-action-snapshot.md](admin-route-action-snapshot.md) | adding, moving, or re-authorizing an admin route and needing to know where the route → handler → `AdminAction` contract is enforced | | [admin-route-action-snapshot.md](admin-route-action-snapshot.md) | adding, moving, or re-authorizing an admin route and needing to know where the route → handler → `AdminAction` contract is enforced |
| [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) | changing the bulk envelope re-wrap sweep, its admin endpoints, the re-wrap primitive, or which objects a rekey may touch | | [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) | changing the bulk envelope re-wrap sweep, its admin endpoints, the re-wrap primitive, or which objects a rekey may touch |
+14
View File
@@ -49,6 +49,20 @@ Outer crates reach ECStore only through `rustfs_ecstore::api`, and only from one
- RustFS startup internals are crate-private: only `startup_entrypoint` is a public startup module of the `rustfs` library (`rustfs/src/lib.rs`), and items inside the other `startup_*` modules use crate visibility. - RustFS startup internals are crate-private: only `startup_entrypoint` is a public startup module of the `rustfs` library (`rustfs/src/lib.rs`), and items inside the other `startup_*` modules use crate visibility.
- The observability dependency baseline is [obs-ecstore-dependency-inventory.md](obs-ecstore-dependency-inventory.md); observability extraction updates it together with the guard. - The observability dependency baseline is [obs-ecstore-dependency-inventory.md](obs-ecstore-dependency-inventory.md); observability extraction updates it together with the guard.
## Scanner, Heal, And ECStore
Heal is split by responsibility, not by the shared word "heal". ECStore owns erasure-set repair primitives: quorum metadata arbitration, EC reconstruction, per-disk rename commit, dangling metadata classification, and orphan data-dir reclamation. These stay in ECStore because they share the same object namespace locks, rename commit model, and data-dir cleanup rules as PUT, DELETE, multipart, lifecycle expiry, rebalance, and decommission. Moving those primitives out would split the lock and commit model across crates.
`crates/heal` owns repair orchestration: queueing, deduplication, admission, scheduling, resume, MRF replay, replacement-disk tracking, and the admin-facing status/control surface. It reaches storage through `HealStorageAPI`; ECStore-originated repair requests flow back through typed repair channels rather than a Cargo dependency on the heal crate.
`crates/scanner` owns discovery, data-usage publication, lifecycle/replication scan actions, bitrot scan dispatch, and scanner-driven repair requests. Scanner may request repair through the heal channel, but it must not directly execute erasure-set repair primitives.
`rustfs-scanner-metrics` owns scanner telemetry DTOs, global scanner counters, lifecycle action labels consumed by metrics, and the short-window latency accumulator used by those metrics. ECStore, lifecycle, observability, admin, and scanner code may depend on this crate for metrics only. `rustfs-scanner-contracts` must not regain metrics, globals, or telemetry implementation; it is reserved for scanner storage or wire contract types.
`remote_scanner` remains scanner-owned for now because it carries the scanner cycle fence, replay protection, stream envelope, and per-bucket scan result protocol. A future scanner storage seam may either move remote disk scan execution behind an ECStore storage capability or move the whole remote scanner protocol with scanner; leaving the wire protocol split across both sides without a documented owner is not allowed.
The scanner usage authority decision is fixed in [scanner-usage-authority-decision.md](scanner-usage-authority-decision.md): scanner usage remains hard-quota authority. A future scanner storage seam must therefore model the concrete publication, cycle-lock, usage-floor, observed-snapshot, and recovery-marker capabilities described in [scanner-usage-publication.md](scanner-usage-publication.md), not a generic key-value abstraction.
## Loss-Prevention Coverage ## Loss-Prevention Coverage
The guard pins specific public re-export lines (its `require_source_line` entries) so contract surfaces cannot silently disappear during cleanup. The canonical lists are the guard script and the owning files, not this page: The guard pins specific public re-export lines (its `require_source_line` entries) so contract surfaces cannot silently disappear during cleanup. The canonical lists are the guard script and the owning files, not this page:
@@ -3,6 +3,8 @@
**Use this when:** changing heal, PUT/multipart commit, delete, lifecycle expiry, or data-movement code that touches the same `(bucket, object)` commit surface; or evaluating whether RustFS needs a persistent per-object healing marker like MinIO's `x-minio-healing`. **Use this when:** changing heal, PUT/multipart commit, delete, lifecycle expiry, or data-movement code that touches the same `(bucket, object)` commit surface; or evaluating whether RustFS needs a persistent per-object healing marker like MinIO's `x-minio-healing`.
**Source of truth:** `crates/ecstore/src/set_disk/ops/heal.rs` (`heal_object_with_explicit_version_regen`, `HealObjectLockKind`, `HEAL_RENAME_INCOMPLETE`), `crates/ecstore/src/set_disk/ops/object.rs` (PUT/DELETE lock sections, `reconcile_old_data_cleanup_receipts`), `crates/ecstore/src/set_disk/core/io_primitives.rs` (`commit_rename_data_dir`, `report_old_data_dir_cleanup`, `reclaim_orphan_data_dirs`), `crates/filemeta/src/fileinfo.rs` (`FileInfo::set_healing`), `crates/heal/src/heal/manager/queue.rs` (dedup keys). **Source of truth:** `crates/ecstore/src/set_disk/ops/heal.rs` (`heal_object_with_explicit_version_regen`, `HealObjectLockKind`, `HEAL_RENAME_INCOMPLETE`), `crates/ecstore/src/set_disk/ops/object.rs` (PUT/DELETE lock sections, `reconcile_old_data_cleanup_receipts`), `crates/ecstore/src/set_disk/core/io_primitives.rs` (`commit_rename_data_dir`, `report_old_data_dir_cleanup`, `reclaim_orphan_data_dirs`), `crates/filemeta/src/fileinfo.rs` (`FileInfo::set_healing`), `crates/heal/src/heal/manager/queue.rs` (dedup keys).
For crate ownership, read [crate-boundaries.md](crate-boundaries.md): ECStore owns erasure-set repair primitives that share this lock and commit model, while `crates/heal` owns repair orchestration.
## Model ## Model
Heal and every foreground or background write path serialize on the same object-level namespace write lock (a quorum lock RPC in distributed mode, the in-process lock manager on a single node; granularity is the object, the version component is always `None`), and heal holds its guard across the whole rename commit. MinIO's `x-minio-healing` marker is an out-of-lock defence against version-cleanup logic inside `RenameData` interleaving with a heal commit; RustFS's commit model has no such interleaving, so no persistent marker exists (`x-minio-healing` does not occur in `crates/` or `rustfs/`) and none is needed. Three layers replace it: Heal and every foreground or background write path serialize on the same object-level namespace write lock (a quorum lock RPC in distributed mode, the in-process lock manager on a single node; granularity is the object, the version component is always `None`), and heal holds its guard across the whole rename commit. MinIO's `x-minio-healing` marker is an out-of-lock defence against version-cleanup logic inside `RenameData` interleaving with a heal commit; RustFS's commit model has no such interleaving, so no persistent marker exists (`x-minio-healing` does not occur in `crates/` or `rustfs/`) and none is needed. Three layers replace it:
@@ -0,0 +1,24 @@
# Scanner Usage Authority Decision
**Use this when:** deciding whether quota admission depends on scanner data usage, removing scanner publication layers, or designing a scanner storage boundary.
**Source of truth:** [scanner-usage-publication.md](scanner-usage-publication.md), `crates/ecstore/src/bucket/quota/checker.rs`, and the scanner publication state under `crates/scanner/src/scanner/`.
## Decision
Date: 2026-09-03
RustFS keeps scanner data usage as authoritative for quota admission.
This selects option A from the backlog decision record: the scanner publication protocol remains necessary while quota admission consumes scanner usage. The cycle epoch, publication CAS, data-movement fence, tier-registry fence, observed snapshot layer, and persisted usage floor are retained and documented as protocol invariants rather than treated as removable compatibility clutter.
## Rationale
Quota is a write-path admission decision, so serving quota from best-effort scanner data would turn temporary scanner lag into under-enforcement. The current design therefore needs an availability story for authoritative usage instead of deleting the proof layers that make it authoritative.
The scanner usage floor provides that availability story. It is a lower bound used when a complete authoritative snapshot is not available, including cold startup, upgrade recovery, and incomplete-cycle repair. Observed snapshots remain useful for admin and observability, but they do not become quota authority.
## Consequences
#2214 is the hard design input for future usage-publication changes. A future proposal may still choose soft quota and MinIO-style best-effort usage, but that would be a product change with its own staged compatibility plan for persisted artifacts.
#2219 may design the scanner storage boundary against the current authoritative protocol. The interface must include the CAS key-value, cycle lock, usage-floor, observed-snapshot, and recovery-marker capabilities needed by [scanner-usage-publication.md](scanner-usage-publication.md); it must not hide those proof obligations behind a generic object-store trait.
@@ -0,0 +1,129 @@
# Scanner Usage Publication Contract
**Use this when:** changing scanner data-usage persistence, quota-visible usage snapshots, scanner cycle state, dirty-usage catch-up, or the conditions under which an observed scanner snapshot may be served.
**Source of truth:** `crates/scanner/src/scanner/usage_store.rs`, `crates/scanner/src/scanner/cycle_state.rs`, `crates/scanner/src/scanner/backlog.rs`, `crates/scanner/src/scanner/leadership.rs`, `crates/scanner/src/data_usage_define.rs`, and quota fallback behavior in `crates/ecstore/src/bucket/quota/checker.rs`.
## Ownership Model
One scanner cycle owns an authoritative publication only after it holds the
cluster scanner leadership claim and proves that the storage publication epoch
has not moved. The leadership claim is persisted in the scanner cycle state,
while data-usage publication admission is owned by ECStore because it knows
whether rebalance, decommission, or another data-movement operation has changed
the generation that scanner results are allowed to describe.
The scanner may compute usage without publication ownership, but it must not
turn that result into authoritative quota-visible state. A complete publication
therefore has three identities:
- the scanner leader epoch that owns the cycle;
- the storage publication epoch that fences data movement;
- the per-object CAS revision on the usage object being replaced.
If any identity changes before commit, the result is a candidate for retry or
observation, not an authoritative baseline.
## Fences
The protocol uses separate fences because they exclude different stale inputs.
They must not be collapsed unless the replacement proves the same exclusions.
| Fence | Owner | Excludes |
|---|---|---|
| Scanner leadership claim | scanner | competing scanner leaders and stale cycle writers |
| Storage publication epoch | ECStore | usage computed across rebalance, decommission, or other data-movement generations |
| Publication lease | scanner peers through ECStore-facing activity probes | remote dirty-usage or maintenance state that has not acknowledged the candidate |
| CAS revision | backing config object store | lost updates to `.usage.v2.json`, `.usage.json`, or cycle-state objects |
| Per-set freshness | scanner aggregation | a merged usage snapshot that combines stale and current set results |
| Tier registry generation | scanner tier accounting | bytes classified against a different warm-tier registry |
| Usage floor identity | scanner publication and ECStore quota fallback | empty or legacy values becoming plausible authoritative quota input |
A reader that cannot prove the required fence for its surface must fail closed
or use the documented observed path below. It must not synthesize an empty usage
snapshot for a missing or corrupt authoritative object.
## Persisted Objects
The persisted objects are part of the compatibility contract. Removing one
requires a compatibility window and a dedicated cleanup entry.
| Object | Owner | Lifecycle |
|---|---|---|
| `.usage-cache.bin` under each bucket and set | scanner disk walk | Rebuilt by scanner from object metadata. Missing data causes a rescan for that bucket/set; corrupt data is not a complete baseline. |
| `.bloomcycle.bin` | scanner cycle state | CAS-updated by the leader. Missing state starts from an uninitialized cycle; corrupt or future state is quarantined before automatic retry. |
| `.usage.v2.json` and `.usage.json` | scanner authoritative publication | `.usage.v2.json` is the primary complete usage snapshot. `.usage.json` is read only as a legacy or companion baseline when it carries a valid persisted identity. Neither bypasses the v2 epoch fence, and readers may treat a snapshot as authoritative only when its baseline identity and completion fields validate. |
| `.usage.observed.json` | scanner observation path | Written when an authoritative publication cannot be proven but a diagnostic snapshot is still useful. It is never a hard-quota authority. |
| `bucket-metadata/.usage.json` | scanner usage floor | Carries the persisted per-bucket floor used by quota during a degraded authoritative-usage window. It is static until the next complete scanner publication. |
| `.bloomcycle.bin.recovery-required.json` | scanner cycle recovery | Quarantines invalid cycle state with retry evidence. Only scanner recovery code updates or clears it. |
| `.scanner-cycle.lock` | scanner runtime lock | Serializes cycle-level work. A missing lock object is not itself usage evidence. |
| `.scanner-pause-backlog.json` | scanner pause and catch-up ledger | Tracks dirty usage, discovered lifecycle work, and full-scan catch-up while authoritative publication is fenced by data movement. It never grants publication admission. |
## Observed Snapshots
Observed snapshots are a diagnostic and availability layer. They may be served
only when the snapshot explicitly reports that it is partial or observational,
and only to consumers that do not make hard quota, durability, or deletion
decisions from it. Admin usage views may expose this state with completeness
flags so operators can see progress while the authoritative publication is
blocked. Quota enforcement must not use an observed snapshot as the current
usage authority.
When an authoritative snapshot is unavailable, quota admission may use the
persisted usage floor. That is an availability fallback, not a fresh count: live
writes do not advance the floor, and overrun is bounded only by writes accepted
before the next complete scanner publication. If no valid persisted floor is
available, quota remains unavailable and fails closed.
## Availability Decision
Decision date: 2026-09-03.
RustFS keeps scanner usage as the authority for hard quota admission. The
publication protocol therefore remains necessary: leadership, storage epoch,
lease, CAS, observed snapshot, and usage-floor layers are the proof machinery
that lets a distributed background scan feed a quota decision without accepting
stale or cross-generation usage as current truth.
The availability contract is:
- the authoritative fast path reads complete in-memory or persisted scanner
usage;
- during upgrade or publication outage, quota may admit against the persisted
per-bucket usage floor;
- the floor is advisory for the outage window and must converge back to a
complete scanner publication;
- a bucket with neither authoritative usage nor a valid floor fails closed.
Changing this decision to a soft-quota model would be a product change, not a
scanner refactor. It would need a staged removal of the authority-specific
layers and compatibility handling for the persisted objects above.
## Deletion And Recovery Rules
Only the owner of an object may delete or quarantine it:
- scanner may rebuild per-set `.usage-cache.bin` after a scan proves the
replacement contents;
- scanner cycle recovery may quarantine invalid `.bloomcycle.bin` and clear the
marker only after a valid cycle state is persisted;
- scanner publication may replace `.usage.v2.json` or legacy companions only
through the publication fences above;
- quota consumers may read the usage floor but must not delete or repair it;
- operators may reset scanner usage state only through the supported scanner
reset surface, which records the reset paths and forces a full rebuild.
Missing, undecodable, or identity-less data is not converted to zero. It is
reported as uninitialized, recovery-required, observed-only, or unavailable
according to the reader's surface.
## Existing Fixes As Invariants
Several prior scanner fixes are consequences of this contract rather than
standalone patches:
- incomplete scanner usage must not become a complete admin or quota baseline,
because completeness and floor identity are part of publication ownership;
- dirty usage and maintenance acknowledgements must fence publication, because
a remote node with unacknowledged work can invalidate the candidate;
- a legacy or backup usage object may help recover availability only when it
carries a valid baseline identity and does not cross the primary epoch fence.
@@ -12,9 +12,9 @@
| Class | Provider (`impl WorkloadAdmissionSnapshotProvider`) | `active` / `queued` / `limit` source | Reports `Unknown` when | | Class | Provider (`impl WorkloadAdmissionSnapshotProvider`) | `active` / `queued` / `limit` source | Reports `Unknown` when |
|---|---|---|---| |---|---|---|---|
| `ForegroundRead` | `ConcurrencyManager` in `rustfs/src/storage/concurrency/manager.rs` (source of truth); re-exposed unchanged by the RustFS runtime provider | disk-read permits in use / `None` (the semaphore exposes no waiter count) / configured max concurrent disk reads | the storage registry has no entry | | `ForegroundRead` | `ConcurrencyManager` in `rustfs/src/storage/concurrency/manager.rs` (source of truth); re-exposed unchanged by the RustFS runtime provider | disk-read permits in use / `None` (the semaphore exposes no waiter count) / configured max concurrent disk reads | the storage registry has no entry |
| `ForegroundWrite` | none | none | always: no write-specific admission owner exposes a read-only surface yet | | `ForegroundWrite` | `ConcurrencyManager` in `rustfs/src/storage/concurrency/manager.rs` (source of truth); re-exposed unchanged by the RustFS runtime provider | foreground-write permits in use or legacy active-write counter / `None` / configured or derived write-admission limit | the storage registry has no entry |
| `Metadata` | `RustFsWorkloadAdmissionSnapshotProvider` in `rustfs/src/workload_admission.rs` | `Open` once the bucket metadata runtime handle exists; no counts | bucket metadata runtime not initialized | | `Metadata` | `RustFsWorkloadAdmissionSnapshotProvider` in `rustfs/src/workload_admission.rs` | `Open` once the bucket metadata runtime handle exists; no counts | bucket metadata runtime not initialized |
| `Scanner` | same | scanner active work-unit counter / none / none | the counter is zero (idle and uninitialized are indistinguishable) | | `Scanner` | same | scanner active work-unit counter / none / configured set-scan limit when nonzero | scanner runtime not initialized |
| `Repair` | same | heal active tasks / heal queue length / `None` (limits live behind the async heal manager state) | heal manager not initialized | | `Repair` | same | heal active tasks / heal queue length / `None` (limits live behind the async heal manager state) | heal manager not initialized |
| `Replication` | same | active regular + large-object + MRF workers / site replication queue count / `None` (limits owned by the async pool and resize policy) | replication runtime not initialized, or queue stats currently locked | | `Replication` | same | active regular + large-object + MRF workers / site replication queue count / `None` (limits owned by the async pool and resize policy) | replication runtime not initialized, or queue stats currently locked |
@@ -28,6 +28,7 @@ Consumers that read the snapshot to self-throttle exist, and they do not change
|---|---|---| |---|---|---|
| Data-movement backpressure (decommission, rebalance) | `crates/ecstore/src/data_movement/backpressure.rs` (`wait_for_data_movement_admission`, `foreground_pressure`) | Delays the next data-movement step while `ForegroundRead` or `ForegroundWrite` usage exceeds the configured high-water percent. ECStore receives the provider through `set_workload_admission_snapshot_provider` (`crates/ecstore/src/lib.rs`), published from `rustfs/src/startup_background.rs`; with no provider the step is admitted immediately. | | Data-movement backpressure (decommission, rebalance) | `crates/ecstore/src/data_movement/backpressure.rs` (`wait_for_data_movement_admission`, `foreground_pressure`) | Delays the next data-movement step while `ForegroundRead` or `ForegroundWrite` usage exceeds the configured high-water percent. ECStore receives the provider through `set_workload_admission_snapshot_provider` (`crates/ecstore/src/lib.rs`), published from `rustfs/src/startup_background.rs`; with no provider the step is admitted immediately. |
| Heal manager mainline throttle | `crates/heal/src/heal/manager.rs` (`new_with_workload_provider`) | When `mainline_throttle_enable` is set, defers heal work while `ForegroundRead` or `ForegroundWrite` utilization exceeds the configured high-water percents; with no provider or the throttle disabled, heal pacing is unchanged. | | Heal manager mainline throttle | `crates/heal/src/heal/manager.rs` (`new_with_workload_provider`) | When `mainline_throttle_enable` is set, defers heal work while `ForegroundRead` or `ForegroundWrite` utilization exceeds the configured high-water percents; with no provider or the throttle disabled, heal pacing is unchanged. |
| Scanner sleeper and scan fan-out | `crates/scanner/src/workload_admission.rs`, `crates/scanner/src/sleeper.rs`, and `crates/scanner/src/scanner_io/guards.rs` | Reads the same provider published from `rustfs/src/startup_background.rs` and combines it with scanner-local foreground read guards. Foreground activity increases scanner sleeps and reduces set/disk scan fan-out to one; cycle budgets still own object, directory, and duration limits. With no provider, scanner keeps the legacy local foreground-read behavior. |
## Boundary Rules ## Boundary Rules
+8
View File
@@ -57,6 +57,14 @@ Promotion rule: never promote a report-only lane to required from one green run.
e2e filters live in `.config/nextest.toml`; extend a profile instead of adding a second selector. Before a profile runs, `scripts/check_test_wiring.py` compares its listing to the committed digest in `.config/e2e-<profile>-selection.txt`, so a silent test drop fails closed. e2e filters live in `.config/nextest.toml`; extend a profile instead of adding a second selector. Before a profile runs, `scripts/check_test_wiring.py` compares its listing to the committed digest in `.config/e2e-<profile>-selection.txt`, so a silent test drop fails closed.
Scanner usage and heal rebuild coverage are intentionally split by risk and
cost. `data_usage_test` runs in the PR `e2e-smoke` lane so changes that affect
authoritative scanner usage publication, quota-visible usage, or admin usage
snapshots get an end-to-end signal before merge review. `heal_erasure_disk_rebuild_test`
runs in `e2e-full` so core erasure heal rebuild regressions are caught no later
than the merge queue or `main` push lane; it also remains in `e2e-nightly` with
the serialized cluster fault-domain suites for scheduled soak signal.
## Scheduled validation ## Scheduled validation
Scheduled lanes never block a PR. Their workflow-local gate fails the run, scheduled failures route to the shared failure-issue action, and `scheduled-validation-freshness.yml` fails when a workflow listed in `.github/scheduled-validations.json` has not run within its `max_age_hours` (a `never_ran_grace_until` entry covers the window before a newly enabled cron's first slot). Cadence is qualitative here; the cron lives in each workflow's `on.schedule`. Scheduled lanes never block a PR. Their workflow-local gate fails the run, scheduled failures route to the shared failure-issue action, and `scheduled-validation-freshness.yml` fails when a workflow listed in `.github/scheduled-validations.json` has not run within its `max_age_hours` (a `never_ran_grace_until` entry covers the window before a newly enabled cron's first slot). Cadence is qualitative here; the cron lives in each workflow's `on.schedule`.
+1 -1
View File
@@ -227,7 +227,7 @@ rustfs-heal = { workspace = true }
rustfs-audit = { workspace = true } rustfs-audit = { workspace = true }
rustfs-common = { workspace = true } rustfs-common = { workspace = true }
rustfs-heal-contracts = { workspace = true } rustfs-heal-contracts = { workspace = true }
rustfs-scanner-contracts = { workspace = true } rustfs-scanner-metrics = { workspace = true }
rustfs-config = { workspace = true, features = ["notify", "server-config-model"] } rustfs-config = { workspace = true, features = ["notify", "server-config-model"] }
rustfs-crypto = { workspace = true } rustfs-crypto = { workspace = true }
rustfs-credentials = { workspace = true } rustfs-credentials = { workspace = true }
+2 -4
View File
@@ -29,10 +29,8 @@ use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
use rustfs_credentials::Credentials; use rustfs_credentials::Credentials;
use rustfs_policy::policy::action::{Action, AdminAction}; use rustfs_policy::policy::action::{Action, AdminAction};
#[cfg(test)] #[cfg(test)]
use rustfs_scanner_contracts::metrics::ScannerLifecycleTransitionSnapshot; use rustfs_scanner_metrics::metrics::ScannerLifecycleTransitionSnapshot;
use rustfs_scanner_contracts::metrics::{ use rustfs_scanner_metrics::metrics::{ScannerLifecycleExpirySnapshot, ScannerMaintenanceControlSnapshot, ScannerMetricsReport};
ScannerLifecycleExpirySnapshot, ScannerMaintenanceControlSnapshot, ScannerMetricsReport,
};
use s3s::header::CONTENT_TYPE; use s3s::header::CONTENT_TYPE;
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
+1 -1
View File
@@ -125,7 +125,7 @@ pub(crate) mod runtime_sources {
pub(crate) type DailyAllTierStats = super::DailyAllTierStats; pub(crate) type DailyAllTierStats = super::DailyAllTierStats;
pub(crate) type ECStore = super::ECStore; pub(crate) type ECStore = super::ECStore;
pub(crate) type NotificationSys = super::NotificationSys; pub(crate) type NotificationSys = super::NotificationSys;
pub(crate) type ScannerMetricsReport = rustfs_scanner_contracts::metrics::ScannerMetricsReport; pub(crate) type ScannerMetricsReport = rustfs_scanner_metrics::metrics::ScannerMetricsReport;
pub(crate) type StorageClassConfig = crate::storage::storage_api::ecstore_config::storageclass::Config; pub(crate) type StorageClassConfig = crate::storage::storage_api::ecstore_config::storageclass::Config;
pub(crate) type TierConfigMgr = crate::storage::storage_api::TierConfigMgr; pub(crate) type TierConfigMgr = crate::storage::storage_api::TierConfigMgr;
} }
+2 -2
View File
@@ -148,7 +148,7 @@ pub(crate) mod runtime {
pub(crate) type NotificationSys = crate::storage::storage_api::NotificationSys; pub(crate) type NotificationSys = crate::storage::storage_api::NotificationSys;
pub(crate) type ObjectStoreResolver = crate::storage::storage_api::ObjectStoreResolver; pub(crate) type ObjectStoreResolver = crate::storage::storage_api::ObjectStoreResolver;
pub(crate) type ReplicationStats = crate::storage::storage_api::ReplicationStats; pub(crate) type ReplicationStats = crate::storage::storage_api::ReplicationStats;
pub(crate) type ScannerMetricsReport = rustfs_scanner_contracts::metrics::ScannerMetricsReport; pub(crate) type ScannerMetricsReport = rustfs_scanner_metrics::metrics::ScannerMetricsReport;
pub(crate) type StorageClassConfig = crate::storage::storage_api::ecstore_config::storageclass::Config; pub(crate) type StorageClassConfig = crate::storage::storage_api::ecstore_config::storageclass::Config;
pub(crate) type TierConfigMgr = crate::storage::storage_api::TierConfigMgr; pub(crate) type TierConfigMgr = crate::storage::storage_api::TierConfigMgr;
pub(crate) type TransitionState = crate::storage::storage_api::TransitionState; pub(crate) type TransitionState = crate::storage::storage_api::TransitionState;
@@ -224,7 +224,7 @@ pub(crate) mod runtime {
} }
pub(crate) async fn collect_scanner_metrics_report() -> ScannerMetricsReport { pub(crate) async fn collect_scanner_metrics_report() -> ScannerMetricsReport {
rustfs_scanner_contracts::metrics::global_metrics().report().await rustfs_scanner_metrics::metrics::global_metrics().report().await
} }
#[cfg(test)] #[cfg(test)]
+1 -1
View File
@@ -25,7 +25,7 @@ use crate::storage_api::cluster::control_plane::{
use crate::workload_admission::workload_admission_registry_snapshot; use crate::workload_admission::workload_admission_registry_snapshot;
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot}; use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot};
use rustfs_io_metrics::internode_metrics::{InternodeMetricsSnapshot, global_internode_metrics}; use rustfs_io_metrics::internode_metrics::{InternodeMetricsSnapshot, global_internode_metrics};
use rustfs_scanner_contracts::metrics::{ScannerMetricsReport, global_metrics}; use rustfs_scanner_metrics::metrics::{ScannerMetricsReport, global_metrics};
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub struct ClusterReadOnlySnapshot { pub struct ClusterReadOnlySnapshot {
+1
View File
@@ -59,6 +59,7 @@ pub(crate) async fn init_background_service_runtime(store: Arc<ECStore>) -> Resu
let workload_provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync> = let workload_provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync> =
Arc::new(RustFsWorkloadAdmissionSnapshotProvider); Arc::new(RustFsWorkloadAdmissionSnapshotProvider);
let _ = set_workload_admission_snapshot_provider(workload_provider.clone()); let _ = set_workload_admission_snapshot_provider(workload_provider.clone());
rustfs_scanner::set_scanner_workload_admission_snapshot_provider(workload_provider.clone());
if enable_heal || enable_scanner { if enable_heal || enable_scanner {
let heal_storage = Arc::new(ECStoreHealStorage::new(store.clone())); let heal_storage = Arc::new(ECStoreHealStorage::new(store.clone()));
+1 -1
View File
@@ -55,7 +55,7 @@ pub(crate) async fn publish_server_addr(addr: &str) {
} }
pub(crate) async fn publish_init_time_now() { pub(crate) async fn publish_init_time_now() {
rustfs_scanner_contracts::set_global_init_time_now().await; rustfs_scanner_metrics::set_global_init_time_now().await;
} }
pub(crate) fn init_kms_service_manager() -> Arc<KmsServiceManager> { pub(crate) fn init_kms_service_manager() -> Arc<KmsServiceManager> {