mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-07 12:35:54 +00:00
feat(scanner): coordinate usage and workload boundaries (#7093)
* test(scanner): wire usage and heal rebuild gates * docs(scanner): define usage authority protocol * docs(heal): clarify scanner and ecstore boundaries * refactor(scanner): split metrics from contracts * feat(scanner): use shared workload snapshots * fix(ecstore): recheck capacity before decommission drain
This commit is contained in:
@@ -1,2 +1,2 @@
|
|||||||
sha256-darwin=21f2a5484bb10bed1f11bd6397fe80f20d60a870e4ed5016a1f3acbdb4e6f735
|
sha256-darwin=bb8c16cd63a94ff5e5e400d891ccc980710529dd5cb13a5a0c955c6c313bccd9
|
||||||
sha256-linux=29fd001e70d0114b59d65dd2a516c1f96a5fea1d6bb8fb315550a27f94d7b39b
|
sha256-linux=e9a8d64e73f627c4d26c236dbbba690c9ee03a9e26d42a4244515b4439365535
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
sha256=d90fa941cb737fe6fcf39ca13d45440183e77a7e047880c025af7e6bfe1d0517
|
sha256=a2542dc86bbff56b2177efc621785c56fa7e8d813b209b7d935e1e41a9f0ad15
|
||||||
|
|||||||
+18
-7
@@ -407,10 +407,15 @@ test-group = 'ecstore-serial-flaky'
|
|||||||
# harness_self}_test` stays in e2e-full and the fault / concurrency /
|
# harness_self}_test` stays in e2e-full and the fault / concurrency /
|
||||||
# real-source modules stay in e2e-nightly; this is an allowlist, not a module
|
# real-source modules stay in e2e-nightly; this is an allowlist, not a module
|
||||||
# clause, so a new ODM test never lands here silently.
|
# clause, so a new ODM test never lands here silently.
|
||||||
|
#
|
||||||
|
# Scanner authoritative usage publication (backlog#2213): data_usage_test is
|
||||||
|
# the PR-lane e2e coverage for scanner usage snapshots consumed by quota and
|
||||||
|
# admin surfaces. It uses the same single-node, random-port, isolated-temp-dir
|
||||||
|
# fixture as the existing smoke modules.
|
||||||
[profile.e2e-smoke]
|
[profile.e2e-smoke]
|
||||||
default-filter = """
|
default-filter = """
|
||||||
package(e2e_test) & (
|
package(e2e_test) & (
|
||||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat)_test::|^fake_s3_target::/)
|
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||||
| test(/^reliant::lifecycle::/)
|
| test(/^reliant::lifecycle::/)
|
||||||
| test(/^reliant::tiering::/)
|
| test(/^reliant::tiering::/)
|
||||||
@@ -488,12 +493,16 @@ path = "junit.xml"
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# e2e-nightly profile — destructive multi-process cluster fault domains
|
# e2e-nightly profile — destructive multi-process cluster fault domains
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# These seven modules are deliberately outside e2e-full's merge budget. Each
|
# These eight modules are deliberately outside e2e-full's merge budget. Each
|
||||||
# starts a real multi-process or multi-disk topology and exercises node/disk
|
# starts a real multi-process or multi-disk topology and exercises node/disk
|
||||||
# loss, quorum, cleanup, notification fan-in, or admin-timeout behavior. The
|
# loss, quorum, cleanup, notification fan-in, or admin-timeout behavior. The
|
||||||
# consolidated nightly workflow runs them serially to avoid resource
|
# consolidated nightly workflow runs them serially to avoid resource
|
||||||
# starvation; failures are never retried.
|
# starvation; failures are never retried.
|
||||||
#
|
#
|
||||||
|
# heal_erasure_disk_rebuild_test also runs in e2e-full so core heal rebuild
|
||||||
|
# regressions are caught no later than the merge/main lane. It remains here for
|
||||||
|
# nightly serial coverage with the other cluster fault domains.
|
||||||
|
#
|
||||||
# On-demand migration (backlog#2158 ODM-11) joins by the second clause: the
|
# On-demand migration (backlog#2158 ODM-11) joins by the second clause: the
|
||||||
# fault matrix waits out the 30 s circuit-breaker window, the concurrency
|
# fault matrix waits out the 30 s circuit-breaker window, the concurrency
|
||||||
# matrix drives 100-deep bursts, and the real-source cases start a second
|
# matrix drives 100-deep bursts, and the real-source cases start a second
|
||||||
@@ -542,11 +551,13 @@ path = "junit.xml"
|
|||||||
# The filter is "the whole e2e_test crate MINUS the sets owned by other lanes":
|
# The filter is "the whole e2e_test crate MINUS the sets owned by other lanes":
|
||||||
# * protocols:: — FTPS/SFTP/WebDAV, run from the dedicated protocol profile
|
# * protocols:: — FTPS/SFTP/WebDAV, run from the dedicated protocol profile
|
||||||
# with one worker because the suite owns fixed ports.
|
# with one worker because the suite owns fixed ports.
|
||||||
# * the 7 cluster suites that spin up a RustFSTestClusterEnvironment
|
# * cluster suites that spin up a RustFSTestClusterEnvironment
|
||||||
# (cluster_concurrency, cluster_multidrive_pool, stale_multipart_cleanup_cluster,
|
# (cluster_concurrency, cluster_multidrive_pool, stale_multipart_cleanup_cluster,
|
||||||
# namespace_lock_quorum, heal_erasure_disk_rebuild, admin_timeout_regression,
|
# namespace_lock_quorum, admin_timeout_regression, object_lambda) — too
|
||||||
# object_lambda) — too heavy for the merge budget; they run in the
|
# heavy for the merge budget; they run in the e2e-nightly serial
|
||||||
# e2e-nightly serial cluster-fault lane.
|
# cluster-fault lane. heal_erasure_disk_rebuild is intentionally not
|
||||||
|
# excluded here because backlog#2213 promotes core heal rebuild coverage to
|
||||||
|
# this merge/main lane while retaining nightly coverage.
|
||||||
# * replication_extension_test — repl-1 already splits it into the PR
|
# * replication_extension_test — repl-1 already splits it into the PR
|
||||||
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (56 slow) lanes and reserves
|
# `e2e-smoke` (20 fast) and `e2e-repl-nightly` (56 slow) lanes and reserves
|
||||||
# it for those, so e2e-full does not double-run it.
|
# it for those, so e2e-full does not double-run it.
|
||||||
@@ -562,7 +573,7 @@ path = "junit.xml"
|
|||||||
default-filter = """
|
default-filter = """
|
||||||
package(e2e_test)
|
package(e2e_test)
|
||||||
& !test(/^protocols::/)
|
& !test(/^protocols::/)
|
||||||
& !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
|
& !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
|
||||||
& !test(/^replication_extension_test::/)
|
& !test(/^replication_extension_test::/)
|
||||||
& !test(/^replication_target_matrix_test::/)
|
& !test(/^replication_target_matrix_test::/)
|
||||||
& !test(/^on_demand_migration::(concurrency_test|fault_test|real_source_test)::/)
|
& !test(/^on_demand_migration::(concurrency_test|fault_test|real_source_test)::/)
|
||||||
|
|||||||
+2
-2
@@ -92,7 +92,7 @@ refactors.
|
|||||||
|
|
||||||
| Domain | Current workspace crates | Responsibility |
|
| Domain | Current workspace crates | Responsibility |
|
||||||
|--------|--------------------------|----------------|
|
|--------|--------------------------|----------------|
|
||||||
| Foundation | `checksums`, `common`, `config`, `data-usage`, `heal-contracts`, `scanner-contracts`, `utils` | Shared configuration, data-usage models, heal/scanner domain contracts, utilities, and checksums. |
|
| Foundation | `checksums`, `common`, `config`, `data-usage`, `heal-contracts`, `scanner-contracts`, `scanner-metrics`, `utils` | Shared configuration, data-usage models, heal/scanner domain contracts and telemetry types, utilities, and checksums. |
|
||||||
| I/O and storage | `concurrency`, `ecstore`, `filemeta`, `heal`, `io-core`, `io-metrics`, `lifecycle`, `lock`, `object-capacity`, `object-data-cache`, `replication`, `rio`, `rio-v2`, `s3-client`, `scanner`, `storage-api` | Erasure-coded object storage, metadata, recovery, lifecycle, replication, locking, cache, I/O pipelines, and the engine-side S3 client for remote tier/transition targets. |
|
| I/O and storage | `concurrency`, `ecstore`, `filemeta`, `heal`, `io-core`, `io-metrics`, `lifecycle`, `lock`, `object-capacity`, `object-data-cache`, `replication`, `rio`, `rio-v2`, `s3-client`, `scanner`, `storage-api` | Erasure-coded object storage, metadata, recovery, lifecycle, replication, locking, cache, I/O pipelines, and the engine-side S3 client for remote tier/transition targets. |
|
||||||
| Security and identity | `credentials`, `crypto`, `iam`, `keystone`, `kms`, `policy`, `security-governance`, `signer`, `tls-runtime`, `trusted-proxies` | Credentials, authentication, authorization, encryption, key management, TLS, and security contracts. |
|
| Security and identity | `credentials`, `crypto`, `iam`, `keystone`, `kms`, `policy`, `security-governance`, `signer`, `tls-runtime`, `trusted-proxies` | Credentials, authentication, authorization, encryption, key management, TLS, and security contracts. |
|
||||||
| Protocols and contracts | `extension-schema`, `madmin`, `protos`, `protocols`, `s3-ops`, `s3-types`, `s3select-api`, `s3select-query` | Admin, inter-node, S3, S3 Select, and optional protocol contracts. |
|
| Protocols and contracts | `extension-schema`, `madmin`, `protos`, `protocols`, `s3-ops`, `s3-types`, `s3select-api`, `s3select-query` | Admin, inter-node, S3, S3 Select, and optional protocol contracts. |
|
||||||
@@ -135,7 +135,7 @@ default build (lifecycle:
|
|||||||
`crates/ecstore/src/bucket/replication/replication_state.rs`) — a naming
|
`crates/ecstore/src/bucket/replication/replication_state.rs`) — a naming
|
||||||
collision, not copies; renaming is tracked in rustfs/backlog#1847.
|
collision, not copies; renaming is tracked in rustfs/backlog#1847.
|
||||||
- `LastMinuteLatency` has two deliberately different implementations: the
|
- `LastMinuteLatency` has two deliberately different implementations: the
|
||||||
per-second bucketed accumulator in `crates/scanner-contracts/src/last_minute.rs` and
|
per-second bucketed accumulator in `crates/scanner-metrics/src/last_minute.rs` and
|
||||||
the in-memory endpoint-health sample tracker in
|
the in-memory endpoint-health sample tracker in
|
||||||
`crates/ecstore/src/bucket/bucket_target_sys.rs` (its doc comment explains
|
`crates/ecstore/src/bucket/bucket_target_sys.rs` (its doc comment explains
|
||||||
why it stays local).
|
why it stays local).
|
||||||
|
|||||||
Generated
+10
-5
@@ -9517,7 +9517,7 @@ dependencies = [
|
|||||||
"rustfs-s3select-api",
|
"rustfs-s3select-api",
|
||||||
"rustfs-s3select-query",
|
"rustfs-s3select-query",
|
||||||
"rustfs-scanner",
|
"rustfs-scanner",
|
||||||
"rustfs-scanner-contracts",
|
"rustfs-scanner-metrics",
|
||||||
"rustfs-security-governance",
|
"rustfs-security-governance",
|
||||||
"rustfs-signer",
|
"rustfs-signer",
|
||||||
"rustfs-storage-api",
|
"rustfs-storage-api",
|
||||||
@@ -9763,7 +9763,7 @@ dependencies = [
|
|||||||
"rustfs-rio-v2",
|
"rustfs-rio-v2",
|
||||||
"rustfs-s3-client",
|
"rustfs-s3-client",
|
||||||
"rustfs-s3-types",
|
"rustfs-s3-types",
|
||||||
"rustfs-scanner-contracts",
|
"rustfs-scanner-metrics",
|
||||||
"rustfs-storage-api",
|
"rustfs-storage-api",
|
||||||
"rustfs-uring",
|
"rustfs-uring",
|
||||||
"rustfs-utils",
|
"rustfs-utils",
|
||||||
@@ -10119,7 +10119,7 @@ dependencies = [
|
|||||||
"proptest",
|
"proptest",
|
||||||
"rustfs-config",
|
"rustfs-config",
|
||||||
"rustfs-replication",
|
"rustfs-replication",
|
||||||
"rustfs-scanner-contracts",
|
"rustfs-scanner-metrics",
|
||||||
"rustfs-storage-api",
|
"rustfs-storage-api",
|
||||||
"s3s",
|
"s3s",
|
||||||
"serial_test",
|
"serial_test",
|
||||||
@@ -10317,7 +10317,7 @@ dependencies = [
|
|||||||
"rustfs-iam",
|
"rustfs-iam",
|
||||||
"rustfs-io-metrics",
|
"rustfs-io-metrics",
|
||||||
"rustfs-notify",
|
"rustfs-notify",
|
||||||
"rustfs-scanner-contracts",
|
"rustfs-scanner-metrics",
|
||||||
"rustfs-security-governance",
|
"rustfs-security-governance",
|
||||||
"rustfs-storage-api",
|
"rustfs-storage-api",
|
||||||
"rustfs-utils",
|
"rustfs-utils",
|
||||||
@@ -10674,6 +10674,7 @@ dependencies = [
|
|||||||
"rand 0.10.2",
|
"rand 0.10.2",
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
"rustfs-common",
|
"rustfs-common",
|
||||||
|
"rustfs-concurrency",
|
||||||
"rustfs-config",
|
"rustfs-config",
|
||||||
"rustfs-credentials",
|
"rustfs-credentials",
|
||||||
"rustfs-data-usage",
|
"rustfs-data-usage",
|
||||||
@@ -10682,7 +10683,7 @@ dependencies = [
|
|||||||
"rustfs-heal-contracts",
|
"rustfs-heal-contracts",
|
||||||
"rustfs-lock",
|
"rustfs-lock",
|
||||||
"rustfs-s3-types",
|
"rustfs-s3-types",
|
||||||
"rustfs-scanner-contracts",
|
"rustfs-scanner-metrics",
|
||||||
"rustfs-storage-api",
|
"rustfs-storage-api",
|
||||||
"rustfs-utils",
|
"rustfs-utils",
|
||||||
"s3s",
|
"s3s",
|
||||||
@@ -10704,6 +10705,10 @@ dependencies = [
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "rustfs-scanner-contracts"
|
name = "rustfs-scanner-contracts"
|
||||||
version = "1.0.0-rc.5"
|
version = "1.0.0-rc.5"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustfs-scanner-metrics"
|
||||||
|
version = "1.0.0-rc.5"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"chrono",
|
"chrono",
|
||||||
"jiff",
|
"jiff",
|
||||||
|
|||||||
+3
-1
@@ -52,7 +52,8 @@ members = [
|
|||||||
"crates/s3select-api", # S3 Select API interface
|
"crates/s3select-api", # S3 Select API interface
|
||||||
"crates/s3select-query", # S3 Select query engine
|
"crates/s3select-query", # S3 Select query engine
|
||||||
"crates/scanner", # Scanner for data integrity checks and health monitoring
|
"crates/scanner", # Scanner for data integrity checks and health monitoring
|
||||||
"crates/scanner-contracts", # Scanner metrics and cycle contracts
|
"crates/scanner-contracts", # Scanner storage and wire contracts
|
||||||
|
"crates/scanner-metrics", # Scanner metrics and cycle telemetry
|
||||||
"crates/security-governance", # Security governance contracts
|
"crates/security-governance", # Security governance contracts
|
||||||
"crates/extension-schema", # Extension schema contracts
|
"crates/extension-schema", # Extension schema contracts
|
||||||
"crates/signer", # client signer
|
"crates/signer", # client signer
|
||||||
@@ -94,6 +95,7 @@ rustfs = { path = "./rustfs", version = "1.0.0-rc.5" }
|
|||||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.5" }
|
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.5" }
|
||||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.5" }
|
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.5" }
|
||||||
rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.5" }
|
rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.5" }
|
||||||
|
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.0-rc.5" }
|
||||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.5" }
|
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.5" }
|
||||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.5" }
|
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.5" }
|
||||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.5" }
|
rustfs-common = { path = "crates/common", version = "1.0.0-rc.5" }
|
||||||
|
|||||||
@@ -130,7 +130,7 @@ rustfs-concurrency.workspace = true
|
|||||||
rustfs-credentials = { workspace = true }
|
rustfs-credentials = { workspace = true }
|
||||||
rustfs-common.workspace = true
|
rustfs-common.workspace = true
|
||||||
rustfs-heal-contracts.workspace = true
|
rustfs-heal-contracts.workspace = true
|
||||||
rustfs-scanner-contracts.workspace = true
|
rustfs-scanner-metrics.workspace = true
|
||||||
rustfs-policy.workspace = true
|
rustfs-policy.workspace = true
|
||||||
rustfs-protos.workspace = true
|
rustfs-protos.workspace = true
|
||||||
rustfs-replication.workspace = true
|
rustfs-replication.workspace = true
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ use crate::bucket::lifecycle::lifecycle;
|
|||||||
use crate::object_api::ObjectInfo;
|
use crate::object_api::ObjectInfo;
|
||||||
use crate::services::event_notification::{EventArgs, send_event};
|
use crate::services::event_notification::{EventArgs, send_event};
|
||||||
use rustfs_s3_types::EventName;
|
use rustfs_s3_types::EventName;
|
||||||
use rustfs_scanner_contracts::metrics::IlmAction;
|
use rustfs_scanner_metrics::metrics::IlmAction;
|
||||||
|
|
||||||
const LIFECYCLE_EXPIRY_USER_AGENT: &str = "Internal: [ILM-Expiry]";
|
const LIFECYCLE_EXPIRY_USER_AGENT: &str = "Internal: [ILM-Expiry]";
|
||||||
const LIFECYCLE_TRANSITION_USER_AGENT: &str = "Internal: [ILM-Transition]";
|
const LIFECYCLE_TRANSITION_USER_AGENT: &str = "Internal: [ILM-Transition]";
|
||||||
|
|||||||
@@ -84,7 +84,7 @@ use rustfs_data_usage::TierStats;
|
|||||||
use rustfs_filemeta::{
|
use rustfs_filemeta::{
|
||||||
FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info, is_restored_object_on_disk,
|
FileInfo, FileInfoOpts, NULL_VERSION_ID, RestoreStatusOps, TRANSITION_COMPLETE, get_file_info, is_restored_object_on_disk,
|
||||||
};
|
};
|
||||||
use rustfs_scanner_contracts::metrics::{
|
use rustfs_scanner_metrics::metrics::{
|
||||||
IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
|
IlmAction, Metrics, ScannerLifecycleExpiryStateUpdate, ScannerLifecycleTransitionStateUpdate, global_metrics,
|
||||||
};
|
};
|
||||||
use rustfs_utils::{
|
use rustfs_utils::{
|
||||||
@@ -5596,7 +5596,7 @@ mod tests {
|
|||||||
use rustfs_filemeta::{FileInfo, FileMeta};
|
use rustfs_filemeta::{FileInfo, FileMeta};
|
||||||
#[cfg(feature = "test-util")]
|
#[cfg(feature = "test-util")]
|
||||||
use rustfs_s3_client::transition_api::ReaderImpl;
|
use rustfs_s3_client::transition_api::ReaderImpl;
|
||||||
use rustfs_scanner_contracts::metrics::{IlmAction, global_metrics};
|
use rustfs_scanner_metrics::metrics::{IlmAction, global_metrics};
|
||||||
use s3s::dto::{
|
use s3s::dto::{
|
||||||
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry,
|
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule, MetadataEntry,
|
||||||
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, OutputLocation, RestoreRequest,
|
ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule, OutputLocation, RestoreRequest,
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use rustfs_scanner_contracts::metrics::IlmAction;
|
use rustfs_scanner_metrics::metrics::IlmAction;
|
||||||
|
|
||||||
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
|
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
|
||||||
use crate::bucket::replication::ReplicationLifecycleBridge;
|
use crate::bucket::replication::ReplicationLifecycleBridge;
|
||||||
@@ -77,7 +77,7 @@ mod tests {
|
|||||||
use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
|
use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
|
||||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||||
use rustfs_scanner_contracts::metrics::IlmAction;
|
use rustfs_scanner_metrics::metrics::IlmAction;
|
||||||
use s3s::dto::{
|
use s3s::dto::{
|
||||||
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication,
|
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication,
|
||||||
DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus,
|
DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus,
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ use super::{BucketQuota, QuotaCheckResult, QuotaError, QuotaOperation};
|
|||||||
use crate::bucket::metadata_sys::{BucketMetadataSys, update, update_if_incarnation};
|
use crate::bucket::metadata_sys::{BucketMetadataSys, update, update_if_incarnation};
|
||||||
use crate::data_usage::get_bucket_usage_memory;
|
use crate::data_usage::get_bucket_usage_memory;
|
||||||
use rustfs_config::QUOTA_CONFIG_FILE;
|
use rustfs_config::QUOTA_CONFIG_FILE;
|
||||||
use rustfs_scanner_contracts::metrics::Metric;
|
use rustfs_scanner_metrics::metrics::Metric;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
@@ -120,9 +120,9 @@ impl QuotaChecker {
|
|||||||
|
|
||||||
let duration = start_time.elapsed();
|
let duration = start_time.elapsed();
|
||||||
// inc_time is now a plain fn (not async) — no .await needed.
|
// inc_time is now a plain fn (not async) — no .await needed.
|
||||||
rustfs_scanner_contracts::metrics::Metrics::inc_time(Metric::QuotaCheck, duration);
|
rustfs_scanner_metrics::metrics::Metrics::inc_time(Metric::QuotaCheck, duration);
|
||||||
if !allowed {
|
if !allowed {
|
||||||
rustfs_scanner_contracts::metrics::Metrics::inc_time(Metric::QuotaViolation, duration);
|
rustfs_scanner_metrics::metrics::Metrics::inc_time(Metric::QuotaViolation, duration);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(result)
|
Ok(result)
|
||||||
@@ -185,7 +185,7 @@ impl QuotaChecker {
|
|||||||
.await
|
.await
|
||||||
.map_err(QuotaError::StorageError)?;
|
.map_err(QuotaError::StorageError)?;
|
||||||
|
|
||||||
rustfs_scanner_contracts::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
|
rustfs_scanner_metrics::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
|
||||||
Ok(updated_at)
|
Ok(updated_at)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -206,7 +206,7 @@ impl QuotaChecker {
|
|||||||
}
|
}
|
||||||
.map_err(QuotaError::StorageError)?;
|
.map_err(QuotaError::StorageError)?;
|
||||||
|
|
||||||
rustfs_scanner_contracts::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
|
rustfs_scanner_metrics::metrics::Metrics::inc_time(Metric::QuotaSync, start_time.elapsed());
|
||||||
Ok(updated_at)
|
Ok(updated_at)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -13504,7 +13504,9 @@ impl ECStore {
|
|||||||
}
|
}
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
let result = self.decommission_in_background(rx.clone(), idx, entry_budget).await;
|
let result = self
|
||||||
|
.decommission_in_background(rx.clone(), idx, generation, entry_budget)
|
||||||
|
.await;
|
||||||
|
|
||||||
if let Err(err) = &result
|
if let Err(err) = &result
|
||||||
&& (is_decommission_capacity_blocked_error(err) || is_decommission_target_capacity_error(err))
|
&& (is_decommission_capacity_blocked_error(err) || is_decommission_target_capacity_error(err))
|
||||||
@@ -13989,8 +13991,10 @@ impl ECStore {
|
|||||||
self: &Arc<Self>,
|
self: &Arc<Self>,
|
||||||
rx: CancellationToken,
|
rx: CancellationToken,
|
||||||
idx: usize,
|
idx: usize,
|
||||||
|
generation: OffsetDateTime,
|
||||||
entry_budget: Arc<Semaphore>,
|
entry_budget: Arc<Semaphore>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
|
self.ensure_decommission_runtime_capacity_available(idx, generation).await?;
|
||||||
let pool = get_by_index(self.pools.as_slice(), idx, "load decommission background pool")?.clone();
|
let pool = get_by_index(self.pools.as_slice(), idx, "load decommission background pool")?.clone();
|
||||||
|
|
||||||
let pending = {
|
let pending = {
|
||||||
@@ -17253,6 +17257,44 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial_test::serial]
|
||||||
|
async fn decommission_worker_rechecks_runtime_capacity_before_empty_background_completion() {
|
||||||
|
let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
|
||||||
|
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
|
||||||
|
let enough = vec![
|
||||||
|
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
|
||||||
|
DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
|
||||||
|
];
|
||||||
|
set_decommission_capacity_info_overrides_for_test(store.id, vec![enough.clone()]);
|
||||||
|
store
|
||||||
|
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
|
||||||
|
.await
|
||||||
|
.expect("the initial reservation should be activated");
|
||||||
|
|
||||||
|
let shortage = vec![enough[0], DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)];
|
||||||
|
set_decommission_capacity_info_overrides_for_test(store.id, vec![shortage]);
|
||||||
|
let canceler = DecommissionCanceler::new(CancellationToken::new());
|
||||||
|
store.decommission_cancelers.write().await[0] = Some(canceler.clone());
|
||||||
|
store
|
||||||
|
.do_decommission_in_routine(canceler, 0, Arc::new(Semaphore::new(1)))
|
||||||
|
.await
|
||||||
|
.expect("runtime capacity shortage should pause the worker before background completion");
|
||||||
|
|
||||||
|
let local = store.pool_meta.read().await;
|
||||||
|
let info = local.pools[0]
|
||||||
|
.decommission
|
||||||
|
.as_ref()
|
||||||
|
.expect("the blocked decommission state should remain present");
|
||||||
|
assert!(!info.complete && !info.failed && !info.canceled);
|
||||||
|
assert!(info.capacity_blocked_reason.is_some());
|
||||||
|
assert!(
|
||||||
|
info.capacity_reservation
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(DecommissionCapacityReservation::active)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
fn pool_meta_replica_test_meta(cmd_line: &str) -> PoolMeta {
|
fn pool_meta_replica_test_meta(cmd_line: &str) -> PoolMeta {
|
||||||
PoolMeta {
|
PoolMeta {
|
||||||
version: POOL_META_VERSION,
|
version: POOL_META_VERSION,
|
||||||
|
|||||||
@@ -286,7 +286,7 @@ pub struct QuotaAdmission {
|
|||||||
pub struct LifecycleDeleteAllRequest {
|
pub struct LifecycleDeleteAllRequest {
|
||||||
pub(crate) version_id: Option<Uuid>,
|
pub(crate) version_id: Option<Uuid>,
|
||||||
pub(crate) delete_marker: bool,
|
pub(crate) delete_marker: bool,
|
||||||
pub(crate) action: rustfs_scanner_contracts::metrics::IlmAction,
|
pub(crate) action: rustfs_scanner_metrics::metrics::IlmAction,
|
||||||
pub(crate) rule_id: String,
|
pub(crate) rule_id: String,
|
||||||
pub(crate) phase: LifecycleDeleteAllPhase,
|
pub(crate) phase: LifecycleDeleteAllPhase,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ use rustfs_madmin::metrics::{
|
|||||||
ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, ScannerSourceWorkSnapshot as MadminScannerSourceWorkSnapshot,
|
ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, ScannerSourceWorkSnapshot as MadminScannerSourceWorkSnapshot,
|
||||||
ScannerUsageFreshnessSnapshot as MadminScannerUsageFreshnessSnapshot, TimedAction as MadminTimedAction,
|
ScannerUsageFreshnessSnapshot as MadminScannerUsageFreshnessSnapshot, TimedAction as MadminTimedAction,
|
||||||
};
|
};
|
||||||
use rustfs_scanner_contracts::metrics::global_metrics;
|
use rustfs_scanner_metrics::metrics::global_metrics;
|
||||||
use rustfs_utils::os::get_drive_stats;
|
use rustfs_utils::os::get_drive_stats;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
@@ -82,7 +82,7 @@ fn unix_millis_to_jiff_timestamp(millis: u64, fallback: Timestamp) -> Timestamp
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn to_madmin_scanner_metrics(metrics: rustfs_scanner_contracts::metrics::ScannerMetricsReport) -> MadminScannerMetrics {
|
fn to_madmin_scanner_metrics(metrics: rustfs_scanner_metrics::metrics::ScannerMetricsReport) -> MadminScannerMetrics {
|
||||||
MadminScannerMetrics {
|
MadminScannerMetrics {
|
||||||
collected_at: metrics.collected_at,
|
collected_at: metrics.collected_at,
|
||||||
current_cycle: metrics.current_cycle,
|
current_cycle: metrics.current_cycle,
|
||||||
@@ -565,7 +565,7 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
|
|||||||
mod test {
|
mod test {
|
||||||
use super::*;
|
use super::*;
|
||||||
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
|
||||||
use rustfs_scanner_contracts::metrics::CurrentCycle;
|
use rustfs_scanner_metrics::metrics::CurrentCycle;
|
||||||
use serial_test::serial;
|
use serial_test::serial;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
@@ -619,7 +619,7 @@ mod test {
|
|||||||
#[test]
|
#[test]
|
||||||
fn scanner_metrics_mapping_preserves_partial_source_status() {
|
fn scanner_metrics_mapping_preserves_partial_source_status() {
|
||||||
let current_started = Utc::now() - chrono::Duration::seconds(5);
|
let current_started = Utc::now() - chrono::Duration::seconds(5);
|
||||||
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
|
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
|
||||||
current_cycle_active: true,
|
current_cycle_active: true,
|
||||||
current_started: chrono_to_jiff_timestamp(current_started),
|
current_started: chrono_to_jiff_timestamp(current_started),
|
||||||
last_cycle_partial_source: "usage".to_string(),
|
last_cycle_partial_source: "usage".to_string(),
|
||||||
@@ -628,7 +628,7 @@ mod test {
|
|||||||
cycle_recovery_required_total: 2,
|
cycle_recovery_required_total: 2,
|
||||||
cycle_last_progress_age: 17,
|
cycle_last_progress_age: 17,
|
||||||
leader_lease_without_progress: true,
|
leader_lease_without_progress: true,
|
||||||
partial_cycles_by_source: vec![rustfs_scanner_contracts::metrics::ScannerSourceCycleSnapshot {
|
partial_cycles_by_source: vec![rustfs_scanner_metrics::metrics::ScannerSourceCycleSnapshot {
|
||||||
source: "usage".to_string(),
|
source: "usage".to_string(),
|
||||||
cycles: 2,
|
cycles: 2,
|
||||||
}],
|
}],
|
||||||
@@ -654,11 +654,11 @@ mod test {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
|
async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
|
||||||
let previous_init_time = *rustfs_scanner_contracts::GLOBAL_INIT_TIME.read().await;
|
let previous_init_time = *rustfs_scanner_metrics::GLOBAL_INIT_TIME.read().await;
|
||||||
let previous_cycle = global_metrics().get_cycle().await;
|
let previous_cycle = global_metrics().get_cycle().await;
|
||||||
let init_time = Utc::now() - chrono::Duration::hours(1);
|
let init_time = Utc::now() - chrono::Duration::hours(1);
|
||||||
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
|
||||||
*rustfs_scanner_contracts::GLOBAL_INIT_TIME.write().await = Some(init_time);
|
*rustfs_scanner_metrics::GLOBAL_INIT_TIME.write().await = Some(init_time);
|
||||||
let cycle = CurrentCycle {
|
let cycle = CurrentCycle {
|
||||||
current: 0,
|
current: 0,
|
||||||
next: 1,
|
next: 1,
|
||||||
@@ -673,7 +673,7 @@ mod test {
|
|||||||
.finish_scan_cycle_work_with_cycle(cycle_start, previous_cycle.clone().unwrap_or_default())
|
.finish_scan_cycle_work_with_cycle(cycle_start, previous_cycle.clone().unwrap_or_default())
|
||||||
.await;
|
.await;
|
||||||
global_metrics().set_cycle(previous_cycle).await;
|
global_metrics().set_cycle(previous_cycle).await;
|
||||||
*rustfs_scanner_contracts::GLOBAL_INIT_TIME.write().await = previous_init_time;
|
*rustfs_scanner_metrics::GLOBAL_INIT_TIME.write().await = previous_init_time;
|
||||||
|
|
||||||
let encoded = rmp_serde::to_vec_named(&realtime).expect("realtime metrics should encode");
|
let encoded = rmp_serde::to_vec_named(&realtime).expect("realtime metrics should encode");
|
||||||
let decoded: RealtimeMetrics = rmp_serde::from_slice(&encoded).expect("realtime metrics should decode");
|
let decoded: RealtimeMetrics = rmp_serde::from_slice(&encoded).expect("realtime metrics should decode");
|
||||||
@@ -686,8 +686,8 @@ mod test {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn scanner_metrics_mapping_preserves_pacing_pressure() {
|
fn scanner_metrics_mapping_preserves_pacing_pressure() {
|
||||||
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
|
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
|
||||||
pacing_pressure: rustfs_scanner_contracts::metrics::ScannerPacingPressureSnapshot {
|
pacing_pressure: rustfs_scanner_metrics::metrics::ScannerPacingPressureSnapshot {
|
||||||
primary_pressure: "cycle_budget".to_string(),
|
primary_pressure: "cycle_budget".to_string(),
|
||||||
current_queued_scans: 4,
|
current_queued_scans: 4,
|
||||||
current_active_scans: 2,
|
current_active_scans: 2,
|
||||||
@@ -712,12 +712,12 @@ mod test {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn scanner_metrics_mapping_preserves_lifecycle_transition_status() {
|
fn scanner_metrics_mapping_preserves_lifecycle_transition_status() {
|
||||||
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
|
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
|
||||||
current_cycle_lifecycle_expiry_actions: 2,
|
current_cycle_lifecycle_expiry_actions: 2,
|
||||||
current_cycle_lifecycle_transition_actions: 3,
|
current_cycle_lifecycle_transition_actions: 3,
|
||||||
last_cycle_lifecycle_expiry_actions: 5,
|
last_cycle_lifecycle_expiry_actions: 5,
|
||||||
last_cycle_lifecycle_transition_actions: 7,
|
last_cycle_lifecycle_transition_actions: 7,
|
||||||
lifecycle_expiry: rustfs_scanner_contracts::metrics::ScannerLifecycleExpirySnapshot {
|
lifecycle_expiry: rustfs_scanner_metrics::metrics::ScannerLifecycleExpirySnapshot {
|
||||||
current_queue_capacity: 16,
|
current_queue_capacity: 16,
|
||||||
current_queued: 5,
|
current_queued: 5,
|
||||||
current_active: 2,
|
current_active: 2,
|
||||||
@@ -729,7 +729,7 @@ mod test {
|
|||||||
scanner_not_enqueued: 2,
|
scanner_not_enqueued: 2,
|
||||||
delete_failed: 1,
|
delete_failed: 1,
|
||||||
},
|
},
|
||||||
lifecycle_transition: rustfs_scanner_contracts::metrics::ScannerLifecycleTransitionSnapshot {
|
lifecycle_transition: rustfs_scanner_metrics::metrics::ScannerLifecycleTransitionSnapshot {
|
||||||
current_queue_capacity: 16,
|
current_queue_capacity: 16,
|
||||||
current_queued: 5,
|
current_queued: 5,
|
||||||
current_active: 2,
|
current_active: 2,
|
||||||
@@ -778,10 +778,10 @@ mod test {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn scanner_metrics_mapping_preserves_maintenance_control_status() {
|
fn scanner_metrics_mapping_preserves_maintenance_control_status() {
|
||||||
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
|
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
|
||||||
maintenance_control: rustfs_scanner_contracts::metrics::ScannerMaintenanceControlSnapshot {
|
maintenance_control: rustfs_scanner_metrics::metrics::ScannerMaintenanceControlSnapshot {
|
||||||
primary_control: "blocked_source".to_string(),
|
primary_control: "blocked_source".to_string(),
|
||||||
sources: vec![rustfs_scanner_contracts::metrics::ScannerMaintenanceSourceSnapshot {
|
sources: vec![rustfs_scanner_metrics::metrics::ScannerMaintenanceSourceSnapshot {
|
||||||
source: "lifecycle".to_string(),
|
source: "lifecycle".to_string(),
|
||||||
state: "blocked".to_string(),
|
state: "blocked".to_string(),
|
||||||
reason: "missed_work".to_string(),
|
reason: "missed_work".to_string(),
|
||||||
@@ -815,8 +815,8 @@ mod test {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn scanner_metrics_mapping_preserves_usage_freshness_status() {
|
fn scanner_metrics_mapping_preserves_usage_freshness_status() {
|
||||||
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
|
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
|
||||||
usage_freshness: rustfs_scanner_contracts::metrics::ScannerUsageFreshnessSnapshot {
|
usage_freshness: rustfs_scanner_metrics::metrics::ScannerUsageFreshnessSnapshot {
|
||||||
dirty_pending_buckets: 3,
|
dirty_pending_buckets: 3,
|
||||||
last_dirty_mark_unix_secs: 10,
|
last_dirty_mark_unix_secs: 10,
|
||||||
last_dirty_clear_unix_secs: 11,
|
last_dirty_clear_unix_secs: 11,
|
||||||
@@ -857,7 +857,7 @@ mod test {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn scanner_metrics_mapping_preserves_distributed_status_fields() {
|
fn scanner_metrics_mapping_preserves_distributed_status_fields() {
|
||||||
let scanner = to_madmin_scanner_metrics(rustfs_scanner_contracts::metrics::ScannerMetricsReport {
|
let scanner = to_madmin_scanner_metrics(rustfs_scanner_metrics::metrics::ScannerMetricsReport {
|
||||||
active_scan_paths: 2,
|
active_scan_paths: 2,
|
||||||
oldest_active_path_age_seconds: 45,
|
oldest_active_path_age_seconds: 45,
|
||||||
active_paths: vec!["disk-a/bucket-a".to_string(), "disk-b/bucket-b".to_string()],
|
active_paths: vec!["disk-a/bucket-a".to_string(), "disk-b/bucket-b".to_string()],
|
||||||
@@ -913,7 +913,7 @@ mod test {
|
|||||||
cycle_max_directories: 38,
|
cycle_max_directories: 38,
|
||||||
bitrot_cycle_enabled: true,
|
bitrot_cycle_enabled: true,
|
||||||
bitrot_cycle_seconds: 39.0,
|
bitrot_cycle_seconds: 39.0,
|
||||||
scan_checkpoint: Some(rustfs_scanner_contracts::metrics::ScannerCheckpointReport {
|
scan_checkpoint: Some(rustfs_scanner_metrics::metrics::ScannerCheckpointReport {
|
||||||
version: 1,
|
version: 1,
|
||||||
resume_after: "bucket-a/prefix-a".to_string(),
|
resume_after: "bucket-a/prefix-a".to_string(),
|
||||||
reason: "directories".to_string(),
|
reason: "directories".to_string(),
|
||||||
@@ -923,7 +923,7 @@ mod test {
|
|||||||
scan_checkpoint_cleared: 41,
|
scan_checkpoint_cleared: 41,
|
||||||
scan_checkpoint_ignored: 42,
|
scan_checkpoint_ignored: 42,
|
||||||
scan_checkpoint_stale: 43,
|
scan_checkpoint_stale: 43,
|
||||||
source_work: vec![rustfs_scanner_contracts::metrics::ScannerSourceWorkSnapshot {
|
source_work: vec![rustfs_scanner_metrics::metrics::ScannerSourceWorkSnapshot {
|
||||||
source: "usage".to_string(),
|
source: "usage".to_string(),
|
||||||
checked: 44,
|
checked: 44,
|
||||||
queued: 45,
|
queued: 45,
|
||||||
@@ -932,7 +932,7 @@ mod test {
|
|||||||
skipped: 48,
|
skipped: 48,
|
||||||
missed: 49,
|
missed: 49,
|
||||||
}],
|
}],
|
||||||
current_cycle_source_work: vec![rustfs_scanner_contracts::metrics::ScannerSourceWorkSnapshot {
|
current_cycle_source_work: vec![rustfs_scanner_metrics::metrics::ScannerSourceWorkSnapshot {
|
||||||
source: "lifecycle".to_string(),
|
source: "lifecycle".to_string(),
|
||||||
checked: 50,
|
checked: 50,
|
||||||
queued: 51,
|
queued: 51,
|
||||||
@@ -941,7 +941,7 @@ mod test {
|
|||||||
skipped: 54,
|
skipped: 54,
|
||||||
missed: 55,
|
missed: 55,
|
||||||
}],
|
}],
|
||||||
last_cycle_source_work: vec![rustfs_scanner_contracts::metrics::ScannerSourceWorkSnapshot {
|
last_cycle_source_work: vec![rustfs_scanner_metrics::metrics::ScannerSourceWorkSnapshot {
|
||||||
source: "heal".to_string(),
|
source: "heal".to_string(),
|
||||||
checked: 56,
|
checked: 56,
|
||||||
queued: 57,
|
queued: 57,
|
||||||
@@ -950,7 +950,7 @@ mod test {
|
|||||||
skipped: 60,
|
skipped: 60,
|
||||||
missed: 61,
|
missed: 61,
|
||||||
}],
|
}],
|
||||||
replication_repair: vec![rustfs_scanner_contracts::metrics::ScannerReplicationRepairSnapshot {
|
replication_repair: vec![rustfs_scanner_metrics::metrics::ScannerReplicationRepairSnapshot {
|
||||||
source: "bucket_replication".to_string(),
|
source: "bucket_replication".to_string(),
|
||||||
kind: "object".to_string(),
|
kind: "object".to_string(),
|
||||||
scanner_role: "repair_admission".to_string(),
|
scanner_role: "repair_admission".to_string(),
|
||||||
@@ -962,7 +962,7 @@ mod test {
|
|||||||
skipped: 66,
|
skipped: 66,
|
||||||
missed: 67,
|
missed: 67,
|
||||||
}],
|
}],
|
||||||
current_cycle_replication_repair: vec![rustfs_scanner_contracts::metrics::ScannerReplicationRepairSnapshot {
|
current_cycle_replication_repair: vec![rustfs_scanner_metrics::metrics::ScannerReplicationRepairSnapshot {
|
||||||
source: "bucket_replication".to_string(),
|
source: "bucket_replication".to_string(),
|
||||||
kind: "delete_marker".to_string(),
|
kind: "delete_marker".to_string(),
|
||||||
scanner_role: "repair_admission".to_string(),
|
scanner_role: "repair_admission".to_string(),
|
||||||
@@ -974,7 +974,7 @@ mod test {
|
|||||||
skipped: 72,
|
skipped: 72,
|
||||||
missed: 73,
|
missed: 73,
|
||||||
}],
|
}],
|
||||||
last_cycle_replication_repair: vec![rustfs_scanner_contracts::metrics::ScannerReplicationRepairSnapshot {
|
last_cycle_replication_repair: vec![rustfs_scanner_metrics::metrics::ScannerReplicationRepairSnapshot {
|
||||||
source: "site_replication".to_string(),
|
source: "site_replication".to_string(),
|
||||||
kind: "active_resync".to_string(),
|
kind: "active_resync".to_string(),
|
||||||
scanner_role: "boundary_signal".to_string(),
|
scanner_role: "boundary_signal".to_string(),
|
||||||
|
|||||||
@@ -1168,7 +1168,7 @@ mod lifecycle_delete_all_plan_tests {
|
|||||||
crate::object_api::LifecycleDeleteAllRequest {
|
crate::object_api::LifecycleDeleteAllRequest {
|
||||||
version_id: Some(version_id),
|
version_id: Some(version_id),
|
||||||
delete_marker: true,
|
delete_marker: true,
|
||||||
action: rustfs_scanner_contracts::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
|
action: rustfs_scanner_metrics::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
|
||||||
rule_id: "rule".to_string(),
|
rule_id: "rule".to_string(),
|
||||||
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
||||||
}
|
}
|
||||||
@@ -1361,7 +1361,7 @@ mod lifecycle_delete_all_plan_tests {
|
|||||||
let request = crate::object_api::LifecycleDeleteAllRequest {
|
let request = crate::object_api::LifecycleDeleteAllRequest {
|
||||||
version_id: None,
|
version_id: None,
|
||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction,
|
action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
|
||||||
rule_id: "rule".to_string(),
|
rule_id: "rule".to_string(),
|
||||||
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
||||||
};
|
};
|
||||||
@@ -19363,7 +19363,7 @@ mod delete_objects_lock_gating_tests {
|
|||||||
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
|
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
|
||||||
version_id: Some(trigger_version_id),
|
version_id: Some(trigger_version_id),
|
||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction,
|
action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
|
||||||
rule_id: "rule".to_string(),
|
rule_id: "rule".to_string(),
|
||||||
phase: crate::object_api::LifecycleDeleteAllPhase::History,
|
phase: crate::object_api::LifecycleDeleteAllPhase::History,
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -15934,7 +15934,7 @@ mod tests {
|
|||||||
.expect("unknown transition metadata should be written");
|
.expect("unknown transition metadata should be written");
|
||||||
}
|
}
|
||||||
let lifecycle_event = crate::bucket::lifecycle::lifecycle::Event {
|
let lifecycle_event = crate::bucket::lifecycle::lifecycle::Event {
|
||||||
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction,
|
action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
|
||||||
rule_id: "delete-all-versions".to_string(),
|
rule_id: "delete-all-versions".to_string(),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
@@ -16186,7 +16186,7 @@ mod tests {
|
|||||||
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
|
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
|
||||||
version_id: original.version_id,
|
version_id: original.version_id,
|
||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction,
|
action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
|
||||||
rule_id: "rule".to_string(),
|
rule_id: "rule".to_string(),
|
||||||
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -6448,7 +6448,7 @@ mod tests {
|
|||||||
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
|
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
|
||||||
version_id: Some(version_id),
|
version_id: Some(version_id),
|
||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction,
|
action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
|
||||||
rule_id: "delete-all".to_string(),
|
rule_id: "delete-all".to_string(),
|
||||||
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -1322,7 +1322,7 @@ mod tests {
|
|||||||
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
|
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
|
||||||
version_id: Some(trigger_id),
|
version_id: Some(trigger_id),
|
||||||
delete_marker: false,
|
delete_marker: false,
|
||||||
action: rustfs_scanner_contracts::metrics::IlmAction::DeleteAllVersionsAction,
|
action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
|
||||||
rule_id: "rule".to_string(),
|
rule_id: "rule".to_string(),
|
||||||
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
||||||
}),
|
}),
|
||||||
@@ -1492,7 +1492,7 @@ mod tests {
|
|||||||
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
|
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
|
||||||
version_id: Some(marker_id),
|
version_id: Some(marker_id),
|
||||||
delete_marker: true,
|
delete_marker: true,
|
||||||
action: rustfs_scanner_contracts::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
|
action: rustfs_scanner_metrics::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
|
||||||
rule_id: "rule".to_string(),
|
rule_id: "rule".to_string(),
|
||||||
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
||||||
}),
|
}),
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ hotpath-cpu = [
|
|||||||
hotpath.workspace = true
|
hotpath.workspace = true
|
||||||
async-trait.workspace = true
|
async-trait.workspace = true
|
||||||
metrics.workspace = true
|
metrics.workspace = true
|
||||||
rustfs-scanner-contracts.workspace = true
|
rustfs-scanner-metrics.workspace = true
|
||||||
rustfs-config = { workspace = true, features = ["constants"] }
|
rustfs-config = { workspace = true, features = ["constants"] }
|
||||||
rustfs-replication.workspace = true
|
rustfs-replication.workspace = true
|
||||||
rustfs-storage-api.workspace = true
|
rustfs-storage-api.workspace = true
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ const ERR_LIFECYCLE_EXPIRED_OBJECT_DELETE_MARKER_WITH_TAGS: &str =
|
|||||||
const ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION: &str = "Rule must have at least one of Expiration, Transition, NoncurrentVersionExpiration, NoncurrentVersionTransition, or DelMarkerExpiration";
|
const ERR_LIFECYCLE_RULE_MUST_HAVE_ACTION: &str = "Rule must have at least one of Expiration, Transition, NoncurrentVersionExpiration, NoncurrentVersionTransition, or DelMarkerExpiration";
|
||||||
const ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT: &str = "Legacy Prefix and Filter cannot both be present in a lifecycle rule. Use Filter.Prefix instead of the top-level Prefix element.";
|
const ERR_LIFECYCLE_PREFIX_FILTER_CONFLICT: &str = "Legacy Prefix and Filter cannot both be present in a lifecycle rule. Use Filter.Prefix instead of the top-level Prefix element.";
|
||||||
|
|
||||||
pub use rustfs_scanner_contracts::metrics::IlmAction;
|
pub use rustfs_scanner_metrics::metrics::IlmAction;
|
||||||
|
|
||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
pub trait RuleValidate {
|
pub trait RuleValidate {
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ use time::OffsetDateTime;
|
|||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
use rustfs_replication::ReplicationStatusType;
|
use rustfs_replication::ReplicationStatusType;
|
||||||
use rustfs_scanner_contracts::metrics::IlmAction;
|
use rustfs_scanner_metrics::metrics::IlmAction;
|
||||||
|
|
||||||
use crate::object_lock;
|
use crate::object_lock;
|
||||||
use crate::{Event, Lifecycle, ObjectOpts};
|
use crate::{Event, Lifecycle, ObjectOpts};
|
||||||
@@ -197,7 +197,7 @@ mod tests {
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use rustfs_scanner_contracts::metrics::IlmAction;
|
use rustfs_scanner_metrics::metrics::IlmAction;
|
||||||
use s3s::dto::{
|
use s3s::dto::{
|
||||||
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule,
|
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, LifecycleExpiration, LifecycleRule,
|
||||||
NoncurrentVersionExpiration, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule,
|
NoncurrentVersionExpiration, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRetentionMode, ObjectLockRule,
|
||||||
|
|||||||
@@ -21,4 +21,4 @@ mod tagging;
|
|||||||
pub use core::*;
|
pub use core::*;
|
||||||
pub use evaluator::Evaluator;
|
pub use evaluator::Evaluator;
|
||||||
pub use rustfs_replication::{ReplicationStatusType, VersionPurgeStatusType};
|
pub use rustfs_replication::{ReplicationStatusType, VersionPurgeStatusType};
|
||||||
pub use rustfs_scanner_contracts::metrics::IlmAction;
|
pub use rustfs_scanner_metrics::metrics::IlmAction;
|
||||||
|
|||||||
@@ -106,7 +106,7 @@ hotpath.workspace = true
|
|||||||
rustfs-audit = { workspace = true }
|
rustfs-audit = { workspace = true }
|
||||||
rustfs-common = { workspace = true }
|
rustfs-common = { workspace = true }
|
||||||
rustfs-heal-contracts = { workspace = true }
|
rustfs-heal-contracts = { workspace = true }
|
||||||
rustfs-scanner-contracts = { workspace = true }
|
rustfs-scanner-metrics = { workspace = true }
|
||||||
rustfs-config = { workspace = true, features = ["observability"] }
|
rustfs-config = { workspace = true, features = ["observability"] }
|
||||||
# NOTE: This dependency on rustfs-ecstore is a known architectural limitation.
|
# NOTE: This dependency on rustfs-ecstore is a known architectural limitation.
|
||||||
# The obs crate imports types from ecstore for metrics collection.
|
# The obs crate imports types from ecstore for metrics collection.
|
||||||
|
|||||||
@@ -493,7 +493,7 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::metrics::report::report_metrics;
|
use crate::metrics::report::report_metrics;
|
||||||
use metrics_util::debugging::DebuggingRecorder;
|
use metrics_util::debugging::DebuggingRecorder;
|
||||||
use rustfs_scanner_contracts::metrics::{Metric, Metrics};
|
use rustfs_scanner_metrics::metrics::{Metric, Metrics};
|
||||||
|
|
||||||
fn prometheus_counter_name(name: &str) -> String {
|
fn prometheus_counter_name(name: &str) -> String {
|
||||||
if name.ends_with("_total") {
|
if name.ends_with("_total") {
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ use rustfs_io_metrics::{
|
|||||||
ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, s3_op_metrics_snapshot,
|
ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, s3_op_metrics_snapshot,
|
||||||
snapshot_process_resource_and_system, snapshot_process_resource_and_system_with,
|
snapshot_process_resource_and_system, snapshot_process_resource_and_system_with,
|
||||||
};
|
};
|
||||||
use rustfs_scanner_contracts::metrics::{
|
use rustfs_scanner_metrics::metrics::{
|
||||||
ScannerActiveBucketDriveSnapshot, ScannerBucketDriveResultSnapshot, ScannerMetricsReport, ScannerSourceWorkSnapshot,
|
ScannerActiveBucketDriveSnapshot, ScannerBucketDriveResultSnapshot, ScannerMetricsReport, ScannerSourceWorkSnapshot,
|
||||||
global_metrics,
|
global_metrics,
|
||||||
};
|
};
|
||||||
@@ -1679,7 +1679,7 @@ pub async fn collect_compression_cluster_stats() -> Option<CompressionClusterSta
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use rustfs_scanner_contracts::metrics::ScannerSourceWorkSnapshot;
|
use rustfs_scanner_metrics::metrics::ScannerSourceWorkSnapshot;
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
use std::net::{Shutdown, TcpListener, TcpStream};
|
use std::net::{Shutdown, TcpListener, TcpStream};
|
||||||
use std::thread;
|
use std::thread;
|
||||||
@@ -2134,7 +2134,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn ilm_detail_stats_keep_expiry_and_transition_results_separate() {
|
fn ilm_detail_stats_keep_expiry_and_transition_results_separate() {
|
||||||
let report = ScannerMetricsReport {
|
let report = ScannerMetricsReport {
|
||||||
lifecycle_expiry: rustfs_scanner_contracts::metrics::ScannerLifecycleExpirySnapshot {
|
lifecycle_expiry: rustfs_scanner_metrics::metrics::ScannerLifecycleExpirySnapshot {
|
||||||
current_queued: 2,
|
current_queued: 2,
|
||||||
current_active: 1,
|
current_active: 1,
|
||||||
scanner_queued: 10,
|
scanner_queued: 10,
|
||||||
@@ -2142,7 +2142,7 @@ mod tests {
|
|||||||
delete_failed: 4,
|
delete_failed: 4,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
},
|
||||||
lifecycle_transition: rustfs_scanner_contracts::metrics::ScannerLifecycleTransitionSnapshot {
|
lifecycle_transition: rustfs_scanner_metrics::metrics::ScannerLifecycleTransitionSnapshot {
|
||||||
current_queued: 5,
|
current_queued: 5,
|
||||||
current_active: 6,
|
current_active: 6,
|
||||||
queue_full: 7,
|
queue_full: 7,
|
||||||
|
|||||||
@@ -20,26 +20,16 @@ license.workspace = true
|
|||||||
repository.workspace = true
|
repository.workspace = true
|
||||||
rust-version.workspace = true
|
rust-version.workspace = true
|
||||||
homepage.workspace = true
|
homepage.workspace = true
|
||||||
description = "Scanner metrics and lifecycle-cycle contracts shared by the scanner, storage engine, and observability layers."
|
description = "Scanner storage and wire contracts shared by the scanner and storage engine."
|
||||||
keywords = ["scanner", "contracts", "metrics", "rustfs", "Minio"]
|
keywords = ["scanner", "contracts", "rustfs", "Minio"]
|
||||||
categories = ["web-programming", "development-tools", "data-structures"]
|
categories = ["web-programming", "development-tools", "data-structures"]
|
||||||
|
|
||||||
[lints]
|
[lints]
|
||||||
workspace = true
|
workspace = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
chrono = { workspace = true, features = ["serde"] }
|
|
||||||
jiff = { workspace = true, features = ["serde"] }
|
|
||||||
metrics = { workspace = true }
|
|
||||||
rmp-serde = { workspace = true }
|
|
||||||
rustfs-heal-contracts = { workspace = true }
|
|
||||||
serde = { workspace = true, features = ["derive"] }
|
|
||||||
tokio = { workspace = true, features = ["sync"] }
|
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
serde_json = { workspace = true }
|
|
||||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
|
||||||
uuid = { workspace = true, features = ["v4"] }
|
|
||||||
|
|
||||||
[lib]
|
[lib]
|
||||||
doctest = false
|
doctest = false
|
||||||
|
|||||||
@@ -11,9 +11,3 @@
|
|||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub mod init_time;
|
|
||||||
pub mod last_minute;
|
|
||||||
pub mod metrics;
|
|
||||||
|
|
||||||
pub use init_time::{GLOBAL_INIT_TIME, get_global_init_time, set_global_init_time_now};
|
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
# Copyright 2024 RustFS Team
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
[package]
|
||||||
|
name = "rustfs-scanner-metrics"
|
||||||
|
version.workspace = true
|
||||||
|
edition.workspace = true
|
||||||
|
license.workspace = true
|
||||||
|
repository.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
|
homepage.workspace = true
|
||||||
|
description = "Scanner metrics and lifecycle-cycle telemetry shared by the scanner, storage engine, and observability layers."
|
||||||
|
keywords = ["scanner", "metrics", "rustfs", "Minio"]
|
||||||
|
categories = ["web-programming", "development-tools", "data-structures"]
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
chrono = { workspace = true, features = ["serde"] }
|
||||||
|
jiff = { workspace = true, features = ["serde"] }
|
||||||
|
metrics = { workspace = true }
|
||||||
|
rmp-serde = { workspace = true }
|
||||||
|
rustfs-heal-contracts = { workspace = true }
|
||||||
|
serde = { workspace = true, features = ["derive"] }
|
||||||
|
tokio = { workspace = true, features = ["sync"] }
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
serde_json = { workspace = true }
|
||||||
|
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
||||||
|
uuid = { workspace = true, features = ["v4"] }
|
||||||
|
|
||||||
|
[lib]
|
||||||
|
doctest = false
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
pub mod init_time;
|
||||||
|
pub mod last_minute;
|
||||||
|
pub mod metrics;
|
||||||
|
|
||||||
|
pub use init_time::{GLOBAL_INIT_TIME, get_global_init_time, set_global_init_time_now};
|
||||||
@@ -73,8 +73,9 @@ hotpath-cpu = [
|
|||||||
hotpath.workspace = true
|
hotpath.workspace = true
|
||||||
rustfs-config = { workspace = true, features = ["server-config-model"] }
|
rustfs-config = { workspace = true, features = ["server-config-model"] }
|
||||||
rustfs-common = { workspace = true }
|
rustfs-common = { workspace = true }
|
||||||
|
rustfs-concurrency = { workspace = true }
|
||||||
rustfs-heal-contracts = { workspace = true }
|
rustfs-heal-contracts = { workspace = true }
|
||||||
rustfs-scanner-contracts = { workspace = true }
|
rustfs-scanner-metrics = { workspace = true }
|
||||||
rustfs-credentials = { workspace = true }
|
rustfs-credentials = { workspace = true }
|
||||||
rustfs-utils = { workspace = true }
|
rustfs-utils = { workspace = true }
|
||||||
tokio = { workspace = true, features = ["fs", "sync", "time", "macros", "rt-multi-thread"] }
|
tokio = { workspace = true, features = ["fs", "sync", "time", "macros", "rt-multi-thread"] }
|
||||||
|
|||||||
@@ -70,6 +70,7 @@ mod scanner_heal_admission_baseline;
|
|||||||
pub mod scanner_io;
|
pub mod scanner_io;
|
||||||
pub mod sleeper;
|
pub mod sleeper;
|
||||||
pub(crate) mod storage_api;
|
pub(crate) mod storage_api;
|
||||||
|
mod workload_admission;
|
||||||
|
|
||||||
pub use data_usage_define::*;
|
pub use data_usage_define::*;
|
||||||
pub use error::ScannerError;
|
pub use error::ScannerError;
|
||||||
@@ -80,7 +81,7 @@ pub use remote_scanner::{
|
|||||||
remote_scanner_request_matches_envelope, serve_remote_scanner_request, validate_remote_scanner_request_fence,
|
remote_scanner_request_matches_envelope, serve_remote_scanner_request, validate_remote_scanner_request_fence,
|
||||||
};
|
};
|
||||||
pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
|
pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
|
||||||
pub use rustfs_scanner_contracts::last_minute;
|
pub use rustfs_scanner_metrics::last_minute;
|
||||||
pub use scanner::{
|
pub use scanner::{
|
||||||
ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason,
|
ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason,
|
||||||
ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, ScannerUsageStateResetResult,
|
ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, ScannerUsageStateResetResult,
|
||||||
@@ -96,6 +97,7 @@ pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
|||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
||||||
pub use storage_api::scan::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
|
pub use storage_api::scan::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
|
||||||
|
pub use workload_admission::set_scanner_workload_admission_snapshot_provider;
|
||||||
|
|
||||||
static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
|
static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
|
||||||
static SCANNER_RUNTIME_INSTANCES: AtomicU64 = AtomicU64::new(0);
|
static SCANNER_RUNTIME_INSTANCES: AtomicU64 = AtomicU64::new(0);
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ use crate::{
|
|||||||
use hmac::{Hmac, KeyInit, Mac};
|
use hmac::{Hmac, KeyInit, Mac};
|
||||||
use rustfs_credentials::try_get_rpc_token;
|
use rustfs_credentials::try_get_rpc_token;
|
||||||
use rustfs_heal_contracts::heal_channel::HealScanMode;
|
use rustfs_heal_contracts::heal_channel::HealScanMode;
|
||||||
use rustfs_scanner_contracts::metrics::{Metric, Metrics};
|
use rustfs_scanner_metrics::metrics::{Metric, Metrics};
|
||||||
use rustfs_utils::path::path_join_buf;
|
use rustfs_utils::path::path_join_buf;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use sha2::Sha256;
|
use sha2::Sha256;
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELA
|
|||||||
use rustfs_data_usage::observed_data_usage_is_newer;
|
use rustfs_data_usage::observed_data_usage_is_newer;
|
||||||
use rustfs_heal_contracts::heal_channel::HealScanMode;
|
use rustfs_heal_contracts::heal_channel::HealScanMode;
|
||||||
use rustfs_lock::{NamespaceLockGuard, error::LockError};
|
use rustfs_lock::{NamespaceLockGuard, error::LockError};
|
||||||
use rustfs_scanner_contracts::metrics::{
|
use rustfs_scanner_metrics::metrics::{
|
||||||
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource,
|
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource,
|
||||||
emit_scan_cycle_complete, emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded,
|
emit_scan_cycle_complete, emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded,
|
||||||
global_metrics,
|
global_metrics,
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ use rustfs_heal_contracts::heal_channel::{
|
|||||||
HEAL_DELETE_DANGLING, HealAdmissionDropReason, HealAdmissionResult, HealChannelPriority, HealChannelRequest,
|
HEAL_DELETE_DANGLING, HealAdmissionDropReason, HealAdmissionResult, HealChannelPriority, HealChannelRequest,
|
||||||
HealRequestSource, HealScanMode, send_heal_request_with_admission,
|
HealRequestSource, HealScanMode, send_heal_request_with_admission,
|
||||||
};
|
};
|
||||||
use rustfs_scanner_contracts::metrics::{
|
use rustfs_scanner_metrics::metrics::{
|
||||||
CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource,
|
CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource,
|
||||||
UpdateCurrentPathFn, current_path_updater, global_metrics,
|
UpdateCurrentPathFn, current_path_updater, global_metrics,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ use rustfs_data_usage::{BucketTargetUsageInfo, BucketUsageInfo};
|
|||||||
use rustfs_filemeta::FileMeta;
|
use rustfs_filemeta::FileMeta;
|
||||||
use rustfs_heal_contracts::heal_channel::HealScanMode;
|
use rustfs_heal_contracts::heal_channel::HealScanMode;
|
||||||
use rustfs_lock::{LockError, NamespaceLockGuard};
|
use rustfs_lock::{LockError, NamespaceLockGuard};
|
||||||
use rustfs_scanner_contracts::metrics::{
|
use rustfs_scanner_metrics::metrics::{
|
||||||
Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics,
|
Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics,
|
||||||
};
|
};
|
||||||
use rustfs_utils::path::path_join_buf;
|
use rustfs_utils::path::path_join_buf;
|
||||||
|
|||||||
@@ -147,13 +147,13 @@ impl Drop for DiskBucketScanActiveGuard {
|
|||||||
|
|
||||||
pub(super) struct BucketDriveFailureGuard {
|
pub(super) struct BucketDriveFailureGuard {
|
||||||
failed: bool,
|
failed: bool,
|
||||||
source: rustfs_scanner_contracts::metrics::ScannerWorkSource,
|
source: rustfs_scanner_metrics::metrics::ScannerWorkSource,
|
||||||
bucket: String,
|
bucket: String,
|
||||||
drive: String,
|
drive: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BucketDriveFailureGuard {
|
impl BucketDriveFailureGuard {
|
||||||
pub(super) fn new(source: rustfs_scanner_contracts::metrics::ScannerWorkSource, bucket: &str, drive: &str) -> Self {
|
pub(super) fn new(source: rustfs_scanner_metrics::metrics::ScannerWorkSource, bucket: &str, drive: &str) -> Self {
|
||||||
Self {
|
Self {
|
||||||
failed: true,
|
failed: true,
|
||||||
source,
|
source,
|
||||||
@@ -245,7 +245,7 @@ pub(super) fn scanner_concurrency_limit(configured: usize, available: usize) ->
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
if crate::current_foreground_read_activity() > 0 {
|
if crate::workload_admission::foreground_workload_activity() > 0 {
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -285,7 +285,7 @@ pub(super) fn scanner_task_join_error(stage: &str, err: tokio::task::JoinError)
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use rustfs_scanner_contracts::metrics::{ScannerWorkSource, global_metrics};
|
use rustfs_scanner_metrics::metrics::{ScannerWorkSource, global_metrics};
|
||||||
use tokio::sync::oneshot;
|
use tokio::sync::oneshot;
|
||||||
|
|
||||||
fn active_bucket_drive_count(source: ScannerWorkSource, bucket: &str, drive: &str) -> u64 {
|
fn active_bucket_drive_count(source: ScannerWorkSource, bucket: &str, drive: &str) -> u64 {
|
||||||
|
|||||||
@@ -161,8 +161,8 @@ impl ScannerIODisk for Disk {
|
|||||||
let bucket = cache.info.name.clone();
|
let bucket = cache.info.name.clone();
|
||||||
let disk_path = self.path().to_string_lossy().to_string();
|
let disk_path = self.path().to_string_lossy().to_string();
|
||||||
let source = match scan_mode {
|
let source = match scan_mode {
|
||||||
HealScanMode::Deep => rustfs_scanner_contracts::metrics::ScannerWorkSource::Bitrot,
|
HealScanMode::Deep => rustfs_scanner_metrics::metrics::ScannerWorkSource::Bitrot,
|
||||||
HealScanMode::Normal | HealScanMode::Unknown => rustfs_scanner_contracts::metrics::ScannerWorkSource::Usage,
|
HealScanMode::Normal | HealScanMode::Unknown => rustfs_scanner_metrics::metrics::ScannerWorkSource::Usage,
|
||||||
};
|
};
|
||||||
global_metrics().record_scan_bucket_drive_start(source, &bucket, &disk_path);
|
global_metrics().record_scan_bucket_drive_start(source, &bucket, &disk_path);
|
||||||
let mut failure_guard = BucketDriveFailureGuard::new(source, &bucket, &disk_path);
|
let mut failure_guard = BucketDriveFailureGuard::new(source, &bucket, &disk_path);
|
||||||
|
|||||||
@@ -26,12 +26,32 @@ use crate::{
|
|||||||
ScannerPutObjReader, UNKNOWN_TIER, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests,
|
ScannerPutObjReader, UNKNOWN_TIER, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests,
|
||||||
init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
|
init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
|
||||||
};
|
};
|
||||||
|
use rustfs_concurrency::{
|
||||||
|
AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadAdmissionSnapshotProvider,
|
||||||
|
WorkloadClass,
|
||||||
|
};
|
||||||
use rustfs_filemeta::FileInfo;
|
use rustfs_filemeta::FileInfo;
|
||||||
use serial_test::serial;
|
use serial_test::serial;
|
||||||
|
use std::sync::Arc;
|
||||||
use temp_env::with_var;
|
use temp_env::with_var;
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct FixedWorkloadProvider {
|
||||||
|
snapshot: WorkloadAdmissionRegistrySnapshot,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkloadAdmissionSnapshotProvider for FixedWorkloadProvider {
|
||||||
|
fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot {
|
||||||
|
self.snapshot.clone()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn install_scanner_workload_provider(snapshot: WorkloadAdmissionRegistrySnapshot) {
|
||||||
|
crate::set_scanner_workload_admission_snapshot_provider(Arc::new(FixedWorkloadProvider { snapshot }));
|
||||||
|
}
|
||||||
|
|
||||||
fn bucket_info(name: &str) -> BucketInfo {
|
fn bucket_info(name: &str) -> BucketInfo {
|
||||||
BucketInfo {
|
BucketInfo {
|
||||||
name: name.to_string(),
|
name: name.to_string(),
|
||||||
@@ -1079,6 +1099,7 @@ async fn bucket_cache_pending_heal_reaches_cycle_maintenance_state() {
|
|||||||
#[serial]
|
#[serial]
|
||||||
fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
|
fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
|
||||||
crate::reset_foreground_read_activity_for_test();
|
crate::reset_foreground_read_activity_for_test();
|
||||||
|
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
assert_eq!(scanner_concurrency_limit(0, 4), 4);
|
assert_eq!(scanner_concurrency_limit(0, 4), 4);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1086,6 +1107,7 @@ fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
|
|||||||
#[serial]
|
#[serial]
|
||||||
fn scanner_concurrency_limit_caps_to_configured_value() {
|
fn scanner_concurrency_limit_caps_to_configured_value() {
|
||||||
crate::reset_foreground_read_activity_for_test();
|
crate::reset_foreground_read_activity_for_test();
|
||||||
|
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
assert_eq!(scanner_concurrency_limit(2, 4), 2);
|
assert_eq!(scanner_concurrency_limit(2, 4), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1093,6 +1115,7 @@ fn scanner_concurrency_limit_caps_to_configured_value() {
|
|||||||
#[serial]
|
#[serial]
|
||||||
fn scanner_concurrency_limit_never_exceeds_available_work() {
|
fn scanner_concurrency_limit_never_exceeds_available_work() {
|
||||||
crate::reset_foreground_read_activity_for_test();
|
crate::reset_foreground_read_activity_for_test();
|
||||||
|
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
assert_eq!(scanner_concurrency_limit(8, 4), 4);
|
assert_eq!(scanner_concurrency_limit(8, 4), 4);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1100,6 +1123,7 @@ fn scanner_concurrency_limit_never_exceeds_available_work() {
|
|||||||
#[serial]
|
#[serial]
|
||||||
fn scanner_concurrency_limit_handles_no_available_work() {
|
fn scanner_concurrency_limit_handles_no_available_work() {
|
||||||
crate::reset_foreground_read_activity_for_test();
|
crate::reset_foreground_read_activity_for_test();
|
||||||
|
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
assert_eq!(scanner_concurrency_limit(2, 0), 0);
|
assert_eq!(scanner_concurrency_limit(2, 0), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1107,16 +1131,33 @@ fn scanner_concurrency_limit_handles_no_available_work() {
|
|||||||
#[serial]
|
#[serial]
|
||||||
fn scanner_concurrency_limit_yields_to_foreground_reads() {
|
fn scanner_concurrency_limit_yields_to_foreground_reads() {
|
||||||
crate::reset_foreground_read_activity_for_test();
|
crate::reset_foreground_read_activity_for_test();
|
||||||
|
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
crate::set_foreground_read_activity(8);
|
crate::set_foreground_read_activity(8);
|
||||||
assert_eq!(scanner_concurrency_limit(0, 4), 1);
|
assert_eq!(scanner_concurrency_limit(0, 4), 1);
|
||||||
assert_eq!(scanner_concurrency_limit(3, 4), 1);
|
assert_eq!(scanner_concurrency_limit(3, 4), 1);
|
||||||
crate::reset_foreground_read_activity_for_test();
|
crate::reset_foreground_read_activity_for_test();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial]
|
||||||
|
fn scanner_concurrency_limit_yields_to_shared_foreground_pressure() {
|
||||||
|
crate::reset_foreground_read_activity_for_test();
|
||||||
|
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
|
install_scanner_workload_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
|
||||||
|
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Open).with_counts(Some(2), None, Some(16)),
|
||||||
|
]));
|
||||||
|
|
||||||
|
assert_eq!(scanner_concurrency_limit(0, 4), 1);
|
||||||
|
assert_eq!(scanner_concurrency_limit(3, 4), 1);
|
||||||
|
|
||||||
|
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
#[serial]
|
||||||
fn scanner_concurrency_limit_yields_to_streaming_reads() {
|
fn scanner_concurrency_limit_yields_to_streaming_reads() {
|
||||||
crate::reset_foreground_read_activity_for_test();
|
crate::reset_foreground_read_activity_for_test();
|
||||||
|
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
let _guard = crate::ForegroundReadGuard::new();
|
let _guard = crate::ForegroundReadGuard::new();
|
||||||
|
|
||||||
assert_eq!(scanner_concurrency_limit(0, 4), 1);
|
assert_eq!(scanner_concurrency_limit(0, 4), 1);
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ use rustfs_config::{
|
|||||||
DEFAULT_SCANNER_IDLE_MODE, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED,
|
DEFAULT_SCANNER_IDLE_MODE, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED,
|
||||||
ENV_SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed,
|
ENV_SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed,
|
||||||
};
|
};
|
||||||
use rustfs_scanner_contracts::metrics::global_metrics;
|
use rustfs_scanner_metrics::metrics::global_metrics;
|
||||||
use tokio::time::Duration;
|
use tokio::time::Duration;
|
||||||
|
|
||||||
const MIN_SLEEP: Duration = Duration::from_millis(1);
|
const MIN_SLEEP: Duration = Duration::from_millis(1);
|
||||||
@@ -29,8 +29,8 @@ const SCANNER_SPEED_FAST: u8 = 1;
|
|||||||
const SCANNER_SPEED_DEFAULT: u8 = 2;
|
const SCANNER_SPEED_DEFAULT: u8 = 2;
|
||||||
const SCANNER_SPEED_SLOW: u8 = 3;
|
const SCANNER_SPEED_SLOW: u8 = 3;
|
||||||
const SCANNER_SPEED_SLOWEST: u8 = 4;
|
const SCANNER_SPEED_SLOWEST: u8 = 4;
|
||||||
const FOREGROUND_READ_BACKOFF_PER_REQUEST_MS: u64 = 10;
|
const FOREGROUND_WORKLOAD_BACKOFF_PER_REQUEST_MS: u64 = 10;
|
||||||
const FOREGROUND_READ_BACKOFF_MAX_MS: u64 = 250;
|
const FOREGROUND_WORKLOAD_BACKOFF_MAX_MS: u64 = 250;
|
||||||
|
|
||||||
static SCANNER_DEFAULT_SPEED_PRESET: AtomicU8 = AtomicU8::new(SCANNER_SPEED_DEFAULT);
|
static SCANNER_DEFAULT_SPEED_PRESET: AtomicU8 = AtomicU8::new(SCANNER_SPEED_DEFAULT);
|
||||||
|
|
||||||
@@ -78,15 +78,15 @@ pub(crate) fn scanner_yield_every_n_objects() -> u64 {
|
|||||||
rustfs_utils::get_env_u64(ENV_SCANNER_YIELD_EVERY_N_OBJECTS, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS)
|
rustfs_utils::get_env_u64(ENV_SCANNER_YIELD_EVERY_N_OBJECTS, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn foreground_read_backoff_duration(active_reads: u64) -> Duration {
|
fn foreground_workload_backoff_duration(active_foreground_workloads: u64) -> Duration {
|
||||||
if active_reads == 0 {
|
if active_foreground_workloads == 0 {
|
||||||
return Duration::ZERO;
|
return Duration::ZERO;
|
||||||
}
|
}
|
||||||
|
|
||||||
Duration::from_millis(
|
Duration::from_millis(
|
||||||
active_reads
|
active_foreground_workloads
|
||||||
.saturating_mul(FOREGROUND_READ_BACKOFF_PER_REQUEST_MS)
|
.saturating_mul(FOREGROUND_WORKLOAD_BACKOFF_PER_REQUEST_MS)
|
||||||
.min(FOREGROUND_READ_BACKOFF_MAX_MS),
|
.min(FOREGROUND_WORKLOAD_BACKOFF_MAX_MS),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -147,14 +147,15 @@ impl DynamicSleeper {
|
|||||||
}
|
}
|
||||||
let (factor, max_sleep) = self.read_params();
|
let (factor, max_sleep) = self.read_params();
|
||||||
if factor == 0.0 || max_sleep.is_zero() {
|
if factor == 0.0 || max_sleep.is_zero() {
|
||||||
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity());
|
let foreground_sleep =
|
||||||
|
foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
|
||||||
if !foreground_sleep.is_zero() {
|
if !foreground_sleep.is_zero() {
|
||||||
tokio::time::sleep(foreground_sleep).await;
|
tokio::time::sleep(foreground_sleep).await;
|
||||||
global_metrics().record_scanner_throttle_sleep(foreground_sleep);
|
global_metrics().record_scanner_throttle_sleep(foreground_sleep);
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity());
|
let foreground_sleep = foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
|
||||||
let sleep_dur = Duration::from_secs_f64(MIN_SLEEP.as_secs_f64() * factor)
|
let sleep_dur = Duration::from_secs_f64(MIN_SLEEP.as_secs_f64() * factor)
|
||||||
.min(max_sleep)
|
.min(max_sleep)
|
||||||
.max(foreground_sleep);
|
.max(foreground_sleep);
|
||||||
@@ -235,7 +236,8 @@ impl SleepTimer {
|
|||||||
}
|
}
|
||||||
let (factor, max_sleep) = self.sleeper.read_params();
|
let (factor, max_sleep) = self.sleeper.read_params();
|
||||||
if factor == 0.0 || max_sleep.is_zero() {
|
if factor == 0.0 || max_sleep.is_zero() {
|
||||||
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity());
|
let foreground_sleep =
|
||||||
|
foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
|
||||||
if !foreground_sleep.is_zero() {
|
if !foreground_sleep.is_zero() {
|
||||||
tokio::time::sleep(foreground_sleep).await;
|
tokio::time::sleep(foreground_sleep).await;
|
||||||
global_metrics().record_scanner_throttle_sleep(foreground_sleep);
|
global_metrics().record_scanner_throttle_sleep(foreground_sleep);
|
||||||
@@ -243,7 +245,7 @@ impl SleepTimer {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let elapsed = self.start.elapsed();
|
let elapsed = self.start.elapsed();
|
||||||
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity());
|
let foreground_sleep = foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
|
||||||
let sleep_dur = Duration::from_secs_f64(elapsed.as_secs_f64() * factor)
|
let sleep_dur = Duration::from_secs_f64(elapsed.as_secs_f64() * factor)
|
||||||
.max(MIN_SLEEP)
|
.max(MIN_SLEEP)
|
||||||
.min(max_sleep)
|
.min(max_sleep)
|
||||||
@@ -304,10 +306,10 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn foreground_read_backoff_is_capped() {
|
fn foreground_workload_backoff_is_capped() {
|
||||||
assert_eq!(foreground_read_backoff_duration(0), Duration::ZERO);
|
assert_eq!(foreground_workload_backoff_duration(0), Duration::ZERO);
|
||||||
assert_eq!(foreground_read_backoff_duration(1), Duration::from_millis(10));
|
assert_eq!(foreground_workload_backoff_duration(1), Duration::from_millis(10));
|
||||||
assert_eq!(foreground_read_backoff_duration(80), Duration::from_millis(250));
|
assert_eq!(foreground_workload_backoff_duration(80), Duration::from_millis(250));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -0,0 +1,165 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
use std::sync::{Arc, LazyLock, RwLock};
|
||||||
|
|
||||||
|
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshotProvider, WorkloadClass};
|
||||||
|
|
||||||
|
type WorkloadSnapshotProviderRef = Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>;
|
||||||
|
|
||||||
|
static SCANNER_WORKLOAD_ADMISSION_PROVIDER: LazyLock<RwLock<Option<WorkloadSnapshotProviderRef>>> =
|
||||||
|
LazyLock::new(|| RwLock::new(None));
|
||||||
|
|
||||||
|
pub fn set_scanner_workload_admission_snapshot_provider(provider: WorkloadSnapshotProviderRef) {
|
||||||
|
*SCANNER_WORKLOAD_ADMISSION_PROVIDER
|
||||||
|
.write()
|
||||||
|
.unwrap_or_else(|err| err.into_inner()) = Some(provider);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn scanner_workload_admission_snapshot_provider() -> Option<WorkloadSnapshotProviderRef> {
|
||||||
|
SCANNER_WORKLOAD_ADMISSION_PROVIDER
|
||||||
|
.read()
|
||||||
|
.unwrap_or_else(|err| err.into_inner())
|
||||||
|
.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn clear_scanner_workload_admission_snapshot_provider_for_test() {
|
||||||
|
*SCANNER_WORKLOAD_ADMISSION_PROVIDER
|
||||||
|
.write()
|
||||||
|
.unwrap_or_else(|err| err.into_inner()) = None;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn foreground_workload_activity() -> u64 {
|
||||||
|
let local_activity = crate::current_foreground_read_activity();
|
||||||
|
let Some(provider) = scanner_workload_admission_snapshot_provider() else {
|
||||||
|
return local_activity;
|
||||||
|
};
|
||||||
|
|
||||||
|
local_activity.max(foreground_activity_from_snapshot(&provider.workload_admission_snapshot()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn foreground_activity_from_snapshot(snapshot: &WorkloadAdmissionRegistrySnapshot) -> u64 {
|
||||||
|
[WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite]
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|class| snapshot.get(class))
|
||||||
|
.map(|entry| {
|
||||||
|
entry
|
||||||
|
.active
|
||||||
|
.or_else(|| {
|
||||||
|
matches!(entry.state, AdmissionState::Saturated).then(|| entry.limit.filter(|limit| *limit > 0).unwrap_or(1))
|
||||||
|
})
|
||||||
|
.map(usize_to_u64_saturated)
|
||||||
|
.unwrap_or(0)
|
||||||
|
})
|
||||||
|
.max()
|
||||||
|
.unwrap_or(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn usize_to_u64_saturated(value: usize) -> u64 {
|
||||||
|
u64::try_from(value).unwrap_or(u64::MAX)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshot};
|
||||||
|
use serial_test::serial;
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
struct FixedWorkloadProvider {
|
||||||
|
snapshot: WorkloadAdmissionRegistrySnapshot,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkloadAdmissionSnapshotProvider for FixedWorkloadProvider {
|
||||||
|
fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot {
|
||||||
|
self.snapshot.clone()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn install_provider(snapshot: WorkloadAdmissionRegistrySnapshot) {
|
||||||
|
set_scanner_workload_admission_snapshot_provider(Arc::new(FixedWorkloadProvider { snapshot }));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial]
|
||||||
|
fn foreground_workload_activity_falls_back_to_local_read_activity() {
|
||||||
|
clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
|
crate::reset_foreground_read_activity_for_test();
|
||||||
|
crate::set_foreground_read_activity(3);
|
||||||
|
|
||||||
|
assert_eq!(foreground_workload_activity(), 3);
|
||||||
|
|
||||||
|
crate::reset_foreground_read_activity_for_test();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial]
|
||||||
|
fn foreground_workload_activity_uses_shared_provider_counts() {
|
||||||
|
clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
|
crate::reset_foreground_read_activity_for_test();
|
||||||
|
install_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
|
||||||
|
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Open).with_counts(
|
||||||
|
Some(5),
|
||||||
|
None,
|
||||||
|
Some(8),
|
||||||
|
),
|
||||||
|
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Open).with_counts(
|
||||||
|
Some(2),
|
||||||
|
None,
|
||||||
|
Some(4),
|
||||||
|
),
|
||||||
|
]));
|
||||||
|
|
||||||
|
assert_eq!(foreground_workload_activity(), 5);
|
||||||
|
|
||||||
|
clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial]
|
||||||
|
fn foreground_workload_activity_treats_saturation_without_counts_as_pressure() {
|
||||||
|
clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
|
crate::reset_foreground_read_activity_for_test();
|
||||||
|
install_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
|
||||||
|
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Saturated).with_counts(
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
Some(7),
|
||||||
|
),
|
||||||
|
]));
|
||||||
|
|
||||||
|
assert_eq!(foreground_workload_activity(), 7);
|
||||||
|
|
||||||
|
clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial]
|
||||||
|
fn foreground_workload_activity_treats_zero_limit_saturation_as_pressure() {
|
||||||
|
clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
|
crate::reset_foreground_read_activity_for_test();
|
||||||
|
install_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
|
||||||
|
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Saturated).with_counts(
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
Some(0),
|
||||||
|
),
|
||||||
|
]));
|
||||||
|
|
||||||
|
assert_eq!(foreground_workload_activity(), 1);
|
||||||
|
|
||||||
|
clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -42,6 +42,8 @@ Required headings and strings in these files are asserted by `scripts/check_arch
|
|||||||
| [workload-admission-contracts.md](workload-admission-contracts.md) | adding a workload class or snapshot provider, or consuming admission state from a background job |
|
| [workload-admission-contracts.md](workload-admission-contracts.md) | adding a workload class or snapshot provider, or consuming admission state from a background job |
|
||||||
| [background-controller-contract.md](background-controller-contract.md) | adding a status snapshot or reconcile surface for a background service, or being tempted to fold several services into a generic controller |
|
| [background-controller-contract.md](background-controller-contract.md) | adding a status snapshot or reconcile surface for a background service, or being tempted to fold several services into a generic controller |
|
||||||
| [background-services-inventory.md](background-services-inventory.md) | you need one audited background service's desired source, current-status inputs, status surface, and declared side effects |
|
| [background-services-inventory.md](background-services-inventory.md) | you need one audited background service's desired source, current-status inputs, status surface, and declared side effects |
|
||||||
|
| [scanner-usage-publication.md](scanner-usage-publication.md) | changing scanner data-usage cache publication, quota-visible usage snapshots, scanner cycle recovery, or the persisted scanner usage artifacts |
|
||||||
|
| [scanner-usage-authority-decision.md](scanner-usage-authority-decision.md) | deciding whether quota admission depends on scanner data usage, removing scanner publication layers, or designing a scanner storage boundary |
|
||||||
| [config-model-boundary-adr.md](config-model-boundary-adr.md) | touching the server-config model (`Config`, `KV`, `KVS`) or its persistence, or asking which crate owns which part of server configuration |
|
| [config-model-boundary-adr.md](config-model-boundary-adr.md) | touching the server-config model (`Config`, `KV`, `KVS`) or its persistence, or asking which crate owns which part of server configuration |
|
||||||
| [admin-route-action-snapshot.md](admin-route-action-snapshot.md) | adding, moving, or re-authorizing an admin route and needing to know where the route → handler → `AdminAction` contract is enforced |
|
| [admin-route-action-snapshot.md](admin-route-action-snapshot.md) | adding, moving, or re-authorizing an admin route and needing to know where the route → handler → `AdminAction` contract is enforced |
|
||||||
| [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) | changing the bulk envelope re-wrap sweep, its admin endpoints, the re-wrap primitive, or which objects a rekey may touch |
|
| [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) | changing the bulk envelope re-wrap sweep, its admin endpoints, the re-wrap primitive, or which objects a rekey may touch |
|
||||||
|
|||||||
@@ -49,6 +49,20 @@ Outer crates reach ECStore only through `rustfs_ecstore::api`, and only from one
|
|||||||
- RustFS startup internals are crate-private: only `startup_entrypoint` is a public startup module of the `rustfs` library (`rustfs/src/lib.rs`), and items inside the other `startup_*` modules use crate visibility.
|
- RustFS startup internals are crate-private: only `startup_entrypoint` is a public startup module of the `rustfs` library (`rustfs/src/lib.rs`), and items inside the other `startup_*` modules use crate visibility.
|
||||||
- The observability dependency baseline is [obs-ecstore-dependency-inventory.md](obs-ecstore-dependency-inventory.md); observability extraction updates it together with the guard.
|
- The observability dependency baseline is [obs-ecstore-dependency-inventory.md](obs-ecstore-dependency-inventory.md); observability extraction updates it together with the guard.
|
||||||
|
|
||||||
|
## Scanner, Heal, And ECStore
|
||||||
|
|
||||||
|
Heal is split by responsibility, not by the shared word "heal". ECStore owns erasure-set repair primitives: quorum metadata arbitration, EC reconstruction, per-disk rename commit, dangling metadata classification, and orphan data-dir reclamation. These stay in ECStore because they share the same object namespace locks, rename commit model, and data-dir cleanup rules as PUT, DELETE, multipart, lifecycle expiry, rebalance, and decommission. Moving those primitives out would split the lock and commit model across crates.
|
||||||
|
|
||||||
|
`crates/heal` owns repair orchestration: queueing, deduplication, admission, scheduling, resume, MRF replay, replacement-disk tracking, and the admin-facing status/control surface. It reaches storage through `HealStorageAPI`; ECStore-originated repair requests flow back through typed repair channels rather than a Cargo dependency on the heal crate.
|
||||||
|
|
||||||
|
`crates/scanner` owns discovery, data-usage publication, lifecycle/replication scan actions, bitrot scan dispatch, and scanner-driven repair requests. Scanner may request repair through the heal channel, but it must not directly execute erasure-set repair primitives.
|
||||||
|
|
||||||
|
`rustfs-scanner-metrics` owns scanner telemetry DTOs, global scanner counters, lifecycle action labels consumed by metrics, and the short-window latency accumulator used by those metrics. ECStore, lifecycle, observability, admin, and scanner code may depend on this crate for metrics only. `rustfs-scanner-contracts` must not regain metrics, globals, or telemetry implementation; it is reserved for scanner storage or wire contract types.
|
||||||
|
|
||||||
|
`remote_scanner` remains scanner-owned for now because it carries the scanner cycle fence, replay protection, stream envelope, and per-bucket scan result protocol. A future scanner storage seam may either move remote disk scan execution behind an ECStore storage capability or move the whole remote scanner protocol with scanner; leaving the wire protocol split across both sides without a documented owner is not allowed.
|
||||||
|
|
||||||
|
The scanner usage authority decision is fixed in [scanner-usage-authority-decision.md](scanner-usage-authority-decision.md): scanner usage remains hard-quota authority. A future scanner storage seam must therefore model the concrete publication, cycle-lock, usage-floor, observed-snapshot, and recovery-marker capabilities described in [scanner-usage-publication.md](scanner-usage-publication.md), not a generic key-value abstraction.
|
||||||
|
|
||||||
## Loss-Prevention Coverage
|
## Loss-Prevention Coverage
|
||||||
|
|
||||||
The guard pins specific public re-export lines (its `require_source_line` entries) so contract surfaces cannot silently disappear during cleanup. The canonical lists are the guard script and the owning files, not this page:
|
The guard pins specific public re-export lines (its `require_source_line` entries) so contract surfaces cannot silently disappear during cleanup. The canonical lists are the guard script and the owning files, not this page:
|
||||||
|
|||||||
@@ -3,6 +3,8 @@
|
|||||||
**Use this when:** changing heal, PUT/multipart commit, delete, lifecycle expiry, or data-movement code that touches the same `(bucket, object)` commit surface; or evaluating whether RustFS needs a persistent per-object healing marker like MinIO's `x-minio-healing`.
|
**Use this when:** changing heal, PUT/multipart commit, delete, lifecycle expiry, or data-movement code that touches the same `(bucket, object)` commit surface; or evaluating whether RustFS needs a persistent per-object healing marker like MinIO's `x-minio-healing`.
|
||||||
**Source of truth:** `crates/ecstore/src/set_disk/ops/heal.rs` (`heal_object_with_explicit_version_regen`, `HealObjectLockKind`, `HEAL_RENAME_INCOMPLETE`), `crates/ecstore/src/set_disk/ops/object.rs` (PUT/DELETE lock sections, `reconcile_old_data_cleanup_receipts`), `crates/ecstore/src/set_disk/core/io_primitives.rs` (`commit_rename_data_dir`, `report_old_data_dir_cleanup`, `reclaim_orphan_data_dirs`), `crates/filemeta/src/fileinfo.rs` (`FileInfo::set_healing`), `crates/heal/src/heal/manager/queue.rs` (dedup keys).
|
**Source of truth:** `crates/ecstore/src/set_disk/ops/heal.rs` (`heal_object_with_explicit_version_regen`, `HealObjectLockKind`, `HEAL_RENAME_INCOMPLETE`), `crates/ecstore/src/set_disk/ops/object.rs` (PUT/DELETE lock sections, `reconcile_old_data_cleanup_receipts`), `crates/ecstore/src/set_disk/core/io_primitives.rs` (`commit_rename_data_dir`, `report_old_data_dir_cleanup`, `reclaim_orphan_data_dirs`), `crates/filemeta/src/fileinfo.rs` (`FileInfo::set_healing`), `crates/heal/src/heal/manager/queue.rs` (dedup keys).
|
||||||
|
|
||||||
|
For crate ownership, read [crate-boundaries.md](crate-boundaries.md): ECStore owns erasure-set repair primitives that share this lock and commit model, while `crates/heal` owns repair orchestration.
|
||||||
|
|
||||||
## Model
|
## Model
|
||||||
|
|
||||||
Heal and every foreground or background write path serialize on the same object-level namespace write lock (a quorum lock RPC in distributed mode, the in-process lock manager on a single node; granularity is the object, the version component is always `None`), and heal holds its guard across the whole rename commit. MinIO's `x-minio-healing` marker is an out-of-lock defence against version-cleanup logic inside `RenameData` interleaving with a heal commit; RustFS's commit model has no such interleaving, so no persistent marker exists (`x-minio-healing` does not occur in `crates/` or `rustfs/`) and none is needed. Three layers replace it:
|
Heal and every foreground or background write path serialize on the same object-level namespace write lock (a quorum lock RPC in distributed mode, the in-process lock manager on a single node; granularity is the object, the version component is always `None`), and heal holds its guard across the whole rename commit. MinIO's `x-minio-healing` marker is an out-of-lock defence against version-cleanup logic inside `RenameData` interleaving with a heal commit; RustFS's commit model has no such interleaving, so no persistent marker exists (`x-minio-healing` does not occur in `crates/` or `rustfs/`) and none is needed. Three layers replace it:
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# Scanner Usage Authority Decision
|
||||||
|
|
||||||
|
**Use this when:** deciding whether quota admission depends on scanner data usage, removing scanner publication layers, or designing a scanner storage boundary.
|
||||||
|
**Source of truth:** [scanner-usage-publication.md](scanner-usage-publication.md), `crates/ecstore/src/bucket/quota/checker.rs`, and the scanner publication state under `crates/scanner/src/scanner/`.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Date: 2026-09-03
|
||||||
|
|
||||||
|
RustFS keeps scanner data usage as authoritative for quota admission.
|
||||||
|
|
||||||
|
This selects option A from the backlog decision record: the scanner publication protocol remains necessary while quota admission consumes scanner usage. The cycle epoch, publication CAS, data-movement fence, tier-registry fence, observed snapshot layer, and persisted usage floor are retained and documented as protocol invariants rather than treated as removable compatibility clutter.
|
||||||
|
|
||||||
|
## Rationale
|
||||||
|
|
||||||
|
Quota is a write-path admission decision, so serving quota from best-effort scanner data would turn temporary scanner lag into under-enforcement. The current design therefore needs an availability story for authoritative usage instead of deleting the proof layers that make it authoritative.
|
||||||
|
|
||||||
|
The scanner usage floor provides that availability story. It is a lower bound used when a complete authoritative snapshot is not available, including cold startup, upgrade recovery, and incomplete-cycle repair. Observed snapshots remain useful for admin and observability, but they do not become quota authority.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
#2214 is the hard design input for future usage-publication changes. A future proposal may still choose soft quota and MinIO-style best-effort usage, but that would be a product change with its own staged compatibility plan for persisted artifacts.
|
||||||
|
|
||||||
|
#2219 may design the scanner storage boundary against the current authoritative protocol. The interface must include the CAS key-value, cycle lock, usage-floor, observed-snapshot, and recovery-marker capabilities needed by [scanner-usage-publication.md](scanner-usage-publication.md); it must not hide those proof obligations behind a generic object-store trait.
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
# Scanner Usage Publication Contract
|
||||||
|
|
||||||
|
**Use this when:** changing scanner data-usage persistence, quota-visible usage snapshots, scanner cycle state, dirty-usage catch-up, or the conditions under which an observed scanner snapshot may be served.
|
||||||
|
**Source of truth:** `crates/scanner/src/scanner/usage_store.rs`, `crates/scanner/src/scanner/cycle_state.rs`, `crates/scanner/src/scanner/backlog.rs`, `crates/scanner/src/scanner/leadership.rs`, `crates/scanner/src/data_usage_define.rs`, and quota fallback behavior in `crates/ecstore/src/bucket/quota/checker.rs`.
|
||||||
|
|
||||||
|
## Ownership Model
|
||||||
|
|
||||||
|
One scanner cycle owns an authoritative publication only after it holds the
|
||||||
|
cluster scanner leadership claim and proves that the storage publication epoch
|
||||||
|
has not moved. The leadership claim is persisted in the scanner cycle state,
|
||||||
|
while data-usage publication admission is owned by ECStore because it knows
|
||||||
|
whether rebalance, decommission, or another data-movement operation has changed
|
||||||
|
the generation that scanner results are allowed to describe.
|
||||||
|
|
||||||
|
The scanner may compute usage without publication ownership, but it must not
|
||||||
|
turn that result into authoritative quota-visible state. A complete publication
|
||||||
|
therefore has three identities:
|
||||||
|
|
||||||
|
- the scanner leader epoch that owns the cycle;
|
||||||
|
- the storage publication epoch that fences data movement;
|
||||||
|
- the per-object CAS revision on the usage object being replaced.
|
||||||
|
|
||||||
|
If any identity changes before commit, the result is a candidate for retry or
|
||||||
|
observation, not an authoritative baseline.
|
||||||
|
|
||||||
|
## Fences
|
||||||
|
|
||||||
|
The protocol uses separate fences because they exclude different stale inputs.
|
||||||
|
They must not be collapsed unless the replacement proves the same exclusions.
|
||||||
|
|
||||||
|
| Fence | Owner | Excludes |
|
||||||
|
|---|---|---|
|
||||||
|
| Scanner leadership claim | scanner | competing scanner leaders and stale cycle writers |
|
||||||
|
| Storage publication epoch | ECStore | usage computed across rebalance, decommission, or other data-movement generations |
|
||||||
|
| Publication lease | scanner peers through ECStore-facing activity probes | remote dirty-usage or maintenance state that has not acknowledged the candidate |
|
||||||
|
| CAS revision | backing config object store | lost updates to `.usage.v2.json`, `.usage.json`, or cycle-state objects |
|
||||||
|
| Per-set freshness | scanner aggregation | a merged usage snapshot that combines stale and current set results |
|
||||||
|
| Tier registry generation | scanner tier accounting | bytes classified against a different warm-tier registry |
|
||||||
|
| Usage floor identity | scanner publication and ECStore quota fallback | empty or legacy values becoming plausible authoritative quota input |
|
||||||
|
|
||||||
|
A reader that cannot prove the required fence for its surface must fail closed
|
||||||
|
or use the documented observed path below. It must not synthesize an empty usage
|
||||||
|
snapshot for a missing or corrupt authoritative object.
|
||||||
|
|
||||||
|
## Persisted Objects
|
||||||
|
|
||||||
|
The persisted objects are part of the compatibility contract. Removing one
|
||||||
|
requires a compatibility window and a dedicated cleanup entry.
|
||||||
|
|
||||||
|
| Object | Owner | Lifecycle |
|
||||||
|
|---|---|---|
|
||||||
|
| `.usage-cache.bin` under each bucket and set | scanner disk walk | Rebuilt by scanner from object metadata. Missing data causes a rescan for that bucket/set; corrupt data is not a complete baseline. |
|
||||||
|
| `.bloomcycle.bin` | scanner cycle state | CAS-updated by the leader. Missing state starts from an uninitialized cycle; corrupt or future state is quarantined before automatic retry. |
|
||||||
|
| `.usage.v2.json` and `.usage.json` | scanner authoritative publication | `.usage.v2.json` is the primary complete usage snapshot. `.usage.json` is read only as a legacy or companion baseline when it carries a valid persisted identity. Neither bypasses the v2 epoch fence, and readers may treat a snapshot as authoritative only when its baseline identity and completion fields validate. |
|
||||||
|
| `.usage.observed.json` | scanner observation path | Written when an authoritative publication cannot be proven but a diagnostic snapshot is still useful. It is never a hard-quota authority. |
|
||||||
|
| `bucket-metadata/.usage.json` | scanner usage floor | Carries the persisted per-bucket floor used by quota during a degraded authoritative-usage window. It is static until the next complete scanner publication. |
|
||||||
|
| `.bloomcycle.bin.recovery-required.json` | scanner cycle recovery | Quarantines invalid cycle state with retry evidence. Only scanner recovery code updates or clears it. |
|
||||||
|
| `.scanner-cycle.lock` | scanner runtime lock | Serializes cycle-level work. A missing lock object is not itself usage evidence. |
|
||||||
|
| `.scanner-pause-backlog.json` | scanner pause and catch-up ledger | Tracks dirty usage, discovered lifecycle work, and full-scan catch-up while authoritative publication is fenced by data movement. It never grants publication admission. |
|
||||||
|
|
||||||
|
## Observed Snapshots
|
||||||
|
|
||||||
|
Observed snapshots are a diagnostic and availability layer. They may be served
|
||||||
|
only when the snapshot explicitly reports that it is partial or observational,
|
||||||
|
and only to consumers that do not make hard quota, durability, or deletion
|
||||||
|
decisions from it. Admin usage views may expose this state with completeness
|
||||||
|
flags so operators can see progress while the authoritative publication is
|
||||||
|
blocked. Quota enforcement must not use an observed snapshot as the current
|
||||||
|
usage authority.
|
||||||
|
|
||||||
|
When an authoritative snapshot is unavailable, quota admission may use the
|
||||||
|
persisted usage floor. That is an availability fallback, not a fresh count: live
|
||||||
|
writes do not advance the floor, and overrun is bounded only by writes accepted
|
||||||
|
before the next complete scanner publication. If no valid persisted floor is
|
||||||
|
available, quota remains unavailable and fails closed.
|
||||||
|
|
||||||
|
## Availability Decision
|
||||||
|
|
||||||
|
Decision date: 2026-09-03.
|
||||||
|
|
||||||
|
RustFS keeps scanner usage as the authority for hard quota admission. The
|
||||||
|
publication protocol therefore remains necessary: leadership, storage epoch,
|
||||||
|
lease, CAS, observed snapshot, and usage-floor layers are the proof machinery
|
||||||
|
that lets a distributed background scan feed a quota decision without accepting
|
||||||
|
stale or cross-generation usage as current truth.
|
||||||
|
|
||||||
|
The availability contract is:
|
||||||
|
|
||||||
|
- the authoritative fast path reads complete in-memory or persisted scanner
|
||||||
|
usage;
|
||||||
|
- during upgrade or publication outage, quota may admit against the persisted
|
||||||
|
per-bucket usage floor;
|
||||||
|
- the floor is advisory for the outage window and must converge back to a
|
||||||
|
complete scanner publication;
|
||||||
|
- a bucket with neither authoritative usage nor a valid floor fails closed.
|
||||||
|
|
||||||
|
Changing this decision to a soft-quota model would be a product change, not a
|
||||||
|
scanner refactor. It would need a staged removal of the authority-specific
|
||||||
|
layers and compatibility handling for the persisted objects above.
|
||||||
|
|
||||||
|
## Deletion And Recovery Rules
|
||||||
|
|
||||||
|
Only the owner of an object may delete or quarantine it:
|
||||||
|
|
||||||
|
- scanner may rebuild per-set `.usage-cache.bin` after a scan proves the
|
||||||
|
replacement contents;
|
||||||
|
- scanner cycle recovery may quarantine invalid `.bloomcycle.bin` and clear the
|
||||||
|
marker only after a valid cycle state is persisted;
|
||||||
|
- scanner publication may replace `.usage.v2.json` or legacy companions only
|
||||||
|
through the publication fences above;
|
||||||
|
- quota consumers may read the usage floor but must not delete or repair it;
|
||||||
|
- operators may reset scanner usage state only through the supported scanner
|
||||||
|
reset surface, which records the reset paths and forces a full rebuild.
|
||||||
|
|
||||||
|
Missing, undecodable, or identity-less data is not converted to zero. It is
|
||||||
|
reported as uninitialized, recovery-required, observed-only, or unavailable
|
||||||
|
according to the reader's surface.
|
||||||
|
|
||||||
|
## Existing Fixes As Invariants
|
||||||
|
|
||||||
|
Several prior scanner fixes are consequences of this contract rather than
|
||||||
|
standalone patches:
|
||||||
|
|
||||||
|
- incomplete scanner usage must not become a complete admin or quota baseline,
|
||||||
|
because completeness and floor identity are part of publication ownership;
|
||||||
|
- dirty usage and maintenance acknowledgements must fence publication, because
|
||||||
|
a remote node with unacknowledged work can invalidate the candidate;
|
||||||
|
- a legacy or backup usage object may help recover availability only when it
|
||||||
|
carries a valid baseline identity and does not cross the primary epoch fence.
|
||||||
@@ -12,9 +12,9 @@
|
|||||||
| Class | Provider (`impl WorkloadAdmissionSnapshotProvider`) | `active` / `queued` / `limit` source | Reports `Unknown` when |
|
| Class | Provider (`impl WorkloadAdmissionSnapshotProvider`) | `active` / `queued` / `limit` source | Reports `Unknown` when |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| `ForegroundRead` | `ConcurrencyManager` in `rustfs/src/storage/concurrency/manager.rs` (source of truth); re-exposed unchanged by the RustFS runtime provider | disk-read permits in use / `None` (the semaphore exposes no waiter count) / configured max concurrent disk reads | the storage registry has no entry |
|
| `ForegroundRead` | `ConcurrencyManager` in `rustfs/src/storage/concurrency/manager.rs` (source of truth); re-exposed unchanged by the RustFS runtime provider | disk-read permits in use / `None` (the semaphore exposes no waiter count) / configured max concurrent disk reads | the storage registry has no entry |
|
||||||
| `ForegroundWrite` | none | none | always: no write-specific admission owner exposes a read-only surface yet |
|
| `ForegroundWrite` | `ConcurrencyManager` in `rustfs/src/storage/concurrency/manager.rs` (source of truth); re-exposed unchanged by the RustFS runtime provider | foreground-write permits in use or legacy active-write counter / `None` / configured or derived write-admission limit | the storage registry has no entry |
|
||||||
| `Metadata` | `RustFsWorkloadAdmissionSnapshotProvider` in `rustfs/src/workload_admission.rs` | `Open` once the bucket metadata runtime handle exists; no counts | bucket metadata runtime not initialized |
|
| `Metadata` | `RustFsWorkloadAdmissionSnapshotProvider` in `rustfs/src/workload_admission.rs` | `Open` once the bucket metadata runtime handle exists; no counts | bucket metadata runtime not initialized |
|
||||||
| `Scanner` | same | scanner active work-unit counter / none / none | the counter is zero (idle and uninitialized are indistinguishable) |
|
| `Scanner` | same | scanner active work-unit counter / none / configured set-scan limit when nonzero | scanner runtime not initialized |
|
||||||
| `Repair` | same | heal active tasks / heal queue length / `None` (limits live behind the async heal manager state) | heal manager not initialized |
|
| `Repair` | same | heal active tasks / heal queue length / `None` (limits live behind the async heal manager state) | heal manager not initialized |
|
||||||
| `Replication` | same | active regular + large-object + MRF workers / site replication queue count / `None` (limits owned by the async pool and resize policy) | replication runtime not initialized, or queue stats currently locked |
|
| `Replication` | same | active regular + large-object + MRF workers / site replication queue count / `None` (limits owned by the async pool and resize policy) | replication runtime not initialized, or queue stats currently locked |
|
||||||
|
|
||||||
@@ -28,6 +28,7 @@ Consumers that read the snapshot to self-throttle exist, and they do not change
|
|||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Data-movement backpressure (decommission, rebalance) | `crates/ecstore/src/data_movement/backpressure.rs` (`wait_for_data_movement_admission`, `foreground_pressure`) | Delays the next data-movement step while `ForegroundRead` or `ForegroundWrite` usage exceeds the configured high-water percent. ECStore receives the provider through `set_workload_admission_snapshot_provider` (`crates/ecstore/src/lib.rs`), published from `rustfs/src/startup_background.rs`; with no provider the step is admitted immediately. |
|
| Data-movement backpressure (decommission, rebalance) | `crates/ecstore/src/data_movement/backpressure.rs` (`wait_for_data_movement_admission`, `foreground_pressure`) | Delays the next data-movement step while `ForegroundRead` or `ForegroundWrite` usage exceeds the configured high-water percent. ECStore receives the provider through `set_workload_admission_snapshot_provider` (`crates/ecstore/src/lib.rs`), published from `rustfs/src/startup_background.rs`; with no provider the step is admitted immediately. |
|
||||||
| Heal manager mainline throttle | `crates/heal/src/heal/manager.rs` (`new_with_workload_provider`) | When `mainline_throttle_enable` is set, defers heal work while `ForegroundRead` or `ForegroundWrite` utilization exceeds the configured high-water percents; with no provider or the throttle disabled, heal pacing is unchanged. |
|
| Heal manager mainline throttle | `crates/heal/src/heal/manager.rs` (`new_with_workload_provider`) | When `mainline_throttle_enable` is set, defers heal work while `ForegroundRead` or `ForegroundWrite` utilization exceeds the configured high-water percents; with no provider or the throttle disabled, heal pacing is unchanged. |
|
||||||
|
| Scanner sleeper and scan fan-out | `crates/scanner/src/workload_admission.rs`, `crates/scanner/src/sleeper.rs`, and `crates/scanner/src/scanner_io/guards.rs` | Reads the same provider published from `rustfs/src/startup_background.rs` and combines it with scanner-local foreground read guards. Foreground activity increases scanner sleeps and reduces set/disk scan fan-out to one; cycle budgets still own object, directory, and duration limits. With no provider, scanner keeps the legacy local foreground-read behavior. |
|
||||||
|
|
||||||
## Boundary Rules
|
## Boundary Rules
|
||||||
|
|
||||||
|
|||||||
@@ -57,6 +57,14 @@ Promotion rule: never promote a report-only lane to required from one green run.
|
|||||||
|
|
||||||
e2e filters live in `.config/nextest.toml`; extend a profile instead of adding a second selector. Before a profile runs, `scripts/check_test_wiring.py` compares its listing to the committed digest in `.config/e2e-<profile>-selection.txt`, so a silent test drop fails closed.
|
e2e filters live in `.config/nextest.toml`; extend a profile instead of adding a second selector. Before a profile runs, `scripts/check_test_wiring.py` compares its listing to the committed digest in `.config/e2e-<profile>-selection.txt`, so a silent test drop fails closed.
|
||||||
|
|
||||||
|
Scanner usage and heal rebuild coverage are intentionally split by risk and
|
||||||
|
cost. `data_usage_test` runs in the PR `e2e-smoke` lane so changes that affect
|
||||||
|
authoritative scanner usage publication, quota-visible usage, or admin usage
|
||||||
|
snapshots get an end-to-end signal before merge review. `heal_erasure_disk_rebuild_test`
|
||||||
|
runs in `e2e-full` so core erasure heal rebuild regressions are caught no later
|
||||||
|
than the merge queue or `main` push lane; it also remains in `e2e-nightly` with
|
||||||
|
the serialized cluster fault-domain suites for scheduled soak signal.
|
||||||
|
|
||||||
## Scheduled validation
|
## Scheduled validation
|
||||||
|
|
||||||
Scheduled lanes never block a PR. Their workflow-local gate fails the run, scheduled failures route to the shared failure-issue action, and `scheduled-validation-freshness.yml` fails when a workflow listed in `.github/scheduled-validations.json` has not run within its `max_age_hours` (a `never_ran_grace_until` entry covers the window before a newly enabled cron's first slot). Cadence is qualitative here; the cron lives in each workflow's `on.schedule`.
|
Scheduled lanes never block a PR. Their workflow-local gate fails the run, scheduled failures route to the shared failure-issue action, and `scheduled-validation-freshness.yml` fails when a workflow listed in `.github/scheduled-validations.json` has not run within its `max_age_hours` (a `never_ran_grace_until` entry covers the window before a newly enabled cron's first slot). Cadence is qualitative here; the cron lives in each workflow's `on.schedule`.
|
||||||
|
|||||||
+1
-1
@@ -227,7 +227,7 @@ rustfs-heal = { workspace = true }
|
|||||||
rustfs-audit = { workspace = true }
|
rustfs-audit = { workspace = true }
|
||||||
rustfs-common = { workspace = true }
|
rustfs-common = { workspace = true }
|
||||||
rustfs-heal-contracts = { workspace = true }
|
rustfs-heal-contracts = { workspace = true }
|
||||||
rustfs-scanner-contracts = { workspace = true }
|
rustfs-scanner-metrics = { workspace = true }
|
||||||
rustfs-config = { workspace = true, features = ["notify", "server-config-model"] }
|
rustfs-config = { workspace = true, features = ["notify", "server-config-model"] }
|
||||||
rustfs-crypto = { workspace = true }
|
rustfs-crypto = { workspace = true }
|
||||||
rustfs-credentials = { workspace = true }
|
rustfs-credentials = { workspace = true }
|
||||||
|
|||||||
@@ -29,10 +29,8 @@ use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
|
|||||||
use rustfs_credentials::Credentials;
|
use rustfs_credentials::Credentials;
|
||||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use rustfs_scanner_contracts::metrics::ScannerLifecycleTransitionSnapshot;
|
use rustfs_scanner_metrics::metrics::ScannerLifecycleTransitionSnapshot;
|
||||||
use rustfs_scanner_contracts::metrics::{
|
use rustfs_scanner_metrics::metrics::{ScannerLifecycleExpirySnapshot, ScannerMaintenanceControlSnapshot, ScannerMetricsReport};
|
||||||
ScannerLifecycleExpirySnapshot, ScannerMaintenanceControlSnapshot, ScannerMetricsReport,
|
|
||||||
};
|
|
||||||
use s3s::header::CONTENT_TYPE;
|
use s3s::header::CONTENT_TYPE;
|
||||||
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|||||||
@@ -125,7 +125,7 @@ pub(crate) mod runtime_sources {
|
|||||||
pub(crate) type DailyAllTierStats = super::DailyAllTierStats;
|
pub(crate) type DailyAllTierStats = super::DailyAllTierStats;
|
||||||
pub(crate) type ECStore = super::ECStore;
|
pub(crate) type ECStore = super::ECStore;
|
||||||
pub(crate) type NotificationSys = super::NotificationSys;
|
pub(crate) type NotificationSys = super::NotificationSys;
|
||||||
pub(crate) type ScannerMetricsReport = rustfs_scanner_contracts::metrics::ScannerMetricsReport;
|
pub(crate) type ScannerMetricsReport = rustfs_scanner_metrics::metrics::ScannerMetricsReport;
|
||||||
pub(crate) type StorageClassConfig = crate::storage::storage_api::ecstore_config::storageclass::Config;
|
pub(crate) type StorageClassConfig = crate::storage::storage_api::ecstore_config::storageclass::Config;
|
||||||
pub(crate) type TierConfigMgr = crate::storage::storage_api::TierConfigMgr;
|
pub(crate) type TierConfigMgr = crate::storage::storage_api::TierConfigMgr;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -148,7 +148,7 @@ pub(crate) mod runtime {
|
|||||||
pub(crate) type NotificationSys = crate::storage::storage_api::NotificationSys;
|
pub(crate) type NotificationSys = crate::storage::storage_api::NotificationSys;
|
||||||
pub(crate) type ObjectStoreResolver = crate::storage::storage_api::ObjectStoreResolver;
|
pub(crate) type ObjectStoreResolver = crate::storage::storage_api::ObjectStoreResolver;
|
||||||
pub(crate) type ReplicationStats = crate::storage::storage_api::ReplicationStats;
|
pub(crate) type ReplicationStats = crate::storage::storage_api::ReplicationStats;
|
||||||
pub(crate) type ScannerMetricsReport = rustfs_scanner_contracts::metrics::ScannerMetricsReport;
|
pub(crate) type ScannerMetricsReport = rustfs_scanner_metrics::metrics::ScannerMetricsReport;
|
||||||
pub(crate) type StorageClassConfig = crate::storage::storage_api::ecstore_config::storageclass::Config;
|
pub(crate) type StorageClassConfig = crate::storage::storage_api::ecstore_config::storageclass::Config;
|
||||||
pub(crate) type TierConfigMgr = crate::storage::storage_api::TierConfigMgr;
|
pub(crate) type TierConfigMgr = crate::storage::storage_api::TierConfigMgr;
|
||||||
pub(crate) type TransitionState = crate::storage::storage_api::TransitionState;
|
pub(crate) type TransitionState = crate::storage::storage_api::TransitionState;
|
||||||
@@ -224,7 +224,7 @@ pub(crate) mod runtime {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn collect_scanner_metrics_report() -> ScannerMetricsReport {
|
pub(crate) async fn collect_scanner_metrics_report() -> ScannerMetricsReport {
|
||||||
rustfs_scanner_contracts::metrics::global_metrics().report().await
|
rustfs_scanner_metrics::metrics::global_metrics().report().await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ use crate::storage_api::cluster::control_plane::{
|
|||||||
use crate::workload_admission::workload_admission_registry_snapshot;
|
use crate::workload_admission::workload_admission_registry_snapshot;
|
||||||
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot};
|
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot};
|
||||||
use rustfs_io_metrics::internode_metrics::{InternodeMetricsSnapshot, global_internode_metrics};
|
use rustfs_io_metrics::internode_metrics::{InternodeMetricsSnapshot, global_internode_metrics};
|
||||||
use rustfs_scanner_contracts::metrics::{ScannerMetricsReport, global_metrics};
|
use rustfs_scanner_metrics::metrics::{ScannerMetricsReport, global_metrics};
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub struct ClusterReadOnlySnapshot {
|
pub struct ClusterReadOnlySnapshot {
|
||||||
|
|||||||
@@ -59,6 +59,7 @@ pub(crate) async fn init_background_service_runtime(store: Arc<ECStore>) -> Resu
|
|||||||
let workload_provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync> =
|
let workload_provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync> =
|
||||||
Arc::new(RustFsWorkloadAdmissionSnapshotProvider);
|
Arc::new(RustFsWorkloadAdmissionSnapshotProvider);
|
||||||
let _ = set_workload_admission_snapshot_provider(workload_provider.clone());
|
let _ = set_workload_admission_snapshot_provider(workload_provider.clone());
|
||||||
|
rustfs_scanner::set_scanner_workload_admission_snapshot_provider(workload_provider.clone());
|
||||||
|
|
||||||
if enable_heal || enable_scanner {
|
if enable_heal || enable_scanner {
|
||||||
let heal_storage = Arc::new(ECStoreHealStorage::new(store.clone()));
|
let heal_storage = Arc::new(ECStoreHealStorage::new(store.clone()));
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ pub(crate) async fn publish_server_addr(addr: &str) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn publish_init_time_now() {
|
pub(crate) async fn publish_init_time_now() {
|
||||||
rustfs_scanner_contracts::set_global_init_time_now().await;
|
rustfs_scanner_metrics::set_global_init_time_now().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn init_kms_service_manager() -> Arc<KmsServiceManager> {
|
pub(crate) fn init_kms_service_manager() -> Arc<KmsServiceManager> {
|
||||||
|
|||||||
Reference in New Issue
Block a user