mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-16 09:58:21 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 77b712dcb6 |
@@ -60,11 +60,6 @@ body-cache-whitelist-check: ## Check the body-cache eligibility gate stays a fai
|
||||
@echo "🧱 Checking body-cache whitelist guard..."
|
||||
./scripts/check_body_cache_whitelist.sh
|
||||
|
||||
.PHONY: fips-wording-check
|
||||
fips-wording-check: ## Check outward docs do not make unsupported FIPS claims
|
||||
@echo "📣 Checking FIPS wording guard..."
|
||||
./scripts/check_fips_wording.sh
|
||||
|
||||
.PHONY: log-analyzer-rules-check
|
||||
log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source
|
||||
@echo "🩺 Checking log-analyzer rule anchors..."
|
||||
|
||||
@@ -19,13 +19,13 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check fips-wording-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||
@echo "✅ Fast development checks passed!"
|
||||
|
||||
+3
-16
@@ -9,8 +9,6 @@
|
||||
# * bucket_lifecycle_ops::tests::concurrent_resend_same_part_commits_one_generation
|
||||
# uses the shared multipart fixture and a deterministic uploadId-lock
|
||||
# handoff, so it must not overlap another process mutating that fixture.
|
||||
# * bucket::metadata_sys::tests::concurrent_config_writes_from_separate_nodes_do_not_lose_writes
|
||||
# uses the shared transaction lock and must not overlap other ecstore tests.
|
||||
#
|
||||
# serial_test's #[serial] attribute does NOT serialize these across runs:
|
||||
# nextest executes each test in its own process, where the in-process
|
||||
@@ -42,7 +40,7 @@ e2e-inline-boundaries = { max-threads = 1 }
|
||||
|
||||
# --- default profile (local): serialize the flaky groups, never retry --------
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & (test(concurrent_resend_same_part_commits_one_generation) | test(concurrent_config_writes_from_separate_nodes_do_not_lose_writes) | test(/^store::bucket::tests::bucket_delete_(mark_delete|purge_removes|default_s3_delete)/))'
|
||||
filter = 'package(rustfs-ecstore) & (test(concurrent_resend_same_part_commits_one_generation) | test(/^store::bucket::tests::bucket_delete_(mark_delete|purge_removes|default_s3_delete)/))'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Serialize the multipart crash-consistency scenarios (dist-2, backlog#1150):
|
||||
@@ -106,9 +104,9 @@ filter = 'package(rustfs-ecstore) & test(/^store::bucket::tests::bucket_delete_(
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
retries = 2
|
||||
|
||||
# Keep deterministic ECStore write handoffs isolated across nextest processes.
|
||||
# Keep the deterministic multipart handoff isolated across nextest processes.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & (test(concurrent_resend_same_part_commits_one_generation) | test(concurrent_config_writes_from_separate_nodes_do_not_lose_writes))'
|
||||
filter = 'package(rustfs-ecstore) & test(concurrent_resend_same_part_commits_one_generation)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# QUARANTINE: OPEN rustfs#4690 — walk_dir stall-budget accounting test depends
|
||||
@@ -214,17 +212,6 @@ default-filter = """
|
||||
"""
|
||||
fail-fast = false
|
||||
|
||||
[profile.e2e-smoke.junit]
|
||||
path = "junit.xml"
|
||||
|
||||
# The pagination boundary cases can stall when a server/listing regression
|
||||
# prevents the continuation request from completing. Keep the timeout scoped
|
||||
# to those known failure modes so legitimate lifecycle/tiering waits retain
|
||||
# their test-level timing budget.
|
||||
[[profile.e2e-smoke.overrides]]
|
||||
filter = 'package(e2e_test) & test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/)'
|
||||
slow-timeout = { period = "60s", terminate-after = 2, grace-period = "10s" }
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-repl-nightly profile — scheduled full replication e2e lane (repl-1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -11500,831 +11500,6 @@
|
||||
],
|
||||
"title": "Compression Operations Rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 332
|
||||
},
|
||||
"id": 531,
|
||||
"panels": [],
|
||||
"title": "Metrics Dimensions Drilldown",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 333
|
||||
},
|
||||
"id": 532,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "sum by (server, name, type) (rate(rustfs_api_requests_requests_total_by_server{job=~\"$job\",server=~\"$server\",name=~\"$api\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server}} | {{name}} | {{type}}"
|
||||
}
|
||||
],
|
||||
"title": "API Requests by Server and API",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byFrameRefID",
|
||||
"options": "A"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "none"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 333
|
||||
},
|
||||
"id": 533,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "max by (server, drive, pool_index, set_index, drive_index, state) (rustfs_system_drive_runtime_state{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})",
|
||||
"legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{state}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"expr": "max by (server, drive, pool_index, set_index, drive_index) (rustfs_system_drive_offline_duration_seconds{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})",
|
||||
"legendFormat": "{{server}} | {{drive}} | offline seconds"
|
||||
}
|
||||
],
|
||||
"title": "Drive Runtime State and Offline Duration",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 341
|
||||
},
|
||||
"id": 534,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "sum by (server, drive, pool_index, set_index, drive_index, api) (rate(rustfs_system_drive_api_calls_total{job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{api}}"
|
||||
}
|
||||
],
|
||||
"title": "Drive API Calls by Operation",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byFrameRefID",
|
||||
"options": "B"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "short"
|
||||
},
|
||||
{
|
||||
"id": "custom.axisPlacement",
|
||||
"value": "right"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 341
|
||||
},
|
||||
"id": 535,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "sum by (server, source, state) (rate(rustfs_scanner_source_work_total{job=~\"$job\",server=~\"$server\",source=~\"$scanner_source\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server}} | {{source}} | {{state}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"expr": "sum by (server, cycle_scope, source, state) (rustfs_scanner_cycle_source_work{job=~\"$job\",server=~\"$server\",source=~\"$scanner_source\"})",
|
||||
"legendFormat": "{{server}} | {{cycle_scope}} | {{source}} | {{state}}"
|
||||
}
|
||||
],
|
||||
"title": "Scanner Source Work by Server",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byFrameRefID",
|
||||
"options": "B"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "short"
|
||||
},
|
||||
{
|
||||
"id": "custom.axisPlacement",
|
||||
"value": "right"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 349
|
||||
},
|
||||
"id": 536,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "sum by (server, bucket, drive, result) (rate(rustfs_scanner_bucket_drive_result_total{job=~\"$job\",server=~\"$server\",bucket=~\"$bucket\",drive=~\"$drive\",result=~\"$scanner_result\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server}} | {{bucket}} | {{drive}} | {{result}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"expr": "sum by (server, cycle_scope, bucket, drive, result) (rustfs_scanner_cycle_bucket_drive_result{job=~\"$job\",server=~\"$server\",bucket=~\"$bucket\",drive=~\"$drive\",result=~\"$scanner_result\"})",
|
||||
"legendFormat": "{{server}} | {{cycle_scope}} | {{bucket}} | {{drive}} | {{result}}"
|
||||
}
|
||||
],
|
||||
"title": "Scanner Bucket Drive Results",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byFrameRefID",
|
||||
"options": "B"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "unit",
|
||||
"value": "Bps"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 349
|
||||
},
|
||||
"id": 537,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "sum by (bucket, target_arn) (rate(rustfs_bucket_replication_target_sent_count{job=~\"$job\",bucket=~\"$bucket\",target_arn=~\"$target_arn\"}[$__rate_interval]))",
|
||||
"legendFormat": "sent objects | {{bucket}} | {{target_arn}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"expr": "sum by (bucket, target_arn) (rate(rustfs_bucket_replication_target_sent_bytes{job=~\"$job\",bucket=~\"$bucket\",target_arn=~\"$target_arn\"}[$__rate_interval]))",
|
||||
"legendFormat": "sent bytes | {{bucket}} | {{target_arn}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "C",
|
||||
"expr": "sum by (bucket, target_arn) (rate(rustfs_bucket_replication_target_total_failed_count{job=~\"$job\",bucket=~\"$bucket\",target_arn=~\"$target_arn\"}[$__rate_interval]))",
|
||||
"legendFormat": "failed objects | {{bucket}} | {{target_arn}}"
|
||||
}
|
||||
],
|
||||
"title": "Bucket Replication Target Flow",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 357
|
||||
},
|
||||
"id": 538,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "12.3.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "max by (server, target_id) (rustfs_audit_target_queue_length_by_server{job=~\"$job\",server=~\"$server\"})",
|
||||
"legendFormat": "audit queue | {{server}} | {{target_id}}"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"expr": "max by (server, action, state) (rustfs_ilm_action_tasks{job=~\"$job\",server=~\"$server\"})",
|
||||
"legendFormat": "ilm | {{server}} | {{action}} | {{state}}"
|
||||
}
|
||||
],
|
||||
"title": "Audit and ILM by Server",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"preload": false,
|
||||
@@ -12376,32 +11551,6 @@
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values(rustfs_system_drive_api_calls_total,api)",
|
||||
"includeAll": true,
|
||||
"label": "Drive API",
|
||||
"multi": true,
|
||||
"name": "drive_api",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(rustfs_system_drive_api_calls_total,api)",
|
||||
"refId": "PrometheusVariableQueryEditor-drive_api"
|
||||
},
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
@@ -12521,136 +11670,6 @@
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values(rustfs_api_requests_requests_total_by_server,server)",
|
||||
"includeAll": true,
|
||||
"label": "Server",
|
||||
"multi": true,
|
||||
"name": "server",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(rustfs_api_requests_requests_total_by_server,server)",
|
||||
"refId": "PrometheusVariableQueryEditor-server"
|
||||
},
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values(rustfs_api_requests_requests_total_by_server,name)",
|
||||
"includeAll": true,
|
||||
"label": "API",
|
||||
"multi": true,
|
||||
"name": "api",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(rustfs_api_requests_requests_total_by_server,name)",
|
||||
"refId": "PrometheusVariableQueryEditor-api"
|
||||
},
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values({__name__=\"rustfs_bucket_replication_target_sent_count\",bucket=~\"$bucket\"},target_arn)",
|
||||
"includeAll": true,
|
||||
"label": "Target ARN",
|
||||
"multi": true,
|
||||
"name": "target_arn",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values({__name__=\"rustfs_bucket_replication_target_sent_count\",bucket=~\"$bucket\"},target_arn)",
|
||||
"refId": "PrometheusVariableQueryEditor-target_arn"
|
||||
},
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values(rustfs_scanner_source_work_total,source)",
|
||||
"includeAll": true,
|
||||
"label": "Scanner Source",
|
||||
"multi": true,
|
||||
"name": "scanner_source",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(rustfs_scanner_source_work_total,source)",
|
||||
"refId": "PrometheusVariableQueryEditor-scanner_source"
|
||||
},
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values(rustfs_scanner_bucket_drive_result_total,result)",
|
||||
"includeAll": true,
|
||||
"label": "Scanner Result",
|
||||
"multi": true,
|
||||
"name": "scanner_result",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(rustfs_scanner_bucket_drive_result_total,result)",
|
||||
"refId": "PrometheusVariableQueryEditor-scanner_result"
|
||||
},
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -17,9 +17,9 @@
|
||||
# =============================================================================
|
||||
#
|
||||
# Metric source: the KMS operation-policy choke point in
|
||||
# crates/kms/src/policy.rs. All label values are bounded static strings
|
||||
# (operation, op_class, outcome, error_class, backend, scope); key identifiers,
|
||||
# key material, and tokens never appear in labels.
|
||||
# crates/kms/src/policy.rs. All label values are static enum strings
|
||||
# (operation, op_class, outcome, error_class); key identifiers, key material,
|
||||
# and tokens never appear in labels.
|
||||
#
|
||||
# Response procedures: docs/operations/kms-observability-runbook.md
|
||||
#
|
||||
@@ -70,9 +70,8 @@ groups:
|
||||
# ------------------------------------------------------------------
|
||||
# 2. KmsBackendHighErrorRate
|
||||
# Sustained share of operations terminating without success
|
||||
# (fatal, budget/deadline exhaustion, admission backpressure,
|
||||
# or an open circuit). The cancelled outcome is excluded because
|
||||
# shutdowns legitimately produce it.
|
||||
# (fatal, budget_exhausted, deadline_exceeded). The cancelled
|
||||
# outcome is excluded because shutdowns legitimately produce it.
|
||||
# The traffic guard keeps a single failure on a near-idle
|
||||
# cluster from firing the alert.
|
||||
# Threshold: 5% for 10m — conservative default, calibrate
|
||||
@@ -95,11 +94,9 @@ groups:
|
||||
summary: "KMS backend non-success ratio above 5% for 10m"
|
||||
description: >-
|
||||
{{ $value | humanizePercentage }} of KMS backend operations
|
||||
are terminating in fatal, budget_exhausted,
|
||||
deadline_exceeded, backpressure_timeout,
|
||||
backpressure_rejected, or circuit_open. Object encryption
|
||||
and decryption paths depending on the KMS are degraded or
|
||||
failing.
|
||||
are terminating in fatal, budget_exhausted, or
|
||||
deadline_exceeded. Object encryption and decryption paths
|
||||
depending on the KMS are degraded or failing.
|
||||
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendhigherrorrate"
|
||||
|
||||
# ==========================================================================
|
||||
@@ -189,26 +186,3 @@ groups:
|
||||
Retryable failures are outlasting the retry budget, so
|
||||
callers are seeing hard failures.
|
||||
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendretrybudgetexhausted"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 6. KmsBackendCircuitOpen
|
||||
# Direct circuit-state signal, independent of operation traffic.
|
||||
# A transient open can recover on its first half-open probe; alert
|
||||
# only when the circuit remains open or half-open for one minute.
|
||||
# ------------------------------------------------------------------
|
||||
- alert: KmsBackendCircuitOpen
|
||||
expr: |
|
||||
rustfs_kms_backend_circuit_open > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
component: kms
|
||||
annotations:
|
||||
summary: "KMS backend circuit open ({{ $labels.backend }}/{{ $labels.scope }})"
|
||||
description: >-
|
||||
The KMS backend circuit for {{ $labels.backend }} scope
|
||||
{{ $labels.scope }} has remained open or half-open for one
|
||||
minute. Operations in this scope can terminate as
|
||||
circuit_open until the half-open probe succeeds or returns
|
||||
a non-retryable failure.
|
||||
runbook_url: "https://github.com/rustfs/rustfs/blob/main/docs/operations/kms-observability-runbook.md#kmsbackendcircuitopen"
|
||||
|
||||
+13
-40
@@ -340,11 +340,9 @@ jobs:
|
||||
- name: Annotate early-stop reason
|
||||
if: failure() && github.event_name == 'pull_request'
|
||||
run: |
|
||||
{
|
||||
echo "## CI early-stop"
|
||||
echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; cancelling run ${GITHUB_RUN_ID} to free runners."
|
||||
echo "Sibling jobs showing **cancelled** were stopped by this job, not by their own failure."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "## CI early-stop" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; cancelling run ${GITHUB_RUN_ID} to free runners." >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Sibling jobs showing **cancelled** were stopped by this job, not by their own failure." >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# curl rather than `gh`: every existing `gh` call in this repo runs on
|
||||
# ubuntu-latest, and the sm-standard-* images are custom and trimmed (they
|
||||
@@ -667,17 +665,15 @@ jobs:
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
# Build the e2e test graph once. The archive is reused by the security
|
||||
# count-floor check and the smoke run below, avoiding a second compile of
|
||||
# the same e2e_test target on cold runners (backlog#1645).
|
||||
- name: Archive e2e smoke test binaries
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
run: |
|
||||
cargo nextest archive --profile e2e-smoke -p e2e_test --archive-file "${NEXTEST_ARCHIVE}"
|
||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --message-format json > "${NEXTEST_LISTING}"
|
||||
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
||||
# Guard the security negative-auth smoke subset (backlog#1151 sec-5)
|
||||
# against a rename or deletion silently dropping it out of the e2e-smoke
|
||||
# filter. The script lists what the profile selects and fails if the count
|
||||
# of security auth-rejection tests falls below the committed floor in
|
||||
# .config/security-smoke-floor.txt (infra-12 count-floor mechanism). Run
|
||||
# before the smoke suite so a thinned gate fails fast; the `nextest list`
|
||||
# here compiles the e2e_test binaries the run below reuses.
|
||||
- name: Check security smoke subset count floor
|
||||
run: ./scripts/check_security_smoke_count.sh check
|
||||
|
||||
# PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
|
||||
# profile.e2e-smoke default-filter in .config/nextest.toml is the single
|
||||
@@ -685,30 +681,7 @@ jobs:
|
||||
# adding new e2e jobs here. Each test spawns its own rustfs server on a
|
||||
# random port and reuses the downloaded debug binary above.
|
||||
- name: Run e2e smoke suite
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/rustfs-e2e-smoke.tar.zst
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
|
||||
run: |
|
||||
cargo nextest run --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" \
|
||||
--status-level all --final-status-level all --failure-output final
|
||||
|
||||
- name: Upload e2e smoke diagnostics
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-smoke-diagnostics-${{ github.run_number }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: e2e-smoke-junit-${{ github.run_number }}
|
||||
path: target/nextest/e2e-smoke/junit.xml
|
||||
if-no-files-found: warn
|
||||
run: cargo nextest run --profile e2e-smoke -p e2e_test
|
||||
|
||||
- name: Install s3s-e2e test tool
|
||||
uses: taiki-e/cache-cargo-install-action@7447f04c51f2ba27ca35e7f1e28fab848c5b3ba7 # v2
|
||||
|
||||
@@ -75,7 +75,6 @@ jobs:
|
||||
INTEROP_PACKAGE: rustfs
|
||||
INTEROP_FEATURES: rio-v2
|
||||
INTEROP_FILTER: "test(minio_generated_read_test::)"
|
||||
INTEROP_REQUIRED_TESTS: '["reads_minio_generated_sse_s3_multipart_fixture", "reads_minio_generated_sse_kms_multipart_fixture", "rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key", "rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext"]'
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
@@ -96,23 +95,20 @@ jobs:
|
||||
# is a perfectly valid filterset that matches zero tests, so the next
|
||||
# rename or module move would leave this job selecting nothing and
|
||||
# reporting success without executing a single interop assertion. Count
|
||||
# the selection and require every core reader test, while allowing new
|
||||
# reader cases to be added without changing this guard.
|
||||
# the selection and fail with a reason instead.
|
||||
#
|
||||
# Count only `filter-match.status == "matches"`: the top-level
|
||||
# `test-count` in the JSON is the package total and ignores `-E` entirely.
|
||||
- name: Assert the interop selector still matches tests
|
||||
run: |
|
||||
set -euo pipefail
|
||||
selection="$(cargo nextest list --run-ignored ignored-only \
|
||||
count="$(cargo nextest list --run-ignored all \
|
||||
-p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
|
||||
-E "$INTEROP_FILTER" --message-format json \
|
||||
| python3 -c 'import json,os,sys; d=json.load(sys.stdin); required=json.loads(os.environ["INTEROP_REQUIRED_TESTS"]); matched=[name for suite in d.get("rust-suites", {}).values() for name,test in suite.get("testcases", {}).items() if test.get("filter-match", {}).get("status") == "matches"]; missing=[test for test in required if not any(name.endswith("minio_generated_read_test::" + test) for name in matched)]; print(len(matched)); print(",".join(missing))')"
|
||||
count="$(printf '%s\n' "$selection" | sed -n '1p')"
|
||||
missing="$(printf '%s\n' "$selection" | sed -n '2p')"
|
||||
| python3 -c 'import json,sys; d=json.load(sys.stdin); print(sum(1 for s in d.get("rust-suites", {}).values() for t in s.get("testcases", {}).values() if t.get("filter-match", {}).get("status") == "matches"))')"
|
||||
echo "interop tests selected: ${count}"
|
||||
if [ -n "${missing}" ]; then
|
||||
echo "::error::Selector '${INTEROP_FILTER}' in package '${INTEROP_PACKAGE}' is missing required tests: ${missing}. The MinIO interop reader tests have moved or been renamed; fix the selector instead of running an incomplete matrix. Context: rustfs/backlog#1638."
|
||||
if [ "${count}" -eq 0 ]; then
|
||||
echo "::error::Selector '${INTEROP_FILTER}' in package '${INTEROP_PACKAGE}' matched 0 tests. The MinIO interop reader tests have moved or been renamed again; fix the selector instead of letting this job pass without running them. Context: rustfs/backlog#1638."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Windows Filesystem Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- "crates/ecstore/src/disk/**"
|
||||
- "crates/ecstore/src/store/init_format.rs"
|
||||
- "crates/ecstore/Cargo.toml"
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- ".github/actions/setup/**"
|
||||
- ".github/workflows/windows-filesystem.yml"
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- "crates/ecstore/src/disk/**"
|
||||
- "crates/ecstore/src/store/init_format.rs"
|
||||
- "crates/ecstore/Cargo.toml"
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- ".github/actions/setup/**"
|
||||
- ".github/workflows/windows-filesystem.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
rename-safety:
|
||||
name: Rename Safety
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: build-x86_64-pc-windows-msvc
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Test guarded rename publication
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib rename_all_ -- --nocapture
|
||||
|
||||
- name: Test Windows handle guards
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib windows_ -- --nocapture
|
||||
|
||||
- name: Test startup temporary-directory cleanup
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib cleanup_tmp_on_startup_ -- --nocapture
|
||||
|
||||
- name: Test fresh format publication
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib fresh_format_load_initializes_all_disks -- --nocapture
|
||||
@@ -347,11 +347,6 @@ cargo run -p rustfs-filemeta --example dump_fileinfo -- "/path/to/file/xl.meta"
|
||||
absent, empty, and nil all mean "no value", never `Uuid::nil()`.
|
||||
- A remote-tier version of `None`/`""` means the tier bucket is unversioned:
|
||||
send **no** `versionId` on tier GET/DELETE.
|
||||
- Structs persisted in the scanner data-usage cache (`DataUsageCacheInfo`,
|
||||
`DataUsageEntry`) carry a hand-written map-encoded `Serialize`. MessagePack
|
||||
encodes derived structs as arrays, where an appended field makes the whole
|
||||
cache a decode error for older readers — keep new fields `#[serde(default)]`
|
||||
and keep the map encoding rather than reverting to `derive(Serialize)`.
|
||||
|
||||
## Naming Conventions
|
||||
|
||||
|
||||
Generated
+21
-22
@@ -1740,9 +1740,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bytesize"
|
||||
version = "2.7.0"
|
||||
version = "2.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b"
|
||||
checksum = "351a3e803ee3c6eaeee6b00076b767514b37c32a73d326c3ec7abddb7d6c3493"
|
||||
|
||||
[[package]]
|
||||
name = "bytestring"
|
||||
@@ -4992,9 +4992,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath"
|
||||
version = "0.23.0"
|
||||
version = "0.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab303f15e2bbd9633a577338c9813a86bc1aef74beb8b536e27f28c80e84befc"
|
||||
checksum = "66750a77f4f6b408a148be5102ef1f3ba7172def7ee92b1cfc75d9f7a3870453"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -5026,9 +5026,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros"
|
||||
version = "0.23.0"
|
||||
version = "0.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4777d4dd3474c9b9c9391be713c6b570da0ac49e992a8cbfed67f60ca0f7e33d"
|
||||
checksum = "afe0e1900d2dbe2e2df8e9522b97ebd7a5598ba18478f57e247957022dffedbe"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5037,15 +5037,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros-meta"
|
||||
version = "0.23.0"
|
||||
version = "0.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "833200923e0ba8150fb91d6a3a39643eef95e604c7394c2f2679905cae13862c"
|
||||
checksum = "21f2f70b29b6f42311acd2fb0b2f91a34d9a573c76fb8a7f51970670a1673a49"
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-meta"
|
||||
version = "0.23.0"
|
||||
version = "0.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eca34dbbafc05f5da2a2696ce2640018fbe29e9932736309efbf3a990f0b2832"
|
||||
checksum = "8b771f77b2f409086bb40ff029f850ce99d17118d4e207df0f28cb32bd7568c7"
|
||||
dependencies = [
|
||||
"hotpath-macros-meta",
|
||||
]
|
||||
@@ -5445,9 +5445,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.12.1"
|
||||
version = "2.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78"
|
||||
checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
@@ -5923,9 +5923,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libredox"
|
||||
version = "0.1.19"
|
||||
version = "0.1.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2026a5056764a10b2bf5d56488cba40da507f5493a6a429340e2004d9ed085fa"
|
||||
checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
@@ -8036,9 +8036,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "psm"
|
||||
version = "0.1.32"
|
||||
version = "0.1.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622"
|
||||
checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea"
|
||||
dependencies = [
|
||||
"ar_archive_writer",
|
||||
"cc",
|
||||
@@ -9642,7 +9642,6 @@ dependencies = [
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
"uuid",
|
||||
"vaultrs",
|
||||
@@ -11405,9 +11404,9 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
|
||||
|
||||
[[package]]
|
||||
name = "stacker"
|
||||
version = "0.1.25"
|
||||
version = "0.1.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967"
|
||||
checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cfg-if",
|
||||
@@ -11714,7 +11713,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -11824,9 +11823,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.55"
|
||||
version = "0.3.54"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134"
|
||||
checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244"
|
||||
dependencies = [
|
||||
"deranged",
|
||||
"js-sys",
|
||||
|
||||
+3
-3
@@ -173,7 +173,7 @@ tower-http = { version = "0.7.0" }
|
||||
# Serialization and Data Formats
|
||||
apache-avro = "0.21.0"
|
||||
bytes = { version = "1.12.1" }
|
||||
bytesize = "2.7.0"
|
||||
bytesize = "2.6.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
form_urlencoded = "1.2.2"
|
||||
@@ -212,7 +212,7 @@ zeroize = { version = "1.9.0" }
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.35" }
|
||||
time = { version = "0.3.55" }
|
||||
time = { version = "0.3.54" }
|
||||
|
||||
# Database
|
||||
deadpool-postgres = { version = "0.14" }
|
||||
@@ -350,7 +350,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13", features = ["extended"] }
|
||||
hotpath = { version = "0.23.0", default-features = false }
|
||||
hotpath = { version = "0.22.0", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
|
||||
+14
-375
@@ -17,7 +17,7 @@ use crate::last_minute::{AccElem, LastMinuteLatency};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::{BTreeSet, HashMap},
|
||||
collections::HashMap,
|
||||
fmt::Display,
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
@@ -708,48 +708,6 @@ struct ScannerDiskBucketScanState {
|
||||
active: u64,
|
||||
}
|
||||
|
||||
type ScannerDiskBucketScanKey = (String, String);
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ScannerDiskBucketScanSnapshot {
|
||||
pub pool: String,
|
||||
pub set: String,
|
||||
pub concurrency_limit: u64,
|
||||
pub queued: u64,
|
||||
pub active: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
struct ScannerBucketDriveResultKey {
|
||||
bucket: String,
|
||||
drive: String,
|
||||
result: String,
|
||||
}
|
||||
|
||||
impl ScannerBucketDriveResultKey {
|
||||
fn new(bucket: impl Into<String>, drive: impl Into<String>, result: impl Into<String>) -> Self {
|
||||
Self {
|
||||
bucket: bucket.into(),
|
||||
drive: drive.into(),
|
||||
result: result.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS: usize = 4096;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct ScannerBucketDriveResults {
|
||||
counts: HashMap<ScannerBucketDriveResultKey, ScannerBucketDriveResultValue>,
|
||||
eviction_index: BTreeSet<(u64, ScannerBucketDriveResultKey)>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct ScannerBucketDriveResultValue {
|
||||
count: u64,
|
||||
last_seen: u64,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Metrics
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -780,11 +738,7 @@ pub struct Metrics {
|
||||
scanner_set_scan_concurrency_limit: AtomicU64,
|
||||
scanner_set_scans_queued: AtomicU64,
|
||||
scanner_set_scans_active: AtomicU64,
|
||||
scanner_disk_bucket_scan_states: Mutex<HashMap<ScannerDiskBucketScanKey, ScannerDiskBucketScanState>>,
|
||||
scanner_bucket_drive_results: Mutex<ScannerBucketDriveResults>,
|
||||
scanner_bucket_drive_result_clock: AtomicU64,
|
||||
current_scan_cycle_bucket_drive_results_start: Mutex<HashMap<ScannerBucketDriveResultKey, u64>>,
|
||||
last_scan_cycle_bucket_drive_results: Mutex<Vec<ScannerBucketDriveResultSnapshot>>,
|
||||
scanner_disk_bucket_scan_states: Mutex<HashMap<String, ScannerDiskBucketScanState>>,
|
||||
scanner_leader_lock_state: RwLock<String>,
|
||||
scanner_leader_lock_held: AtomicBool,
|
||||
scanner_leader_lock_last_error: RwLock<String>,
|
||||
@@ -1004,14 +958,6 @@ pub struct ScannerSourceWorkSnapshot {
|
||||
pub missed: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ScannerBucketDriveResultSnapshot {
|
||||
pub bucket: String,
|
||||
pub drive: String,
|
||||
pub result: String,
|
||||
pub count: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ScannerReplicationRepairSnapshot {
|
||||
pub source: String,
|
||||
@@ -1344,18 +1290,6 @@ pub struct ScannerMetricsReport {
|
||||
pub partial_cycles: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct ScannerRuntimeDetailsReport {
|
||||
#[serde(default)]
|
||||
pub disk_bucket_scan_states: Vec<ScannerDiskBucketScanSnapshot>,
|
||||
#[serde(default)]
|
||||
pub bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
|
||||
#[serde(default)]
|
||||
pub current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
|
||||
#[serde(default)]
|
||||
pub last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
|
||||
}
|
||||
|
||||
impl CurrentCycle {
|
||||
pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
*self = rmp_serde::from_slice(buf)?;
|
||||
@@ -1723,7 +1657,6 @@ pub fn emit_scan_cycle_superseded(duration: Duration) {
|
||||
|
||||
pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
|
||||
let result = if success { "success" } else { "error" };
|
||||
global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
|
||||
metrics::counter!(
|
||||
OTEL_SCANNER_BUCKETS_SCANNED,
|
||||
"result" => result,
|
||||
@@ -1740,7 +1673,6 @@ pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str,
|
||||
}
|
||||
|
||||
pub fn emit_scan_bucket_drive_partial(bucket: &str, disk: &str, duration: Duration) {
|
||||
global_metrics().record_scanner_bucket_drive_result(bucket, disk, SCAN_CYCLE_RESULT_PARTIAL_LABEL);
|
||||
metrics::counter!(
|
||||
OTEL_SCANNER_BUCKETS_SCANNED,
|
||||
"result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL,
|
||||
@@ -1791,10 +1723,6 @@ impl Metrics {
|
||||
scanner_set_scans_queued: AtomicU64::new(0),
|
||||
scanner_set_scans_active: AtomicU64::new(0),
|
||||
scanner_disk_bucket_scan_states: Mutex::new(HashMap::new()),
|
||||
scanner_bucket_drive_results: Mutex::new(ScannerBucketDriveResults::default()),
|
||||
scanner_bucket_drive_result_clock: AtomicU64::new(0),
|
||||
current_scan_cycle_bucket_drive_results_start: Mutex::new(HashMap::new()),
|
||||
last_scan_cycle_bucket_drive_results: Mutex::new(Vec::new()),
|
||||
scanner_leader_lock_state: RwLock::new("unknown".to_string()),
|
||||
scanner_leader_lock_held: AtomicBool::new(false),
|
||||
scanner_leader_lock_last_error: RwLock::new(String::new()),
|
||||
@@ -2365,7 +2293,7 @@ impl Metrics {
|
||||
queued: Option<usize>,
|
||||
active: Option<usize>,
|
||||
) {
|
||||
let key = (pool.to_string(), set.to_string());
|
||||
let key = format!("{pool}/{set}");
|
||||
let mut states = self
|
||||
.scanner_disk_bucket_scan_states
|
||||
.lock()
|
||||
@@ -2382,41 +2310,6 @@ impl Metrics {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_scanner_bucket_drive_result(&self, bucket: &str, drive: &str, result: &str) {
|
||||
if bucket.is_empty() || drive.is_empty() || result.is_empty() {
|
||||
return;
|
||||
}
|
||||
let key = ScannerBucketDriveResultKey::new(bucket, drive, result);
|
||||
let mut results = self
|
||||
.scanner_bucket_drive_results
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let last_seen = self.scanner_bucket_drive_result_clock.fetch_add(1, Ordering::Relaxed);
|
||||
if let Some(previous_last_seen) = results.counts.get_mut(&key).map(|value| {
|
||||
let previous_last_seen = value.last_seen;
|
||||
value.count = value.count.saturating_add(1);
|
||||
value.last_seen = last_seen;
|
||||
previous_last_seen
|
||||
}) {
|
||||
results.eviction_index.remove(&(previous_last_seen, key.clone()));
|
||||
results.eviction_index.insert((last_seen, key));
|
||||
return;
|
||||
}
|
||||
|
||||
if results.counts.len() >= MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS
|
||||
&& let Some((_, stale_key)) = results.eviction_index.pop_first()
|
||||
{
|
||||
results.counts.remove(&stale_key);
|
||||
}
|
||||
|
||||
if results.counts.len() < MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
|
||||
results
|
||||
.counts
|
||||
.insert(key.clone(), ScannerBucketDriveResultValue { count: 1, last_seen });
|
||||
results.eviction_index.insert((last_seen, key));
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Read-side helpers
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -2588,11 +2481,6 @@ impl Metrics {
|
||||
&self.current_scan_cycle_replication_repair_work_start,
|
||||
&replication_repair_snapshot,
|
||||
);
|
||||
let bucket_drive_results = self.scanner_bucket_drive_result_counts();
|
||||
match self.current_scan_cycle_bucket_drive_results_start.lock() {
|
||||
Ok(mut start) => *start = bucket_drive_results,
|
||||
Err(poisoned) => *poisoned.into_inner() = bucket_drive_results,
|
||||
}
|
||||
self.current_scan_cycle_work_active.store(true, Ordering::Release);
|
||||
snapshot
|
||||
}
|
||||
@@ -2605,11 +2493,6 @@ impl Metrics {
|
||||
self.record_scan_cycle_work(work);
|
||||
self.record_scan_cycle_source_work(&source_work);
|
||||
self.record_scan_cycle_replication_repair_work(&replication_repair_work);
|
||||
let bucket_drive_results = self.current_cycle_bucket_drive_result_snapshots();
|
||||
match self.last_scan_cycle_bucket_drive_results.lock() {
|
||||
Ok(mut last) => *last = bucket_drive_results,
|
||||
Err(poisoned) => *poisoned.into_inner() = bucket_drive_results,
|
||||
}
|
||||
self.current_scan_cycle_work_active.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
@@ -2693,105 +2576,6 @@ impl Metrics {
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_bucket_drive_result_counts(&self) -> HashMap<ScannerBucketDriveResultKey, u64> {
|
||||
self.scanner_bucket_drive_results
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.counts
|
||||
.iter()
|
||||
.map(|(key, value)| (key.clone(), value.count))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn scanner_bucket_drive_result_snapshots(
|
||||
counts: impl IntoIterator<Item = (ScannerBucketDriveResultKey, u64)>,
|
||||
) -> Vec<ScannerBucketDriveResultSnapshot> {
|
||||
let mut snapshots = counts
|
||||
.into_iter()
|
||||
.filter(|(_, count)| *count > 0)
|
||||
.map(|(key, count)| ScannerBucketDriveResultSnapshot {
|
||||
bucket: key.bucket,
|
||||
drive: key.drive,
|
||||
result: key.result,
|
||||
count,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
snapshots.sort_by(|left, right| {
|
||||
left.bucket
|
||||
.cmp(&right.bucket)
|
||||
.then_with(|| left.drive.cmp(&right.drive))
|
||||
.then_with(|| left.result.cmp(&right.result))
|
||||
});
|
||||
snapshots
|
||||
}
|
||||
|
||||
fn scanner_bucket_drive_result_counter_snapshots(&self) -> Vec<ScannerBucketDriveResultSnapshot> {
|
||||
Self::scanner_bucket_drive_result_snapshots(self.scanner_bucket_drive_result_counts())
|
||||
}
|
||||
|
||||
fn current_cycle_bucket_drive_result_snapshots(&self) -> Vec<ScannerBucketDriveResultSnapshot> {
|
||||
let current = self.scanner_bucket_drive_result_counts();
|
||||
let start = self
|
||||
.current_scan_cycle_bucket_drive_results_start
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clone();
|
||||
Self::scanner_bucket_drive_result_snapshots(current.into_iter().filter_map(|(key, count)| {
|
||||
let delta = count.saturating_sub(start.get(&key).copied().unwrap_or_default());
|
||||
(delta > 0).then_some((key, delta))
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn scanner_runtime_details_report(&self) -> ScannerRuntimeDetailsReport {
|
||||
self.scanner_runtime_details_report_for_active(self.current_scan_cycle_work_active.load(Ordering::Acquire))
|
||||
}
|
||||
|
||||
fn scanner_runtime_details_report_for_active(&self, current_cycle_active: bool) -> ScannerRuntimeDetailsReport {
|
||||
let current_cycle_bucket_drive_results = if current_cycle_active {
|
||||
self.current_cycle_bucket_drive_result_snapshots()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
ScannerRuntimeDetailsReport {
|
||||
disk_bucket_scan_states: self.scanner_disk_bucket_scan_state_snapshots(),
|
||||
bucket_drive_results: self.scanner_bucket_drive_result_counter_snapshots(),
|
||||
current_cycle_bucket_drive_results,
|
||||
last_cycle_bucket_drive_results: self
|
||||
.last_scan_cycle_bucket_drive_results
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_disk_bucket_scan_state_snapshots(&self) -> Vec<ScannerDiskBucketScanSnapshot> {
|
||||
let mut disk_bucket_scan_states = match self.scanner_disk_bucket_scan_states.lock() {
|
||||
Ok(states) => states
|
||||
.iter()
|
||||
.map(|((pool, set), state)| ScannerDiskBucketScanSnapshot {
|
||||
pool: pool.clone(),
|
||||
set: set.clone(),
|
||||
concurrency_limit: state.concurrency_limit,
|
||||
queued: state.queued,
|
||||
active: state.active,
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
Err(poisoned) => poisoned
|
||||
.into_inner()
|
||||
.iter()
|
||||
.map(|((pool, set), state)| ScannerDiskBucketScanSnapshot {
|
||||
pool: pool.clone(),
|
||||
set: set.clone(),
|
||||
concurrency_limit: state.concurrency_limit,
|
||||
queued: state.queued,
|
||||
active: state.active,
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
};
|
||||
disk_bucket_scan_states.sort_by(|left, right| left.pool.cmp(&right.pool).then_with(|| left.set.cmp(&right.set)));
|
||||
disk_bucket_scan_states
|
||||
}
|
||||
|
||||
fn scanner_source_work_values(&self) -> Vec<ScannerSourceWorkValues> {
|
||||
ScannerWorkSource::all()
|
||||
.iter()
|
||||
@@ -2977,12 +2761,7 @@ impl Metrics {
|
||||
|
||||
/// Build a full metrics report snapshot.
|
||||
pub async fn report(&self) -> ScannerMetricsReport {
|
||||
self.report_with_runtime_details().await.0
|
||||
}
|
||||
|
||||
pub async fn report_with_runtime_details(&self) -> (ScannerMetricsReport, ScannerRuntimeDetailsReport) {
|
||||
let mut m = ScannerMetricsReport::default();
|
||||
let runtime_details;
|
||||
|
||||
let has_cycle = {
|
||||
let cycle = self.cycle_info.read().await;
|
||||
@@ -2996,7 +2775,6 @@ impl Metrics {
|
||||
};
|
||||
m.current_cycle_active = self.current_scan_cycle_work_active.load(Ordering::Acquire);
|
||||
if m.current_cycle_active {
|
||||
// Keep cycle_info before cycle-baseline locks so active scrapes cannot mix two cycle identities.
|
||||
let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
|
||||
let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
|
||||
let current_replication_repair_work =
|
||||
@@ -3019,7 +2797,6 @@ impl Metrics {
|
||||
m.current_cycle_replication_repair =
|
||||
self.scanner_replication_repair_work_snapshots(¤t_replication_repair_work);
|
||||
}
|
||||
runtime_details = self.scanner_runtime_details_report_for_active(m.current_cycle_active);
|
||||
has_cycle
|
||||
};
|
||||
|
||||
@@ -3049,11 +2826,15 @@ impl Metrics {
|
||||
m.current_set_scan_concurrency_limit = self.scanner_set_scan_concurrency_limit.load(Ordering::Relaxed);
|
||||
m.current_set_scans_queued = self.scanner_set_scans_queued.load(Ordering::Relaxed);
|
||||
m.current_set_scans_active = self.scanner_set_scans_active.load(Ordering::Relaxed);
|
||||
let disk_bucket_scan_states = self.scanner_disk_bucket_scan_state_snapshots();
|
||||
let (disk_scan_concurrency_limit, disk_bucket_scans_queued, disk_bucket_scans_active) =
|
||||
disk_bucket_scan_states.iter().fold((0, 0, 0), |acc, state| {
|
||||
(acc.0 + state.concurrency_limit, acc.1 + state.queued, acc.2 + state.active)
|
||||
});
|
||||
match self.scanner_disk_bucket_scan_states.lock() {
|
||||
Ok(states) => states.values().fold((0, 0, 0), |acc, state| {
|
||||
(acc.0 + state.concurrency_limit, acc.1 + state.queued, acc.2 + state.active)
|
||||
}),
|
||||
Err(poisoned) => poisoned.into_inner().values().fold((0, 0, 0), |acc, state| {
|
||||
(acc.0 + state.concurrency_limit, acc.1 + state.queued, acc.2 + state.active)
|
||||
}),
|
||||
};
|
||||
m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
|
||||
m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
|
||||
m.current_disk_bucket_scans_active = disk_bucket_scans_active;
|
||||
@@ -3222,7 +3003,7 @@ impl Metrics {
|
||||
m.pacing_pressure = scanner_pacing_pressure(&m);
|
||||
m.maintenance_control = scanner_maintenance_control(&m);
|
||||
|
||||
(m, runtime_details)
|
||||
m
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4319,137 +4100,6 @@ mod tests {
|
||||
assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_includes_structured_bucket_drive_results() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scanner_bucket_drive_result("photos", "/data1", "success");
|
||||
|
||||
let cycle_start = metrics.start_scan_cycle_work();
|
||||
metrics.record_scanner_bucket_drive_result("photos", "/data1", "partial");
|
||||
|
||||
let active_report = metrics.scanner_runtime_details_report();
|
||||
assert_eq!(
|
||||
active_report.current_cycle_bucket_drive_results,
|
||||
vec![ScannerBucketDriveResultSnapshot {
|
||||
bucket: "photos".to_string(),
|
||||
drive: "/data1".to_string(),
|
||||
result: "partial".to_string(),
|
||||
count: 1,
|
||||
}]
|
||||
);
|
||||
|
||||
metrics.finish_scan_cycle_work(cycle_start);
|
||||
|
||||
let report = metrics.scanner_runtime_details_report();
|
||||
|
||||
assert_eq!(
|
||||
report.bucket_drive_results,
|
||||
vec![
|
||||
ScannerBucketDriveResultSnapshot {
|
||||
bucket: "photos".to_string(),
|
||||
drive: "/data1".to_string(),
|
||||
result: "partial".to_string(),
|
||||
count: 1,
|
||||
},
|
||||
ScannerBucketDriveResultSnapshot {
|
||||
bucket: "photos".to_string(),
|
||||
drive: "/data1".to_string(),
|
||||
result: "success".to_string(),
|
||||
count: 1,
|
||||
},
|
||||
]
|
||||
);
|
||||
assert!(report.current_cycle_bucket_drive_results.is_empty());
|
||||
assert_eq!(
|
||||
report.last_cycle_bucket_drive_results,
|
||||
vec![ScannerBucketDriveResultSnapshot {
|
||||
bucket: "photos".to_string(),
|
||||
drive: "/data1".to_string(),
|
||||
result: "partial".to_string(),
|
||||
count: 1,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_bucket_drive_results_are_bounded() {
|
||||
let metrics = Metrics::new();
|
||||
for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
|
||||
metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
|
||||
}
|
||||
metrics.record_scanner_bucket_drive_result("overflow", "/data1", "success");
|
||||
|
||||
let report = metrics.scanner_runtime_details_report();
|
||||
|
||||
assert_eq!(report.bucket_drive_results.len(), MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS);
|
||||
assert!(
|
||||
report
|
||||
.bucket_drive_results
|
||||
.iter()
|
||||
.any(|snapshot| snapshot.bucket == "overflow")
|
||||
);
|
||||
assert!(
|
||||
report
|
||||
.bucket_drive_results
|
||||
.iter()
|
||||
.all(|snapshot| snapshot.bucket != "bucket-0")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_bucket_drive_result_eviction_keeps_recent_keys() {
|
||||
let metrics = Metrics::new();
|
||||
for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
|
||||
metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
|
||||
}
|
||||
metrics.record_scanner_bucket_drive_result("bucket-0", "/data1", "success");
|
||||
metrics.record_scanner_bucket_drive_result("overflow", "/data1", "success");
|
||||
|
||||
let report = metrics.scanner_runtime_details_report();
|
||||
|
||||
assert_eq!(report.bucket_drive_results.len(), MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS);
|
||||
assert!(
|
||||
report
|
||||
.bucket_drive_results
|
||||
.iter()
|
||||
.any(|snapshot| snapshot.bucket == "bucket-0" && snapshot.count == 2)
|
||||
);
|
||||
assert!(
|
||||
report
|
||||
.bucket_drive_results
|
||||
.iter()
|
||||
.all(|snapshot| snapshot.bucket != "bucket-1")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_bucket_drive_result_eviction_survives_full_refresh() {
|
||||
let metrics = Metrics::new();
|
||||
for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
|
||||
metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
|
||||
}
|
||||
for index in 0..MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS {
|
||||
metrics.record_scanner_bucket_drive_result(&format!("bucket-{index}"), "/data1", "success");
|
||||
}
|
||||
metrics.record_scanner_bucket_drive_result("overflow", "/data1", "success");
|
||||
|
||||
let report = metrics.scanner_runtime_details_report();
|
||||
|
||||
assert_eq!(report.bucket_drive_results.len(), MAX_SCANNER_BUCKET_DRIVE_RESULT_KEYS);
|
||||
assert!(
|
||||
report
|
||||
.bucket_drive_results
|
||||
.iter()
|
||||
.any(|snapshot| snapshot.bucket == "overflow")
|
||||
);
|
||||
assert!(
|
||||
report
|
||||
.bucket_drive_results
|
||||
.iter()
|
||||
.all(|snapshot| snapshot.bucket != "bucket-0")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_includes_usage_freshness_status() {
|
||||
let metrics = Metrics::new();
|
||||
@@ -4615,10 +4265,9 @@ mod tests {
|
||||
};
|
||||
let cycle_ten_start = metrics.start_scan_cycle_work_with_cycle(cycle_ten.clone()).await;
|
||||
metrics.operations[Metric::ScanObject as usize].store(1, Ordering::Relaxed);
|
||||
metrics.record_scanner_bucket_drive_result("cycle-ten", "/data1", "partial");
|
||||
|
||||
let paths = metrics.current_paths.write().await;
|
||||
let mut report = Box::pin(metrics.report_with_runtime_details());
|
||||
let mut report = Box::pin(metrics.report());
|
||||
let waker = std::task::Waker::noop();
|
||||
let mut context = std::task::Context::from_waker(waker);
|
||||
assert!(report.as_mut().poll(&mut context).is_pending());
|
||||
@@ -4635,22 +4284,12 @@ mod tests {
|
||||
})
|
||||
.await;
|
||||
metrics.operations[Metric::ScanObject as usize].store(101, Ordering::Relaxed);
|
||||
metrics.record_scanner_bucket_drive_result("cycle-eleven", "/data1", "partial");
|
||||
|
||||
drop(paths);
|
||||
let (snapshot, runtime_details) = report.await;
|
||||
let snapshot = report.await;
|
||||
|
||||
assert_eq!(snapshot.current_cycle, 10);
|
||||
assert_eq!(snapshot.current_cycle_objects_scanned, 1);
|
||||
assert_eq!(
|
||||
runtime_details.current_cycle_bucket_drive_results,
|
||||
vec![ScannerBucketDriveResultSnapshot {
|
||||
bucket: "cycle-ten".to_string(),
|
||||
drive: "/data1".to_string(),
|
||||
result: "partial".to_string(),
|
||||
count: 1,
|
||||
}]
|
||||
);
|
||||
|
||||
metrics
|
||||
.finish_scan_cycle_work_with_cycle(cycle_eleven_start, CurrentCycle::default())
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use serde::{Deserialize, Serialize, ser::SerializeMap as _};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
hash::{DefaultHasher, Hash, Hasher},
|
||||
@@ -51,36 +51,24 @@ pub fn usage_last_update_is_untrusted_future(existing_last_update: SystemTime, n
|
||||
existing_last_update > now + USAGE_LAST_UPDATE_FUTURE_TOLERANCE
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Default, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[derive(Clone, Copy, Default, Debug, Serialize, Deserialize, PartialEq)]
|
||||
pub struct TierStats {
|
||||
pub total_size: u64,
|
||||
pub num_versions: u64,
|
||||
pub num_objects: u64,
|
||||
pub num_versions: i32,
|
||||
pub num_objects: i32,
|
||||
}
|
||||
|
||||
impl TierStats {
|
||||
pub fn add(&self, u: &TierStats) -> TierStats {
|
||||
TierStats {
|
||||
total_size: self.total_size.saturating_add(u.total_size),
|
||||
num_versions: self.num_versions.saturating_add(u.num_versions),
|
||||
num_objects: self.num_objects.saturating_add(u.num_objects),
|
||||
total_size: self.total_size + u.total_size,
|
||||
num_versions: self.num_versions + u.num_versions,
|
||||
num_objects: self.num_objects + u.num_objects,
|
||||
}
|
||||
}
|
||||
|
||||
/// True when [`TierStats::add`] would report the exact sum instead of saturating.
|
||||
pub fn fits_add(&self, u: &TierStats) -> bool {
|
||||
self.total_size.checked_add(u.total_size).is_some()
|
||||
&& self.num_versions.checked_add(u.num_versions).is_some()
|
||||
&& self.num_objects.checked_add(u.num_objects).is_some()
|
||||
}
|
||||
|
||||
/// True when this tier contributed nothing, i.e. merging it is a no-op.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.total_size == 0 && self.num_versions == 0 && self.num_objects == 0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
|
||||
pub struct AllTierStats {
|
||||
pub tiers: HashMap<String, TierStats>,
|
||||
}
|
||||
@@ -90,35 +78,31 @@ impl AllTierStats {
|
||||
Self { tiers: HashMap::new() }
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.tiers.is_empty()
|
||||
}
|
||||
|
||||
/// Folds a scan summary's per-tier map in.
|
||||
///
|
||||
/// Scanners seed the map with a zeroed entry for every configured tier, so
|
||||
/// empty contributions are skipped to keep the persisted cache from growing
|
||||
/// one key per tier on every folder that never held tiered data.
|
||||
pub fn add_sizes(&mut self, tiers: &HashMap<String, TierStats>) {
|
||||
pub fn add_sizes(&mut self, tiers: HashMap<String, TierStats>) {
|
||||
for (tier, st) in tiers {
|
||||
if st.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry = self.tiers.entry(tier.clone()).or_default();
|
||||
*entry = entry.add(st);
|
||||
self.tiers
|
||||
.insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn merge(&mut self, other: &AllTierStats) {
|
||||
self.add_sizes(&other.tiers);
|
||||
pub fn merge(&mut self, other: AllTierStats) {
|
||||
for (tier, st) in other.tiers {
|
||||
self.tiers
|
||||
.insert(tier.clone(), self.tiers.get(&tier).copied().unwrap_or_default().add(&st));
|
||||
}
|
||||
}
|
||||
|
||||
/// True when [`AllTierStats::merge`] would report exact sums for every tier.
|
||||
pub fn fits_merge(&self, other: &AllTierStats) -> bool {
|
||||
other
|
||||
.tiers
|
||||
.iter()
|
||||
.all(|(tier, right)| self.tiers.get(tier).is_none_or(|left| left.fits_add(right)))
|
||||
pub fn populate_stats(&self, stats: &mut HashMap<String, TierStats>) {
|
||||
for (tier, st) in &self.tiers {
|
||||
stats.insert(
|
||||
tier.clone(),
|
||||
TierStats {
|
||||
total_size: st.total_size,
|
||||
num_versions: st.num_versions,
|
||||
num_objects: st.num_objects,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -199,14 +183,6 @@ pub struct DataUsageInfo {
|
||||
pub objects_total_size: u64,
|
||||
/// Replication info across all buckets
|
||||
pub replication_info: HashMap<String, BucketTargetUsageInfo>,
|
||||
/// Usage per storage class and remote tier across all buckets.
|
||||
///
|
||||
/// Absent on snapshots written before per-tier accounting was published,
|
||||
/// and on clusters with no remote tier configured: the scanner classifies
|
||||
/// objects by tier (including `STANDARD`/`REDUCED_REDUNDANCY`) only once a
|
||||
/// tier exists, so an absent value means "not accounted", never "zero".
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tier_stats: Option<AllTierStats>,
|
||||
|
||||
/// Total number of buckets in this cluster
|
||||
pub buckets_count: u64,
|
||||
@@ -586,7 +562,7 @@ impl ReplicationAllStats {
|
||||
}
|
||||
|
||||
/// Data usage cache entry
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct DataUsageEntry {
|
||||
pub children: DataUsageHashMap,
|
||||
// These fields do not include any children.
|
||||
@@ -601,34 +577,6 @@ pub struct DataUsageEntry {
|
||||
/// Number of objects that failed to scan (e.g., IO errors)
|
||||
#[serde(default)]
|
||||
pub failed_objects: usize,
|
||||
/// Per-tier usage contributed by this entry, present only once a scan
|
||||
/// observed tier-classified objects.
|
||||
#[serde(default)]
|
||||
pub all_tier_stats: Option<AllTierStats>,
|
||||
}
|
||||
|
||||
impl Serialize for DataUsageEntry {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
// Keep entries map-encoded so older readers can ignore fields appended
|
||||
// by newer scanner versions during rolling upgrades. The derived
|
||||
// (array) encoding made any appended field a decode error for them.
|
||||
let mut state = serializer.serialize_map(Some(11))?;
|
||||
state.serialize_entry("children", &self.children)?;
|
||||
state.serialize_entry("size", &self.size)?;
|
||||
state.serialize_entry("objects", &self.objects)?;
|
||||
state.serialize_entry("versions", &self.versions)?;
|
||||
state.serialize_entry("delete_markers", &self.delete_markers)?;
|
||||
state.serialize_entry("obj_sizes", &self.obj_sizes)?;
|
||||
state.serialize_entry("obj_versions", &self.obj_versions)?;
|
||||
state.serialize_entry("replication_stats", &self.replication_stats)?;
|
||||
state.serialize_entry("compacted", &self.compacted)?;
|
||||
state.serialize_entry("failed_objects", &self.failed_objects)?;
|
||||
state.serialize_entry("all_tier_stats", &self.all_tier_stats)?;
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageEntry {
|
||||
@@ -687,22 +635,10 @@ impl DataUsageEntry {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(o_tiers) = other.all_tier_stats.as_ref().filter(|tiers| !tiers.is_empty()) {
|
||||
self.all_tier_stats.get_or_insert_with(AllTierStats::new).merge(o_tiers);
|
||||
}
|
||||
|
||||
self.obj_sizes.merge_from(&other.obj_sizes);
|
||||
self.obj_versions.merge_from(&other.obj_versions);
|
||||
}
|
||||
|
||||
/// Folds a scan summary's per-tier map into this entry.
|
||||
pub fn add_tier_sizes(&mut self, tiers: &HashMap<String, TierStats>) {
|
||||
if tiers.values().all(TierStats::is_empty) {
|
||||
return;
|
||||
}
|
||||
self.all_tier_stats.get_or_insert_with(AllTierStats::new).add_sizes(tiers);
|
||||
}
|
||||
|
||||
pub fn checked_merge(&mut self, other: &DataUsageEntry) -> bool {
|
||||
let scalar_counts_fit = self.objects.checked_add(other.objects).is_some()
|
||||
&& self.versions.checked_add(other.versions).is_some()
|
||||
@@ -762,12 +698,7 @@ impl DataUsageEntry {
|
||||
}
|
||||
};
|
||||
|
||||
let tier_stats_fit = match (&self.all_tier_stats, &other.all_tier_stats) {
|
||||
(_, None) | (None, Some(_)) => true,
|
||||
(Some(left), Some(right)) => left.fits_merge(right),
|
||||
};
|
||||
|
||||
if !scalar_counts_fit || !histograms_fit || !replication_fits || !tier_stats_fit {
|
||||
if !scalar_counts_fit || !histograms_fit || !replication_fits {
|
||||
return false;
|
||||
}
|
||||
self.merge(other);
|
||||
@@ -1107,7 +1038,6 @@ impl DataUsageCache {
|
||||
versions_total_count: flat.versions as u64,
|
||||
delete_markers_total_count: flat.delete_markers as u64,
|
||||
objects_total_size: flat.size as u64,
|
||||
tier_stats: flat.all_tier_stats.filter(|tiers| !tiers.is_empty()),
|
||||
buckets_count: u64::try_from(buckets.len()).unwrap_or(u64::MAX),
|
||||
buckets_usage,
|
||||
usage_snapshot_complete: self.info.snapshot_complete,
|
||||
@@ -1595,172 +1525,6 @@ mod tests {
|
||||
buckets_count: u64,
|
||||
}
|
||||
|
||||
fn tier_entry(tier: &str, stats: TierStats) -> DataUsageEntry {
|
||||
let mut entry = DataUsageEntry::default();
|
||||
entry.add_tier_sizes(&HashMap::from([(tier.to_string(), stats)]));
|
||||
entry
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_stats_survive_entry_merge() {
|
||||
let mut left = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 10,
|
||||
num_versions: 2,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
let mut right = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 5,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
right.add_tier_sizes(&HashMap::from([(
|
||||
"COLD".to_string(),
|
||||
TierStats {
|
||||
total_size: 7,
|
||||
num_versions: 1,
|
||||
num_objects: 0,
|
||||
},
|
||||
)]));
|
||||
|
||||
assert!(left.checked_merge(&right), "merging exact tier totals must be accepted");
|
||||
|
||||
let tiers = &left.all_tier_stats.expect("merged entry keeps tier stats").tiers;
|
||||
assert_eq!(
|
||||
tiers.get("WARM"),
|
||||
Some(&TierStats {
|
||||
total_size: 15,
|
||||
num_versions: 3,
|
||||
num_objects: 2,
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
tiers.get("COLD"),
|
||||
Some(&TierStats {
|
||||
total_size: 7,
|
||||
num_versions: 1,
|
||||
num_objects: 0,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_stats_merge_into_an_untiered_entry() {
|
||||
let mut left = DataUsageEntry::default();
|
||||
let right = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 10,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
|
||||
assert!(left.checked_merge(&right));
|
||||
|
||||
assert_eq!(
|
||||
left.all_tier_stats.expect("tier stats adopted from the merged entry").tiers["WARM"],
|
||||
TierStats {
|
||||
total_size: 10,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_merge_rejects_overflowing_tier_totals() {
|
||||
let mut left = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: u64::MAX,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
let right = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 1,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
|
||||
assert!(!left.checked_merge(&right), "saturating tier totals must not be published");
|
||||
assert_eq!(left.all_tier_stats.expect("left is untouched").tiers["WARM"].total_size, u64::MAX);
|
||||
}
|
||||
|
||||
/// Entry shape released before per-tier accounting, using the derived
|
||||
/// (array) encoding those writers produced.
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct LegacyEntry {
|
||||
children: DataUsageHashMap,
|
||||
size: usize,
|
||||
objects: usize,
|
||||
versions: usize,
|
||||
delete_markers: usize,
|
||||
obj_sizes: SizeHistogram,
|
||||
obj_versions: VersionsHistogram,
|
||||
replication_stats: Option<ReplicationAllStats>,
|
||||
compacted: bool,
|
||||
#[serde(default)]
|
||||
failed_objects: usize,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entries_are_map_encoded_so_appended_fields_stay_readable() {
|
||||
// A derived (array) encoding turns every appended field into a decode
|
||||
// error for readers built before it existed, which would cost a mixed
|
||||
// -version cluster its whole scan cache. Entries must stay map-encoded.
|
||||
let current = tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 3,
|
||||
num_versions: 1,
|
||||
num_objects: 1,
|
||||
},
|
||||
);
|
||||
let mut encoded = Vec::new();
|
||||
current
|
||||
.serialize(&mut rmp_serde::Serializer::new(&mut encoded))
|
||||
.expect("encode current entry");
|
||||
|
||||
let legacy: LegacyEntry = rmp_serde::from_slice(&encoded).expect("legacy reader should ignore the appended field");
|
||||
assert_eq!(legacy.objects, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_array_encoded_entries_still_load() {
|
||||
let legacy = LegacyEntry {
|
||||
children: DataUsageHashMap::default(),
|
||||
size: 12,
|
||||
objects: 3,
|
||||
versions: 4,
|
||||
delete_markers: 1,
|
||||
obj_sizes: SizeHistogram::default(),
|
||||
obj_versions: VersionsHistogram::default(),
|
||||
replication_stats: None,
|
||||
compacted: false,
|
||||
failed_objects: 2,
|
||||
};
|
||||
let mut encoded = Vec::new();
|
||||
legacy
|
||||
.serialize(&mut rmp_serde::Serializer::new(&mut encoded))
|
||||
.expect("encode legacy entry");
|
||||
|
||||
let decoded: DataUsageEntry = rmp_serde::from_slice(&encoded).expect("current reader should default the missing field");
|
||||
|
||||
assert_eq!(decoded.size, 12);
|
||||
assert_eq!(decoded.failed_objects, 2);
|
||||
assert!(decoded.all_tier_stats.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_path_uses_portable_slash_semantics() {
|
||||
for (input, expected) in [
|
||||
@@ -2137,44 +1901,6 @@ mod tests {
|
||||
assert_eq!(info.buckets_count, 2);
|
||||
assert!(info.buckets_usage.is_empty());
|
||||
assert_eq!(info.objects_total_count, 3);
|
||||
assert!(info.tier_stats.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dui_reports_tier_usage_from_the_flattened_tree() {
|
||||
let root_hash = hash_path("root");
|
||||
let bucket_hash = hash_path("bucket-a");
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "root".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
cache.replace_hashed(&root_hash, &None, &DataUsageEntry::default());
|
||||
cache.replace_hashed(
|
||||
&bucket_hash,
|
||||
&Some(root_hash),
|
||||
&tier_entry(
|
||||
"WARM",
|
||||
TierStats {
|
||||
total_size: 40,
|
||||
num_versions: 2,
|
||||
num_objects: 2,
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
let info = cache.dui("root", &["bucket-a".to_string()]);
|
||||
|
||||
assert_eq!(
|
||||
info.tier_stats.expect("child tier usage should roll up to the root").tiers["WARM"],
|
||||
TierStats {
|
||||
total_size: 40,
|
||||
num_versions: 2,
|
||||
num_objects: 2,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -50,21 +50,6 @@ pub const ENV_RUSTFS_BUILD_FEATURES: &str = "RUSTFS_BUILD_FEATURES";
|
||||
pub const TEST_BUCKET: &str = "e2e-test-bucket";
|
||||
const RUSTFS_FULL_FEATURE: &str = "full";
|
||||
|
||||
fn capture_log_path(log_dir: &Path, temp_dir: &str) -> Option<PathBuf> {
|
||||
let temp_name = Path::new(temp_dir).file_name()?.to_string_lossy();
|
||||
Some(log_dir.join(format!("{temp_name}.log")))
|
||||
}
|
||||
|
||||
fn configured_capture_log_path(temp_dir: &str) -> Option<String> {
|
||||
let log_dir = std::env::var_os("RUSTFS_E2E_LOG_DIR")?;
|
||||
if stdfs::create_dir_all(&log_dir).is_err() {
|
||||
warn!(?log_dir, "failed to create configured E2E server log directory");
|
||||
return None;
|
||||
}
|
||||
|
||||
capture_log_path(Path::new(&log_dir), temp_dir).map(|path| path.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
fn build_test_s3_config(endpoint_url: &str, access_key: &str, secret_key: &str, provider_name: &'static str) -> Config {
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, provider_name);
|
||||
let mut config = Config::builder()
|
||||
@@ -376,7 +361,6 @@ impl RustFSTestEnvironment {
|
||||
pub async fn new() -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let temp_dir = format!("/tmp/rustfs_e2e_test_{}", Uuid::new_v4());
|
||||
fs::create_dir_all(&temp_dir).await?;
|
||||
let capture_log_path = configured_capture_log_path(&temp_dir);
|
||||
|
||||
// Use a unique port for each test environment
|
||||
let port = Self::find_available_port().await?;
|
||||
@@ -390,7 +374,7 @@ impl RustFSTestEnvironment {
|
||||
access_key: DEFAULT_ACCESS_KEY.to_string(),
|
||||
secret_key: DEFAULT_SECRET_KEY.to_string(),
|
||||
process: None,
|
||||
capture_log_path,
|
||||
capture_log_path: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -398,7 +382,6 @@ impl RustFSTestEnvironment {
|
||||
pub async fn with_address(address: &str) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let temp_dir = format!("/tmp/rustfs_e2e_test_{}", Uuid::new_v4());
|
||||
fs::create_dir_all(&temp_dir).await?;
|
||||
let capture_log_path = configured_capture_log_path(&temp_dir);
|
||||
|
||||
let url = format!("http://{address}");
|
||||
|
||||
@@ -409,7 +392,7 @@ impl RustFSTestEnvironment {
|
||||
access_key: DEFAULT_ACCESS_KEY.to_string(),
|
||||
secret_key: DEFAULT_SECRET_KEY.to_string(),
|
||||
process: None,
|
||||
capture_log_path,
|
||||
capture_log_path: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1409,14 +1392,6 @@ mod tests {
|
||||
assert_eq!(normalize_rustfs_build_features(" , "), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capture_log_path_uses_temp_directory_basename() {
|
||||
assert_eq!(
|
||||
capture_log_path(Path::new("/tmp/e2e-logs"), "/tmp/rustfs_e2e_test_abc"),
|
||||
Some(PathBuf::from("/tmp/e2e-logs/rustfs_e2e_test_abc.log"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_feature_enables_any_required_feature() {
|
||||
assert!(rustfs_build_feature_enabled(Some("full"), "sftp"));
|
||||
|
||||
@@ -1,351 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression test: a same-key CopyObject that only rewrites metadata must never re-key a
|
||||
//! managed-SSE (SSE-S3 / SSE-KMS) object.
|
||||
//!
|
||||
//! On an **unversioned** bucket the handler marks a same-name copy `metadata_only`, and the
|
||||
//! store layer then updates `xl.meta` in place without touching the data blocks. The handler
|
||||
//! nevertheless strips the source encryption metadata and generates a *fresh* DEK for the
|
||||
//! destination. Combining the two writes "new DEK + old ciphertext": the object is permanently
|
||||
//! undecryptable. The fix forces a full data rewrite whenever the copy re-derives managed
|
||||
//! encryption material, so the stored bytes always match the key metadata beside them.
|
||||
//!
|
||||
//! Companion to `copy_object_version_restore_sse_test` (issue #4238), which pins the same
|
||||
//! invariant for the versioned historical-restore path.
|
||||
|
||||
use super::common::{LocalKMSTestEnvironment, create_key_with_specific_id};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
MetadataDirective, ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration,
|
||||
ServerSideEncryptionRule,
|
||||
};
|
||||
use serial_test::serial;
|
||||
use tracing::info;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() {
|
||||
init_logging();
|
||||
info!("same-key CopyObject with REPLACE metadata must not re-key an SSE-S3 object");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await.expect("failed to create local KMS env");
|
||||
let default_key_id = "rustfs-e2e-test-default-key";
|
||||
let keys_dir = kms_env.kms_keys_dir.clone();
|
||||
create_key_with_specific_id(&keys_dir, default_key_id)
|
||||
.await
|
||||
.expect("failed to create local KMS key");
|
||||
kms_env
|
||||
.base_env
|
||||
.start_rustfs_server_with_env(
|
||||
vec![
|
||||
"--kms-enable",
|
||||
"--kms-backend",
|
||||
"local",
|
||||
"--kms-key-dir",
|
||||
&keys_dir,
|
||||
"--kms-default-key-id",
|
||||
default_key_id,
|
||||
],
|
||||
&[("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true")],
|
||||
)
|
||||
.await
|
||||
.expect("failed to start RustFS with local KMS");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let client = kms_env.base_env.create_s3_client();
|
||||
// Deliberately an UNVERSIONED bucket: that is the branch where the store layer can service
|
||||
// the self-copy as a pure metadata update.
|
||||
let bucket = "copy-object-self-copy-sse-test";
|
||||
let key = "secrets/report.txt";
|
||||
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("failed to create bucket");
|
||||
|
||||
// Content long enough that a truncated/garbled decrypt cannot coincidentally match.
|
||||
let content = b"encrypted payload that must survive a metadata-only self copy -- 0123456789";
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.content_type("text/plain; charset=utf-8")
|
||||
.metadata("stage", "before")
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.body(ByteStream::from_static(content))
|
||||
.send()
|
||||
.await
|
||||
.expect("PUT failed");
|
||||
assert_eq!(put.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
|
||||
|
||||
// Copy the object onto itself, replacing user metadata. This is the `mc cp --attr` /
|
||||
// "edit metadata in place" shape that AWS supports on an existing object.
|
||||
let copy_out = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.metadata_directive(MetadataDirective::Replace)
|
||||
.content_type("text/plain; charset=utf-8")
|
||||
.metadata("stage", "after")
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.send()
|
||||
.await
|
||||
.expect("same-key CopyObject with REPLACE metadata must succeed");
|
||||
assert_eq!(copy_out.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
|
||||
|
||||
// The object must still decrypt to the original plaintext. Before the fix the stored
|
||||
// ciphertext was left untouched while the metadata carried a brand-new DEK, so this GET
|
||||
// either failed outright or returned garbage.
|
||||
let get = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect("GET after self-copy failed: the object was re-keyed without rewriting the ciphertext");
|
||||
assert_eq!(get.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
|
||||
assert_eq!(
|
||||
get.metadata().and_then(|m| m.get("stage")),
|
||||
Some(&"after".to_string()),
|
||||
"REPLACE metadata must take effect"
|
||||
);
|
||||
let body = get.body.collect().await.expect("collect body").into_bytes();
|
||||
assert_eq!(
|
||||
body.as_ref(),
|
||||
content,
|
||||
"object must still decrypt to the original plaintext after a metadata-only self copy"
|
||||
);
|
||||
|
||||
kms_env.base_env.stop_server();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() {
|
||||
init_logging();
|
||||
info!("same-key CopyObject that drops SSE must rewrite the data, not orphan the ciphertext");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await.expect("failed to create local KMS env");
|
||||
let default_key_id = "rustfs-e2e-test-default-key";
|
||||
let keys_dir = kms_env.kms_keys_dir.clone();
|
||||
create_key_with_specific_id(&keys_dir, default_key_id)
|
||||
.await
|
||||
.expect("failed to create local KMS key");
|
||||
kms_env
|
||||
.base_env
|
||||
.start_rustfs_server_with_env(
|
||||
vec![
|
||||
"--kms-enable",
|
||||
"--kms-backend",
|
||||
"local",
|
||||
"--kms-key-dir",
|
||||
&keys_dir,
|
||||
"--kms-default-key-id",
|
||||
default_key_id,
|
||||
],
|
||||
&[("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true")],
|
||||
)
|
||||
.await
|
||||
.expect("failed to start RustFS with local KMS");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let client = kms_env.base_env.create_s3_client();
|
||||
// Unversioned, and deliberately WITHOUT a bucket default-encryption rule, so the copy below
|
||||
// resolves to "no destination encryption".
|
||||
let bucket = "copy-object-self-copy-drop-sse-test";
|
||||
let key = "secrets/report.txt";
|
||||
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("failed to create bucket");
|
||||
|
||||
let content = b"encrypted payload whose ciphertext must not survive as bogus plaintext -- 0123456789";
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.metadata("stage", "before")
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.body(ByteStream::from_static(content))
|
||||
.send()
|
||||
.await
|
||||
.expect("PUT failed");
|
||||
|
||||
// Self-copy with REPLACE and no SSE header. Per AWS semantics the destination ends up
|
||||
// unencrypted. The dangerous outcome is the silent one: the handler strips the source key
|
||||
// metadata while a metadata-only copy leaves the ciphertext in place, so a later GET would
|
||||
// hand back raw ciphertext as if it were plaintext — corruption with no error anywhere.
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.metadata_directive(MetadataDirective::Replace)
|
||||
.metadata("stage", "after")
|
||||
.send()
|
||||
.await
|
||||
.expect("same-key CopyObject dropping SSE must succeed");
|
||||
|
||||
let get = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect("GET after self-copy failed");
|
||||
assert_eq!(
|
||||
get.server_side_encryption(),
|
||||
None,
|
||||
"destination must be unencrypted once the copy drops SSE"
|
||||
);
|
||||
assert_eq!(
|
||||
get.metadata().and_then(|m| m.get("stage")),
|
||||
Some(&"after".to_string()),
|
||||
"REPLACE metadata must take effect"
|
||||
);
|
||||
let body = get.body.collect().await.expect("collect body").into_bytes();
|
||||
assert_eq!(
|
||||
body.as_ref(),
|
||||
content,
|
||||
"object must read back as the original plaintext, not the orphaned ciphertext"
|
||||
);
|
||||
|
||||
kms_env.base_env.stop_server();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decryptable() {
|
||||
init_logging();
|
||||
info!("bucket default encryption must also keep a same-key copy off the metadata-only path");
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await.expect("failed to create local KMS env");
|
||||
let default_key_id = "rustfs-e2e-test-default-key";
|
||||
let keys_dir = kms_env.kms_keys_dir.clone();
|
||||
create_key_with_specific_id(&keys_dir, default_key_id)
|
||||
.await
|
||||
.expect("failed to create local KMS key");
|
||||
kms_env
|
||||
.base_env
|
||||
.start_rustfs_server_with_env(
|
||||
vec![
|
||||
"--kms-enable",
|
||||
"--kms-backend",
|
||||
"local",
|
||||
"--kms-key-dir",
|
||||
&keys_dir,
|
||||
"--kms-default-key-id",
|
||||
default_key_id,
|
||||
],
|
||||
&[("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true")],
|
||||
)
|
||||
.await
|
||||
.expect("failed to start RustFS with local KMS");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let client = kms_env.base_env.create_s3_client();
|
||||
let bucket = "copy-object-self-copy-bucket-default-sse-test";
|
||||
let key = "secrets/report.txt";
|
||||
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect("failed to create bucket");
|
||||
|
||||
// Store the object as PLAINTEXT first: no SSE header and no bucket default rule yet. This is
|
||||
// what makes the case sharp — at copy time the source metadata carries no encryption markers,
|
||||
// so the source-side half of the guard cannot fire.
|
||||
let content = b"plaintext payload that must not be orphaned under a new DEK -- 0123456789";
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.metadata("stage", "before")
|
||||
.body(ByteStream::from_static(content))
|
||||
.send()
|
||||
.await
|
||||
.expect("PUT failed");
|
||||
assert_eq!(put.server_side_encryption(), None, "the object must start out unencrypted");
|
||||
|
||||
// Only NOW enable bucket default encryption. The destination's encryption therefore comes
|
||||
// from the bucket rule and from nowhere else: the source is unencrypted and the copy request
|
||||
// carries no SSE header. A guard that only inspects request headers (MinIO decides
|
||||
// `isTargetEncrypted` from `crypto.S3.IsRequested(r.Header)`) would let this through, yet
|
||||
// `sse_encryption` still mints a fresh DEK from the resolved bucket default — which is why
|
||||
// the guard keys off the *effective* encryption rather than the requested one.
|
||||
let encryption_config = ServerSideEncryptionConfiguration::builder()
|
||||
.rules(
|
||||
ServerSideEncryptionRule::builder()
|
||||
.apply_server_side_encryption_by_default(
|
||||
ServerSideEncryptionByDefault::builder()
|
||||
.sse_algorithm(ServerSideEncryption::Aes256)
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.build()
|
||||
.unwrap();
|
||||
client
|
||||
.put_bucket_encryption()
|
||||
.bucket(bucket)
|
||||
.server_side_encryption_configuration(encryption_config)
|
||||
.send()
|
||||
.await
|
||||
.expect("failed to set bucket default encryption");
|
||||
|
||||
// No SSE header on the copy — the bucket default alone drives the destination encryption.
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.metadata_directive(MetadataDirective::Replace)
|
||||
.metadata("stage", "after")
|
||||
.send()
|
||||
.await
|
||||
.expect("same-key CopyObject under bucket default encryption must succeed");
|
||||
|
||||
let get = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect("GET after self-copy failed: the object was re-keyed without rewriting the ciphertext");
|
||||
assert_eq!(get.server_side_encryption(), Some(&ServerSideEncryption::Aes256));
|
||||
assert_eq!(
|
||||
get.metadata().and_then(|m| m.get("stage")),
|
||||
Some(&"after".to_string()),
|
||||
"REPLACE metadata must take effect"
|
||||
);
|
||||
let body = get.body.collect().await.expect("collect body").into_bytes();
|
||||
assert_eq!(
|
||||
body.as_ref(),
|
||||
content,
|
||||
"object must still decrypt to the original plaintext after a metadata-only self copy"
|
||||
);
|
||||
|
||||
kms_env.base_env.stop_server();
|
||||
}
|
||||
@@ -48,9 +48,6 @@ mod bucket_default_encryption_test;
|
||||
#[cfg(test)]
|
||||
mod encryption_metadata_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod copy_object_self_copy_sse_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod copy_object_version_restore_sse_test;
|
||||
|
||||
|
||||
@@ -734,19 +734,15 @@ async fn get_bucket_replication(
|
||||
}
|
||||
|
||||
async fn enable_bucket_versioning(env: &RustFSTestEnvironment, bucket: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
set_bucket_versioning(env, bucket, BucketVersioningStatus::Enabled).await
|
||||
}
|
||||
|
||||
async fn set_bucket_versioning(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
status: BucketVersioningStatus,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let client = env.create_s3_client();
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(VersioningConfiguration::builder().status(status).build())
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
@@ -1273,48 +1269,6 @@ async fn assert_replication_converged(
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_replication_state<F>(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
description: &str,
|
||||
predicate: F,
|
||||
) -> Result<Vec<ReplicatedVersion>, Box<dyn Error + Send + Sync>>
|
||||
where
|
||||
F: Fn(&[ReplicatedVersion]) -> bool,
|
||||
{
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
let state = list_replication_state(client, bucket).await?;
|
||||
if predicate(&state) {
|
||||
return Ok(state);
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!("{description}; last target state: {state:?}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_replication_key_absent(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
observation: Duration,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let deadline = tokio::time::Instant::now() + observation;
|
||||
loop {
|
||||
let state = list_replication_state(client, bucket).await?;
|
||||
assert!(
|
||||
state.iter().all(|entry| entry.key != key),
|
||||
"unexpected replicated key {bucket}/{key}: {state:?}"
|
||||
);
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Ok(());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_version_body(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
@@ -1991,28 +1945,6 @@ async fn wait_for_remote_target_arn(env: &RustFSTestEnvironment, bucket: &str) -
|
||||
Err(format!("site replication did not configure a remote target for bucket {bucket} in time").into())
|
||||
}
|
||||
|
||||
async fn wait_for_remote_target_health_check(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
arn: &str,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
for _ in 0..40 {
|
||||
let response = list_replication_targets_request(env, Some(bucket)).await?;
|
||||
if response.status() == StatusCode::OK {
|
||||
let targets: Vec<serde_json::Value> = response.json().await?;
|
||||
if targets.iter().any(|target| {
|
||||
target.get("arn").and_then(|value| value.as_str()) == Some(arn)
|
||||
&& target.get("lastOnline").is_some_and(|value| !value.is_null())
|
||||
}) {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
|
||||
Err(format!("replication target {arn} did not complete a successful health check in time").into())
|
||||
}
|
||||
|
||||
async fn site_replication_add(
|
||||
env: &RustFSTestEnvironment,
|
||||
sites: &[PeerSite],
|
||||
@@ -2980,8 +2912,6 @@ async fn test_set_remote_target_allows_self_signed_https_target_with_skip_tls_ve
|
||||
let target_bucket = "replication-self-signed-ok-dst";
|
||||
let object_key = "self-signed-replication.txt";
|
||||
let body = "replication over self-signed https should succeed";
|
||||
let post_health_check_key = "self-signed-replication-after-health-check.txt";
|
||||
let post_health_check_body = "replication should remain available after the target health check";
|
||||
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client
|
||||
@@ -3030,24 +2960,6 @@ async fn test_set_remote_target_allows_self_signed_https_target_with_skip_tls_ve
|
||||
|
||||
wait_for_replicated_object_over_https(&https_client, &target_env, target_bucket, object_key, body).await?;
|
||||
|
||||
wait_for_remote_target_health_check(&source_env, source_bucket, &target_arn).await?;
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(post_health_check_key)
|
||||
.body(ByteStream::from(post_health_check_body.as_bytes().to_vec()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
wait_for_replicated_object_over_https(
|
||||
&https_client,
|
||||
&target_env,
|
||||
target_bucket,
|
||||
post_health_check_key,
|
||||
post_health_check_body,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -3670,326 +3582,6 @@ async fn test_bucket_replication_disabled_delete_marker_does_not_propagate() ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Bounded executable slice for backlog#1620. It deliberately uses real
|
||||
/// source and target RustFS processes and leaves the full MinIO
|
||||
/// interoperability profile for a runner that provisions MinIO credentials
|
||||
/// and a reachable endpoint.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_bucket_replication_acceptance_matrix_local_dual_targets() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
|
||||
let mut target_env_a = RustFSTestEnvironment::new().await?;
|
||||
target_env_a
|
||||
.start_rustfs_server_without_cleanup_with_env(&source_env_vars)
|
||||
.await?;
|
||||
let mut target_env_b = RustFSTestEnvironment::new().await?;
|
||||
target_env_b
|
||||
.start_rustfs_server_without_cleanup_with_env(&source_env_vars)
|
||||
.await?;
|
||||
|
||||
let source_bucket = "replication-acceptance-src";
|
||||
let target_bucket_a = "replication-acceptance-dst-a";
|
||||
let target_bucket_b = "replication-acceptance-dst-b";
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client_a = target_env_a.create_s3_client();
|
||||
let target_client_b = target_env_b.create_s3_client();
|
||||
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client_a.create_bucket().bucket(target_bucket_a).send().await?;
|
||||
target_client_b.create_bucket().bucket(target_bucket_b).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env_a, target_bucket_a).await?;
|
||||
enable_bucket_versioning(&target_env_b, target_bucket_b).await?;
|
||||
|
||||
let target_a_arn = set_replication_target(&source_env, source_bucket, &target_env_a, target_bucket_a).await?;
|
||||
let target_b_arn = set_replication_target(&source_env, source_bucket, &target_env_b, target_bucket_b).await?;
|
||||
let body = format!(
|
||||
r#"<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Role></Role>
|
||||
<Rule>
|
||||
<ID>matrix-prefix</ID>
|
||||
<Priority>100</Priority>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix>prefix/</Prefix></Filter>
|
||||
<DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication>
|
||||
<DeleteReplication><Status>Enabled</Status></DeleteReplication>
|
||||
<ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication>
|
||||
<SourceSelectionCriteria><ReplicaModifications><Status>Enabled</Status></ReplicaModifications></SourceSelectionCriteria>
|
||||
<Destination><Bucket>{target_a_arn}</Bucket></Destination>
|
||||
</Rule>
|
||||
<Rule>
|
||||
<ID>matrix-both-prefix</ID>
|
||||
<Priority>100</Priority>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix>both/</Prefix></Filter>
|
||||
<DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication>
|
||||
<DeleteReplication><Status>Enabled</Status></DeleteReplication>
|
||||
<ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication>
|
||||
<Destination><Bucket>{target_a_arn}</Bucket></Destination>
|
||||
</Rule>
|
||||
<Rule>
|
||||
<ID>matrix-tag</ID>
|
||||
<Priority>100</Priority>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Tag><Key>route</Key><Value>tagged</Value></Tag></Filter>
|
||||
<DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication>
|
||||
<DeleteReplication><Status>Enabled</Status></DeleteReplication>
|
||||
<ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication>
|
||||
<Destination><Bucket>{target_b_arn}</Bucket></Destination>
|
||||
</Rule>
|
||||
<Rule>
|
||||
<ID>matrix-disabled</ID>
|
||||
<Priority>100</Priority>
|
||||
<Status>Disabled</Status>
|
||||
<Filter><Prefix>disabled/</Prefix></Filter>
|
||||
<DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication>
|
||||
<DeleteReplication><Status>Enabled</Status></DeleteReplication>
|
||||
<ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication>
|
||||
<Destination><Bucket>{target_b_arn}</Bucket></Destination>
|
||||
</Rule>
|
||||
<Rule>
|
||||
<ID>matrix-priority-high</ID>
|
||||
<Priority>200</Priority>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix>priority/</Prefix></Filter>
|
||||
<DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication>
|
||||
<DeleteReplication><Status>Enabled</Status></DeleteReplication>
|
||||
<ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication>
|
||||
<Destination><Bucket>{target_a_arn}</Bucket></Destination>
|
||||
</Rule>
|
||||
<Rule>
|
||||
<ID>matrix-priority-low</ID>
|
||||
<Priority>100</Priority>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix>priority/</Prefix></Filter>
|
||||
<DeleteMarkerReplication><Status>Enabled</Status></DeleteMarkerReplication>
|
||||
<DeleteReplication><Status>Enabled</Status></DeleteReplication>
|
||||
<ExistingObjectReplication><Status>Enabled</Status></ExistingObjectReplication>
|
||||
<Destination><Bucket>{target_a_arn}</Bucket></Destination>
|
||||
</Rule>
|
||||
</ReplicationConfiguration>"#
|
||||
);
|
||||
let url = format!("{}/{source_bucket}?replication", source_env.url);
|
||||
let response = signed_request(
|
||||
http::Method::PUT,
|
||||
&url,
|
||||
&source_env.access_key,
|
||||
&source_env.secret_key,
|
||||
Some(body.into_bytes()),
|
||||
Some("application/xml"),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("put replication acceptance matrix failed: {status} {body}").into());
|
||||
}
|
||||
|
||||
let saved_config = get_bucket_replication(&source_env, source_bucket).await?.text().await?;
|
||||
for expected in [
|
||||
"matrix-prefix",
|
||||
"matrix-tag",
|
||||
"matrix-disabled",
|
||||
"matrix-priority-high",
|
||||
"Priority>200",
|
||||
"<Status>Disabled</Status>",
|
||||
"<Key>route</Key>",
|
||||
] {
|
||||
assert!(saved_config.contains(expected), "replication config omitted {expected}: {saved_config}");
|
||||
}
|
||||
|
||||
let version_one = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("prefix/versions.txt")
|
||||
.body(ByteStream::from_static(b"version-one"))
|
||||
.send()
|
||||
.await?;
|
||||
let version_one_id = version_one
|
||||
.version_id()
|
||||
.ok_or("first matrix PUT omitted version ID")?
|
||||
.to_string();
|
||||
let version_two = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("prefix/versions.txt")
|
||||
.body(ByteStream::from_static(b"version-two"))
|
||||
.send()
|
||||
.await?;
|
||||
let version_two_id = version_two
|
||||
.version_id()
|
||||
.ok_or("second matrix PUT omitted version ID")?
|
||||
.to_string();
|
||||
wait_for_replication_state(&target_client_a, target_bucket_a, "prefix object did not replicate", |state| {
|
||||
state
|
||||
.iter()
|
||||
.any(|entry| entry.key == "prefix/versions.txt" && entry.version_id == version_two_id)
|
||||
})
|
||||
.await?;
|
||||
|
||||
let delete_marker = source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key("prefix/versions.txt")
|
||||
.send()
|
||||
.await?;
|
||||
let delete_marker_id = delete_marker
|
||||
.version_id()
|
||||
.ok_or("matrix DELETE omitted marker version ID")?
|
||||
.to_string();
|
||||
wait_for_replication_state(&target_client_a, target_bucket_a, "enabled delete marker did not replicate", |state| {
|
||||
state
|
||||
.iter()
|
||||
.any(|entry| entry.key == "prefix/versions.txt" && entry.delete_marker && entry.version_id == delete_marker_id)
|
||||
})
|
||||
.await?;
|
||||
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key("prefix/versions.txt")
|
||||
.version_id(&version_one_id)
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_replication_state(&target_client_a, target_bucket_a, "enabled version purge did not replicate", |state| {
|
||||
state.iter().all(|entry| entry.version_id != version_one_id)
|
||||
})
|
||||
.await?;
|
||||
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("priority/object.txt")
|
||||
.body(ByteStream::from_static(b"priority winner"))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_user_get_object(&target_client_a, target_bucket_a, "priority/object.txt").await?;
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key("priority/object.txt")
|
||||
.send()
|
||||
.await?;
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
let priority_state = list_replication_state(&target_client_a, target_bucket_a).await?;
|
||||
assert!(
|
||||
priority_state
|
||||
.iter()
|
||||
.any(|entry| entry.key == "priority/object.txt" && !entry.delete_marker),
|
||||
"priority rule did not retain the object version: {priority_state:?}"
|
||||
);
|
||||
assert!(
|
||||
priority_state
|
||||
.iter()
|
||||
.all(|entry| !(entry.key == "priority/object.txt" && entry.delete_marker)),
|
||||
"lower-priority delete-marker rule overrode the higher-priority disabled rule: {priority_state:?}"
|
||||
);
|
||||
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("tagged/object.txt")
|
||||
.tagging("route=tagged")
|
||||
.body(ByteStream::from_static(b"tag filter"))
|
||||
.send()
|
||||
.await?;
|
||||
wait_for_user_get_object(&target_client_b, target_bucket_b, "tagged/object.txt").await?;
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("tagged/no-match.txt")
|
||||
.body(ByteStream::from_static(b"not tagged"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_replication_key_absent(&target_client_b, target_bucket_b, "tagged/no-match.txt", Duration::from_secs(3)).await?;
|
||||
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("both/object.txt")
|
||||
.tagging("route=tagged")
|
||||
.body(ByteStream::from_static(b"mixed targets"))
|
||||
.send()
|
||||
.await?;
|
||||
tokio::try_join!(
|
||||
wait_for_user_get_object(&target_client_a, target_bucket_a, "both/object.txt"),
|
||||
wait_for_user_get_object(&target_client_b, target_bucket_b, "both/object.txt"),
|
||||
)?;
|
||||
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("disabled/object.txt")
|
||||
.body(ByteStream::from_static(b"disabled"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_replication_key_absent(&target_client_a, target_bucket_a, "disabled/object.txt", Duration::from_secs(3)).await?;
|
||||
assert_replication_key_absent(&target_client_b, target_bucket_b, "disabled/object.txt", Duration::from_secs(3)).await?;
|
||||
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key("tagged/object.txt")
|
||||
.send()
|
||||
.await?;
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
let tagged_state = list_replication_state(&target_client_b, target_bucket_b).await?;
|
||||
assert!(
|
||||
tagged_state
|
||||
.iter()
|
||||
.any(|entry| entry.key == "tagged/object.txt" && !entry.delete_marker),
|
||||
"tag rule should retain the replicated data version: {tagged_state:?}"
|
||||
);
|
||||
assert!(
|
||||
tagged_state
|
||||
.iter()
|
||||
.all(|entry| !(entry.key == "tagged/object.txt" && entry.delete_marker)),
|
||||
"tag rule with disabled delete-marker replication created a marker: {tagged_state:?}"
|
||||
);
|
||||
|
||||
set_bucket_versioning(&source_env, source_bucket, BucketVersioningStatus::Suspended).await?;
|
||||
set_bucket_versioning(&target_env_a, target_bucket_a, BucketVersioningStatus::Suspended).await?;
|
||||
let null_put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key("prefix/null.txt")
|
||||
.body(ByteStream::from_static(b"null version"))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(null_put.version_id().is_none(), "suspended source PUT must create a null version");
|
||||
wait_for_replication_state(&target_client_a, target_bucket_a, "null version did not replicate", |state| {
|
||||
state
|
||||
.iter()
|
||||
.any(|entry| entry.key == "prefix/null.txt" && entry.version_id == "null" && !entry.delete_marker)
|
||||
})
|
||||
.await?;
|
||||
let null_delete = source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key("prefix/null.txt")
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
null_delete.version_id().is_none(),
|
||||
"suspended source DELETE must create a null delete marker"
|
||||
);
|
||||
wait_for_replication_state(&target_client_a, target_bucket_a, "null delete marker did not replicate", |state| {
|
||||
state
|
||||
.iter()
|
||||
.any(|entry| entry.key == "prefix/null.txt" && entry.version_id == "null" && entry.delete_marker)
|
||||
})
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_single_bucket_multipart_replication_fans_out_to_multiple_targets() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
|
||||
@@ -178,24 +178,20 @@ pub mod bucket {
|
||||
mrf_backlog_observability_snapshot,
|
||||
};
|
||||
pub use crate::bucket::replication::{
|
||||
BucketReplicationResyncStatus, BucketReplicationStats, BucketStats, DeleteReplicationConfigSnapshot,
|
||||
DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry,
|
||||
MustReplicateOptions, ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
|
||||
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION,
|
||||
REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo,
|
||||
ReplicationBatchAdmission, ReplicationConfig, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
|
||||
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
|
||||
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
||||
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
|
||||
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
|
||||
VersionPurgeStatusType, commit_force_delete_intent, complete_force_delete_intent,
|
||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||
get_global_replication_stats, init_background_replication, invalid_replication_config_status_field,
|
||||
persist_force_delete_intent, read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta,
|
||||
replication_statuses_map, replication_target_arns, resync_start_conflict_id, should_remove_replication_target,
|
||||
should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source, unsupported_replication_config_field,
|
||||
validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
BucketReplicationResyncStatus, BucketStats, DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo,
|
||||
DurableMrfBacklog, DynReplicationPool, MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts,
|
||||
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationConfig, ReplicationConfigurationExt,
|
||||
ReplicationDeleteScheduleInput, ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge,
|
||||
ReplicationObjectIO, ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission,
|
||||
ReplicationScannerBridge, ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage,
|
||||
ReplicationTargetValidationError, ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog,
|
||||
TargetReplicationResyncStatus, VersionPurgeStatusType, delete_replication_state_from_config,
|
||||
delete_replication_version_id, get_global_replication_pool, get_global_replication_stats,
|
||||
init_background_replication, invalid_replication_config_status_field, read_durable_mrf_backlog,
|
||||
replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map, replication_target_arns,
|
||||
resync_start_conflict_id, should_remove_replication_target, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||
unsupported_replication_config_field, validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -44,7 +44,6 @@ use aws_smithy_runtime_api::client::http::{
|
||||
use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
|
||||
use aws_smithy_runtime_api::client::result::ConnectorError;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use futures::{StreamExt, stream};
|
||||
use http::{HeaderMap, HeaderName, HeaderValue, StatusCode, Uri};
|
||||
use hyper_util::client::legacy::Client as HyperClient;
|
||||
use hyper_util::rt::{TokioExecutor, TokioTimer};
|
||||
@@ -69,7 +68,6 @@ use std::path::Path;
|
||||
use std::str::FromStr as _;
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::Weak;
|
||||
use std::time::{Duration, Instant};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tokio::sync::Mutex;
|
||||
@@ -81,7 +79,6 @@ use url::Url;
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEFAULT_HEALTH_CHECK_RELOAD_DURATION: Duration = Duration::from_secs(30 * 60);
|
||||
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||
|
||||
pub static GLOBAL_BUCKET_TARGET_SYS: OnceLock<BucketTargetSys> = OnceLock::new();
|
||||
@@ -258,16 +255,6 @@ fn endpoint_health_key(url: &Url) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
fn target_health(target: &TargetClient) -> EpHealth {
|
||||
let url = target.to_url();
|
||||
EpHealth {
|
||||
endpoint: endpoint_health_key(&url),
|
||||
scheme: url.scheme().to_string(),
|
||||
online: true,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn update_endpoint_health(health: &mut EpHealth, online: bool, latency: Duration, now: OffsetDateTime) {
|
||||
let prev_online = health.online;
|
||||
health.online = online;
|
||||
@@ -285,26 +272,14 @@ fn update_endpoint_health(health: &mut EpHealth, online: bool, latency: Duration
|
||||
health.offline_duration += latency;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Clone, Debug)]
|
||||
struct TargetClientBuildProbe {
|
||||
arn: String,
|
||||
started: Arc<tokio::sync::Semaphore>,
|
||||
release: Arc<tokio::sync::Semaphore>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct BucketTargetSys {
|
||||
pub arn_remotes_map: Arc<RwLock<HashMap<String, ArnTarget>>>,
|
||||
pub targets_map: Arc<RwLock<HashMap<String, Vec<BucketTarget>>>>,
|
||||
pub h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
||||
target_h_mutex: Arc<RwLock<HashMap<String, EpHealth>>>,
|
||||
pub hc_client: Arc<HttpClient>,
|
||||
pub a_mutex: Arc<Mutex<HashMap<String, ArnErrs>>>,
|
||||
pub arn_errs_map: Arc<RwLock<HashMap<String, ArnErrs>>>,
|
||||
target_update_mutexes: Arc<Mutex<HashMap<String, Weak<Mutex<()>>>>>,
|
||||
#[cfg(test)]
|
||||
target_client_build_probe: Arc<Mutex<Option<TargetClientBuildProbe>>>,
|
||||
heartbeat_started: OnceLock<()>,
|
||||
}
|
||||
|
||||
@@ -318,13 +293,9 @@ impl BucketTargetSys {
|
||||
arn_remotes_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
targets_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
target_h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
hc_client: Arc::new(HttpClient::new()),
|
||||
a_mutex: Arc::new(Mutex::new(HashMap::new())),
|
||||
arn_errs_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
target_update_mutexes: Arc::new(Mutex::new(HashMap::new())),
|
||||
#[cfg(test)]
|
||||
target_client_build_probe: Arc::new(Mutex::new(None)),
|
||||
heartbeat_started: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
@@ -339,17 +310,6 @@ impl BucketTargetSys {
|
||||
});
|
||||
}
|
||||
|
||||
async fn target_update_mutex(&self, bucket: &str) -> Arc<Mutex<()>> {
|
||||
let mut mutexes = self.target_update_mutexes.lock().await;
|
||||
mutexes.retain(|_, mutex| mutex.strong_count() > 0);
|
||||
if let Some(mutex) = mutexes.get(bucket).and_then(Weak::upgrade) {
|
||||
return mutex;
|
||||
}
|
||||
let mutex = Arc::new(Mutex::new(()));
|
||||
mutexes.insert(bucket.to_string(), Arc::downgrade(&mutex));
|
||||
mutex
|
||||
}
|
||||
|
||||
pub async fn is_offline(&self, url: &Url) -> bool {
|
||||
let key = endpoint_health_key(url);
|
||||
{
|
||||
@@ -358,6 +318,7 @@ impl BucketTargetSys {
|
||||
return !health.online;
|
||||
}
|
||||
}
|
||||
// Initialize health check if not exists
|
||||
self.init_hc(url).await;
|
||||
false
|
||||
}
|
||||
@@ -384,121 +345,42 @@ impl BucketTargetSys {
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) async fn is_target_offline(&self, target: &Arc<TargetClient>) -> bool {
|
||||
// Lock order: arn_remotes_map, then target_h_mutex. A stale client must not
|
||||
// read or initialize the health state of its replacement.
|
||||
let remotes = self.arn_remotes_map.read().await;
|
||||
let Some(current) = remotes.get(&target.arn).and_then(|remote| remote.client.as_ref()) else {
|
||||
return true;
|
||||
};
|
||||
if !Arc::ptr_eq(current, target) {
|
||||
return true;
|
||||
}
|
||||
{
|
||||
let health_map = self.target_h_mutex.read().await;
|
||||
if let Some(health) = health_map.get(&target.arn) {
|
||||
return !health.online;
|
||||
}
|
||||
}
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
let health = health_map.entry(target.arn.clone()).or_insert_with(|| target_health(target));
|
||||
!health.online
|
||||
}
|
||||
|
||||
pub(crate) async fn mark_target_offline(&self, target: &Arc<TargetClient>) {
|
||||
// Lock order: arn_remotes_map, then target_h_mutex. Ignore failures reported
|
||||
// by a client that has already been replaced.
|
||||
let remotes = self.arn_remotes_map.read().await;
|
||||
let Some(current) = remotes.get(&target.arn).and_then(|remote| remote.client.as_ref()) else {
|
||||
return;
|
||||
};
|
||||
if !Arc::ptr_eq(current, target) {
|
||||
return;
|
||||
}
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
let health = health_map.entry(target.arn.clone()).or_insert_with(|| target_health(target));
|
||||
update_endpoint_health(health, false, Duration::from_secs(0), OffsetDateTime::now_utc());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn init_target_health(&self, target: &TargetClient) {
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
health_map.insert(target.arn.clone(), target_health(target));
|
||||
drop(health_map);
|
||||
self.init_hc(&target.to_url()).await;
|
||||
}
|
||||
|
||||
pub async fn heartbeat(&self) {
|
||||
// Probe interval: `RUSTFS_REPL_HEALTH_CHECK_INTERVAL_MS` (default 5000ms,
|
||||
// clamped to >=10ms), read once when the heartbeat task starts.
|
||||
let mut interval = tokio::time::interval(crate::bucket::replication::replication_timing::health_check_interval());
|
||||
loop {
|
||||
interval.tick().await;
|
||||
self.heartbeat_once().await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn heartbeat_once(&self) {
|
||||
let targets = {
|
||||
let remotes = self.arn_remotes_map.read().await;
|
||||
remotes
|
||||
.values()
|
||||
.filter_map(|target| target.client.clone())
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let endpoints = {
|
||||
let health_map = self.h_mutex.read().await;
|
||||
health_map
|
||||
.iter()
|
||||
.map(|(endpoint, health)| (endpoint.clone(), health.scheme.clone()))
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
let checks = stream::iter(targets.into_iter().map(|target| async move {
|
||||
let start = Instant::now();
|
||||
let online = Self::check_endpoint_health(&target).await;
|
||||
(target, online, start.elapsed())
|
||||
}));
|
||||
let mut checks = checks.buffer_unordered(MAX_CONCURRENT_TARGET_HEALTH_CHECKS);
|
||||
let mut endpoint_checks = HashMap::<String, (String, bool, Duration)>::new();
|
||||
for (endpoint, scheme) in endpoints {
|
||||
// Perform health check
|
||||
let start = Instant::now();
|
||||
let online = self.check_endpoint_health(&endpoint, &scheme).await;
|
||||
let duration = start.elapsed();
|
||||
|
||||
while let Some((target, online, duration)) = checks.next().await {
|
||||
let url = target.to_url();
|
||||
|
||||
{
|
||||
// Lock order: arn_remotes_map, then target_h_mutex. Keeping the remote
|
||||
// read guard prevents a replaced client from receiving stale health.
|
||||
let remotes = self.arn_remotes_map.read().await;
|
||||
let Some(current) = remotes.get(&target.arn).and_then(|remote| remote.client.as_ref()) else {
|
||||
continue;
|
||||
};
|
||||
if !Arc::ptr_eq(current, &target) {
|
||||
continue;
|
||||
{
|
||||
let mut health_map = self.h_mutex.write().await;
|
||||
if let Some(health) = health_map.get_mut(&endpoint) {
|
||||
update_endpoint_health(health, online, duration, OffsetDateTime::now_utc());
|
||||
}
|
||||
}
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
let health = health_map.entry(target.arn.clone()).or_insert_with(|| target_health(&target));
|
||||
update_endpoint_health(health, online, duration, OffsetDateTime::now_utc());
|
||||
}
|
||||
|
||||
let endpoint = endpoint_health_key(&url);
|
||||
endpoint_checks
|
||||
.entry(endpoint)
|
||||
.and_modify(|(_, endpoint_online, endpoint_duration)| {
|
||||
*endpoint_online |= online;
|
||||
*endpoint_duration = (*endpoint_duration).max(duration);
|
||||
})
|
||||
.or_insert_with(|| (url.scheme().to_string(), online, duration));
|
||||
}
|
||||
|
||||
let mut health_map = self.h_mutex.write().await;
|
||||
for (endpoint, (scheme, online, duration)) in endpoint_checks {
|
||||
let health = health_map.entry(endpoint.clone()).or_insert_with(|| EpHealth {
|
||||
endpoint,
|
||||
scheme,
|
||||
online: true,
|
||||
..Default::default()
|
||||
});
|
||||
update_endpoint_health(health, online, duration, OffsetDateTime::now_utc());
|
||||
}
|
||||
}
|
||||
|
||||
async fn check_endpoint_health(target: &TargetClient) -> bool {
|
||||
match tokio::time::timeout(Duration::from_secs(3), target.client.head_bucket().bucket(&target.bucket).send()).await {
|
||||
Ok(Ok(_)) => true,
|
||||
Ok(Err(err)) => err.raw_response().is_some_and(|response| response.status().as_u16() < 500),
|
||||
async fn check_endpoint_health(&self, endpoint: &str, scheme: &str) -> bool {
|
||||
let scheme = if scheme.is_empty() { "https" } else { scheme };
|
||||
let url = format!("{scheme}://{endpoint}/");
|
||||
match self.hc_client.get(url).timeout(Duration::from_secs(3)).send().await {
|
||||
Ok(response) => response.status().as_u16() < 500,
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
@@ -508,20 +390,15 @@ impl BucketTargetSys {
|
||||
health_map.clone()
|
||||
}
|
||||
|
||||
async fn target_health_stats(&self) -> HashMap<String, EpHealth> {
|
||||
let health_map = self.target_h_mutex.read().await;
|
||||
health_map.clone()
|
||||
}
|
||||
|
||||
pub async fn list_targets(&self, bucket: &str, arn_type: &str) -> Vec<BucketTarget> {
|
||||
let health_stats = self.target_health_stats().await;
|
||||
let health_stats = self.health_stats().await;
|
||||
let mut targets = Vec::new();
|
||||
|
||||
if !bucket.is_empty() {
|
||||
if let Ok(bucket_targets) = self.list_bucket_targets(bucket).await {
|
||||
for mut target in bucket_targets.targets {
|
||||
if arn_type.is_empty() || target.target_type.to_string() == arn_type {
|
||||
if let Some(health) = health_stats.get(&target.arn) {
|
||||
if let Some(health) = health_stats.get(&target.endpoint) {
|
||||
target.total_downtime = health.offline_duration;
|
||||
target.online = health.online;
|
||||
target.last_online = health.last_online;
|
||||
@@ -543,7 +420,7 @@ impl BucketTargetSys {
|
||||
for bucket_targets in targets_map.values() {
|
||||
for mut target in bucket_targets.iter().cloned() {
|
||||
if arn_type.is_empty() || target.target_type.to_string() == arn_type {
|
||||
if let Some(health) = health_stats.get(&target.arn) {
|
||||
if let Some(health) = health_stats.get(&target.endpoint) {
|
||||
target.total_downtime = health.offline_duration;
|
||||
target.online = health.online;
|
||||
target.last_online = health.last_online;
|
||||
@@ -576,18 +453,12 @@ impl BucketTargetSys {
|
||||
}
|
||||
|
||||
pub async fn delete(&self, bucket: &str) {
|
||||
let update_mutex = self.target_update_mutex(bucket).await;
|
||||
let _update_guard = update_mutex.lock().await;
|
||||
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex.
|
||||
let mut targets_map = self.targets_map.write().await;
|
||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
|
||||
if let Some(targets) = targets_map.remove(bucket) {
|
||||
for target in targets {
|
||||
arn_remotes_map.remove(&target.arn);
|
||||
health_map.remove(&target.arn);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -819,22 +690,6 @@ impl BucketTargetSys {
|
||||
}
|
||||
|
||||
pub async fn get_remote_target_client_internal(&self, target: &BucketTarget) -> Result<TargetClient, BucketTargetError> {
|
||||
#[cfg(test)]
|
||||
{
|
||||
let probe = self.target_client_build_probe.lock().await.clone();
|
||||
if let Some(probe) = probe
|
||||
&& probe.arn == target.arn
|
||||
{
|
||||
probe.started.add_permits(1);
|
||||
probe
|
||||
.release
|
||||
.acquire()
|
||||
.await
|
||||
.expect("test probe semaphore should remain open")
|
||||
.forget();
|
||||
}
|
||||
}
|
||||
|
||||
let Some(credentials) = &target.credentials else {
|
||||
return Err(BucketTargetError::BucketRemoteTargetNotFound {
|
||||
bucket: target.target_bucket.clone(),
|
||||
@@ -937,25 +792,12 @@ impl BucketTargetSys {
|
||||
}
|
||||
|
||||
pub async fn update_all_targets(&self, bucket: &str, targets: Option<&BucketTargets>) {
|
||||
let update_mutex = self.target_update_mutex(bucket).await;
|
||||
let _update_guard = update_mutex.lock().await;
|
||||
|
||||
let mut clients = Vec::new();
|
||||
if let Some(new_targets) = targets {
|
||||
for target in &new_targets.targets {
|
||||
clients.push((target, self.get_remote_target_client_internal(target).await.map(Arc::new)));
|
||||
}
|
||||
}
|
||||
|
||||
// Lock order: targets_map, then arn_remotes_map, then target_h_mutex.
|
||||
let mut targets_map = self.targets_map.write().await;
|
||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||
let mut health_map = self.target_h_mutex.write().await;
|
||||
// Remove existing targets
|
||||
if let Some(existing_targets) = targets_map.remove(bucket) {
|
||||
for target in existing_targets {
|
||||
arn_remotes_map.remove(&target.arn);
|
||||
health_map.remove(&target.arn);
|
||||
self.update_bandwidth_limit(bucket, &target.arn, 0);
|
||||
}
|
||||
}
|
||||
@@ -964,17 +806,16 @@ impl BucketTargetSys {
|
||||
if let Some(new_targets) = targets
|
||||
&& !new_targets.is_empty()
|
||||
{
|
||||
for (target, client) in clients {
|
||||
match client {
|
||||
for target in &new_targets.targets {
|
||||
match self.get_remote_target_client_internal(target).await {
|
||||
Ok(client) => {
|
||||
arn_remotes_map.insert(
|
||||
target.arn.clone(),
|
||||
ArnTarget {
|
||||
client: Some(client.clone()),
|
||||
client: Some(Arc::new(client)),
|
||||
last_refresh: OffsetDateTime::now_utc(),
|
||||
},
|
||||
);
|
||||
health_map.insert(client.arn.clone(), target_health(&client));
|
||||
self.update_bandwidth_limit(bucket, &target.arn, target.bandwidth_limit);
|
||||
}
|
||||
// The target stays in `targets_map`, so it keeps showing up in
|
||||
@@ -1004,7 +845,25 @@ impl BucketTargetSys {
|
||||
return;
|
||||
}
|
||||
|
||||
self.update_all_targets(bucket, Some(config)).await;
|
||||
for target in config.targets.iter() {
|
||||
let cli = match self.get_remote_target_client_internal(target).await {
|
||||
Ok(cli) => cli,
|
||||
Err(e) => {
|
||||
warn!("get_remote_target_client_internal error:{}", e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
{
|
||||
let arn_target = ArnTarget::with_client(Arc::new(cli));
|
||||
let mut arn_remotes_map = self.arn_remotes_map.write().await;
|
||||
arn_remotes_map.insert(target.arn.clone(), arn_target);
|
||||
}
|
||||
self.update_bandwidth_limit(bucket, &target.arn, target.bandwidth_limit);
|
||||
}
|
||||
|
||||
let mut targets_map = self.targets_map.write().await;
|
||||
targets_map.insert(bucket.to_string(), config.targets.clone());
|
||||
}
|
||||
|
||||
// getRemoteARN gets existing ARN for an endpoint or generates a new one.
|
||||
@@ -2160,7 +2019,7 @@ mod tests {
|
||||
request_uris: Arc::clone(&request_uris),
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let client = s3_client_for_test(443, Some(http_client));
|
||||
let client = s3_client_with_http_client(443, http_client);
|
||||
(
|
||||
TargetClient {
|
||||
endpoint: "https://localhost:443".to_string(),
|
||||
@@ -2179,7 +2038,7 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
fn spawn_https_server(cert: &rcgen::CertifiedKey<rcgen::KeyPair>, requests: usize) -> (u16, std::thread::JoinHandle<()>) {
|
||||
fn spawn_single_request_https_server(cert: &rcgen::CertifiedKey<rcgen::KeyPair>) -> (u16, std::thread::JoinHandle<()>) {
|
||||
use std::io::{Read, Write};
|
||||
|
||||
ensure_rustls_crypto_provider();
|
||||
@@ -2198,116 +2057,42 @@ mod tests {
|
||||
.expect("test TLS server config should build");
|
||||
|
||||
let handle = std::thread::spawn(move || {
|
||||
let server_config = Arc::new(server_config);
|
||||
for _ in 0..requests {
|
||||
let (stream, _) = listener.accept().expect("test TLS client should connect");
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(10)))
|
||||
.expect("test TLS read timeout should configure");
|
||||
stream
|
||||
.set_write_timeout(Some(Duration::from_secs(10)))
|
||||
.expect("test TLS write timeout should configure");
|
||||
let connection = rustls::ServerConnection::new(server_config.clone()).expect("test TLS connection should build");
|
||||
let mut stream = rustls::StreamOwned::new(connection, stream);
|
||||
let mut request = [0_u8; 8192];
|
||||
if stream.read(&mut request).is_err() {
|
||||
continue;
|
||||
}
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.expect("test TLS response should be written");
|
||||
stream.flush().expect("test TLS response should flush");
|
||||
}
|
||||
});
|
||||
|
||||
(port, handle)
|
||||
}
|
||||
|
||||
fn spawn_http_status_server(status: u16) -> (u16, std::thread::JoinHandle<()>) {
|
||||
use std::io::{Read, Write};
|
||||
|
||||
let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("test HTTP listener should bind");
|
||||
let port = listener
|
||||
.local_addr()
|
||||
.expect("test HTTP listener should have an address")
|
||||
.port();
|
||||
let handle = std::thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().expect("test HTTP client should connect");
|
||||
let mut request = [0_u8; 8192];
|
||||
let bytes_read = stream.read(&mut request).expect("test HTTP request should be read");
|
||||
assert!(bytes_read > 0, "test HTTP request should not be empty");
|
||||
write!(stream, "HTTP/1.1 {status} Test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.expect("test HTTP response should be written");
|
||||
});
|
||||
(port, handle)
|
||||
}
|
||||
|
||||
fn spawn_delayed_http_server() -> (
|
||||
u16,
|
||||
tokio::sync::oneshot::Receiver<()>,
|
||||
std::sync::mpsc::Sender<()>,
|
||||
std::thread::JoinHandle<()>,
|
||||
) {
|
||||
use std::io::{Read, Write};
|
||||
|
||||
let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("test HTTP listener should bind");
|
||||
let port = listener
|
||||
.local_addr()
|
||||
.expect("test HTTP listener should have an address")
|
||||
.port();
|
||||
let (accepted_tx, accepted_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let handle = std::thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().expect("test HTTP client should connect");
|
||||
let mut request = [0_u8; 8192];
|
||||
let bytes_read = stream.read(&mut request).expect("test HTTP request should be read");
|
||||
assert!(bytes_read > 0, "test HTTP request should not be empty");
|
||||
accepted_tx.send(()).expect("test should wait for request");
|
||||
release_rx.recv().expect("test should release response");
|
||||
let (stream, _) = listener.accept().expect("test TLS client should connect");
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 500 Test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.expect("test HTTP response should be written");
|
||||
.set_read_timeout(Some(Duration::from_secs(10)))
|
||||
.expect("test TLS read timeout should configure");
|
||||
stream
|
||||
.set_write_timeout(Some(Duration::from_secs(10)))
|
||||
.expect("test TLS write timeout should configure");
|
||||
let connection = rustls::ServerConnection::new(Arc::new(server_config)).expect("test TLS connection should build");
|
||||
let mut stream = rustls::StreamOwned::new(connection, stream);
|
||||
let mut request = [0_u8; 8192];
|
||||
let _ = stream.read(&mut request).expect("test TLS request should be readable");
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.expect("test TLS response should be written");
|
||||
stream.flush().expect("test TLS response should flush");
|
||||
});
|
||||
(port, accepted_rx, release_tx, handle)
|
||||
|
||||
(port, handle)
|
||||
}
|
||||
|
||||
fn s3_client_for_test(port: u16, http_client: Option<SharedHttpClient>) -> S3Client {
|
||||
s3_client_for_endpoint_test(format!("https://localhost:{port}"), http_client)
|
||||
}
|
||||
|
||||
fn s3_client_for_endpoint_test(endpoint: String, http_client: Option<SharedHttpClient>) -> S3Client {
|
||||
fn s3_client_with_http_client(port: u16, http_client: SharedHttpClient) -> S3Client {
|
||||
let credentials = SdkCredentials::builder()
|
||||
.access_key_id("test-access")
|
||||
.secret_access_key("test-secret")
|
||||
.provider_name("bucket_target_tls_test")
|
||||
.build();
|
||||
let mut config = S3Config::builder()
|
||||
.endpoint_url(endpoint)
|
||||
let config = S3Config::builder()
|
||||
.endpoint_url(format!("https://localhost:{port}"))
|
||||
.credentials_provider(SharedCredentialsProvider::new(credentials))
|
||||
.region(SdkRegion::new("us-east-1"))
|
||||
.force_path_style(true)
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest());
|
||||
if let Some(http_client) = http_client {
|
||||
config = config.http_client(http_client);
|
||||
}
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.http_client(http_client)
|
||||
.build();
|
||||
|
||||
S3Client::from_conf(config.build())
|
||||
}
|
||||
|
||||
fn target_client_for_test(arn: &str, endpoint: String, client: S3Client) -> Arc<TargetClient> {
|
||||
Arc::new(TargetClient {
|
||||
endpoint,
|
||||
credentials: None,
|
||||
bucket: "target-bucket".to_string(),
|
||||
storage_class: String::new(),
|
||||
disable_proxy: false,
|
||||
arn: arn.to_string(),
|
||||
reset_id: String::new(),
|
||||
secure: true,
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
})
|
||||
S3Client::from_conf(config)
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2427,32 +2212,17 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_targets_applies_health_stats_by_arn_and_preserves_endpoint_port() {
|
||||
async fn list_targets_applies_health_stats_for_endpoint_with_port() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let arn = "arn:rustfs:replication:us-east-1:bucket:id";
|
||||
let endpoint = "https://remote.example:9443".to_string();
|
||||
let client = target_client_for_test(
|
||||
arn,
|
||||
endpoint.clone(),
|
||||
S3Client::from_conf(
|
||||
S3Config::builder()
|
||||
.endpoint_url(endpoint)
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.build(),
|
||||
),
|
||||
);
|
||||
sys.arn_remotes_map
|
||||
.write()
|
||||
.await
|
||||
.insert(arn.to_string(), ArnTarget::with_client(client.clone()));
|
||||
sys.init_target_health(&client).await;
|
||||
sys.mark_target_offline(&client).await;
|
||||
let url = Url::parse("https://remote.example:9443").expect("url should parse");
|
||||
sys.init_hc(&url).await;
|
||||
sys.mark_offline(&url).await;
|
||||
|
||||
sys.targets_map.write().await.insert(
|
||||
"bucket".to_string(),
|
||||
vec![BucketTarget {
|
||||
endpoint: "remote.example:9443".to_string(),
|
||||
arn: arn.to_string(),
|
||||
arn: "arn:rustfs:replication:us-east-1:bucket:id".to_string(),
|
||||
target_type: BucketTargetType::ReplicationService,
|
||||
..Default::default()
|
||||
}],
|
||||
@@ -2463,97 +2233,6 @@ mod tests {
|
||||
assert_eq!(targets.len(), 1);
|
||||
assert!(!targets[0].online);
|
||||
assert_eq!(targets[0].offline_count, 1);
|
||||
assert_eq!(sys.target_health_stats().await[arn].endpoint, "remote.example:9443");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_health_is_isolated_by_arn_for_shared_endpoint() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let endpoint = "https://shared.example:9443".to_string();
|
||||
let config = || {
|
||||
S3Config::builder()
|
||||
.endpoint_url(endpoint.clone())
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.build()
|
||||
};
|
||||
let first = target_client_for_test("arn:first", endpoint.clone(), S3Client::from_conf(config()));
|
||||
let second = target_client_for_test("arn:second", endpoint.clone(), S3Client::from_conf(config()));
|
||||
|
||||
sys.arn_remotes_map
|
||||
.write()
|
||||
.await
|
||||
.insert(first.arn.clone(), ArnTarget::with_client(first.clone()));
|
||||
sys.arn_remotes_map
|
||||
.write()
|
||||
.await
|
||||
.insert(second.arn.clone(), ArnTarget::with_client(second.clone()));
|
||||
sys.init_target_health(&first).await;
|
||||
sys.init_target_health(&second).await;
|
||||
sys.mark_target_offline(&first).await;
|
||||
|
||||
assert!(sys.is_target_offline(&first).await);
|
||||
assert!(!sys.is_target_offline(&second).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_client_cannot_change_replacement_health_for_same_arn() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let arn = "arn:replacement";
|
||||
let config = |endpoint: &str| {
|
||||
S3Config::builder()
|
||||
.endpoint_url(endpoint)
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.build()
|
||||
};
|
||||
let stale = target_client_for_test(
|
||||
arn,
|
||||
"https://stale.example:9443".to_string(),
|
||||
S3Client::from_conf(config("https://stale.example:9443")),
|
||||
);
|
||||
let current = target_client_for_test(
|
||||
arn,
|
||||
"https://current.example:9443".to_string(),
|
||||
S3Client::from_conf(config("https://current.example:9443")),
|
||||
);
|
||||
sys.arn_remotes_map
|
||||
.write()
|
||||
.await
|
||||
.insert(arn.to_string(), ArnTarget::with_client(current.clone()));
|
||||
sys.init_target_health(¤t).await;
|
||||
|
||||
sys.mark_target_offline(&stale).await;
|
||||
|
||||
assert!(sys.is_target_offline(&stale).await);
|
||||
assert!(!sys.is_target_offline(¤t).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_removes_target_health_by_arn() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let arn = "arn:delete";
|
||||
let endpoint = "https://delete.example:9443".to_string();
|
||||
let client = target_client_for_test(
|
||||
arn,
|
||||
endpoint.clone(),
|
||||
S3Client::from_conf(
|
||||
S3Config::builder()
|
||||
.endpoint_url(endpoint)
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.build(),
|
||||
),
|
||||
);
|
||||
sys.targets_map.write().await.insert(
|
||||
"bucket".to_string(),
|
||||
vec![BucketTarget {
|
||||
arn: arn.to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
);
|
||||
sys.init_target_health(&client).await;
|
||||
|
||||
sys.delete("bucket").await;
|
||||
|
||||
assert!(!sys.target_health_stats().await.contains_key(arn));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2765,216 +2444,6 @@ mod tests {
|
||||
assert_eq!(client.endpoint, "https://192.168.1.10:9000");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_health_check_rejects_untrusted_self_signed_certificate() {
|
||||
let cert = generate_simple_self_signed(vec!["localhost".to_string()]).expect("certificate should generate");
|
||||
let (port, server) = spawn_https_server(&cert, 1);
|
||||
let target =
|
||||
target_client_for_test("arn:default-tls", format!("https://localhost:{port}"), s3_client_for_test(port, None));
|
||||
|
||||
assert!(!BucketTargetSys::check_endpoint_health(&target).await);
|
||||
server.join().expect("test TLS server should stop");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_health_check_honors_skip_tls_verify_client() {
|
||||
let cert = generate_simple_self_signed(vec!["localhost".to_string()]).expect("certificate should generate");
|
||||
let (port, server) = spawn_https_server(&cert, 1);
|
||||
let target = target_client_for_test(
|
||||
"arn:skip-tls",
|
||||
format!("https://localhost:{port}"),
|
||||
s3_client_for_test(port, Some(build_insecure_aws_s3_http_client())),
|
||||
);
|
||||
|
||||
assert!(BucketTargetSys::check_endpoint_health(&target).await);
|
||||
server.join().expect("test TLS server should stop");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_health_check_honors_custom_ca_client() {
|
||||
let cert = generate_simple_self_signed(vec!["localhost".to_string()]).expect("certificate should generate");
|
||||
let http_client = build_aws_s3_http_client_from_target_ca_pem(&cert.cert.pem())
|
||||
.await
|
||||
.expect("custom CA client should build");
|
||||
let (port, server) = spawn_https_server(&cert, 1);
|
||||
let target = target_client_for_test(
|
||||
"arn:custom-ca",
|
||||
format!("https://localhost:{port}"),
|
||||
s3_client_for_test(port, Some(http_client)),
|
||||
);
|
||||
|
||||
assert!(BucketTargetSys::check_endpoint_health(&target).await);
|
||||
server.join().expect("test TLS server should stop");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_health_check_treats_client_errors_as_online_and_server_errors_as_offline() {
|
||||
for (status, expected_online) in [(403, true), (500, false)] {
|
||||
let (port, server) = spawn_http_status_server(status);
|
||||
let endpoint = format!("http://127.0.0.1:{port}");
|
||||
let target = target_client_for_test(
|
||||
&format!("arn:http-{status}"),
|
||||
endpoint.clone(),
|
||||
s3_client_for_endpoint_test(endpoint, None),
|
||||
);
|
||||
|
||||
assert_eq!(BucketTargetSys::check_endpoint_health(&target).await, expected_online);
|
||||
server.join().expect("test HTTP server should stop");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heartbeat_keeps_tls_health_isolated_by_arn_for_shared_endpoint() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let cert = generate_simple_self_signed(vec!["localhost".to_string()]).expect("certificate should generate");
|
||||
let (port, server) = spawn_https_server(&cert, 2);
|
||||
let endpoint = format!("https://localhost:{port}");
|
||||
let strict = target_client_for_test("arn:strict", endpoint.clone(), s3_client_for_test(port, None));
|
||||
let insecure = target_client_for_test(
|
||||
"arn:insecure",
|
||||
endpoint,
|
||||
s3_client_for_test(port, Some(build_insecure_aws_s3_http_client())),
|
||||
);
|
||||
{
|
||||
let mut remotes = sys.arn_remotes_map.write().await;
|
||||
remotes.insert(strict.arn.clone(), ArnTarget::with_client(strict.clone()));
|
||||
remotes.insert(insecure.arn.clone(), ArnTarget::with_client(insecure.clone()));
|
||||
}
|
||||
|
||||
sys.heartbeat_once().await;
|
||||
|
||||
assert!(sys.is_target_offline(&strict).await);
|
||||
assert!(!sys.is_target_offline(&insecure).await);
|
||||
server.join().expect("test TLS server should stop");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heartbeat_discards_result_from_replaced_client_with_same_arn() {
|
||||
let sys = Arc::new(BucketTargetSys::default());
|
||||
let (port, accepted, release, server) = spawn_delayed_http_server();
|
||||
let endpoint = format!("http://127.0.0.1:{port}");
|
||||
let stale = target_client_for_test("arn:replacement", endpoint.clone(), s3_client_for_endpoint_test(endpoint, None));
|
||||
sys.arn_remotes_map
|
||||
.write()
|
||||
.await
|
||||
.insert(stale.arn.clone(), ArnTarget::with_client(stale));
|
||||
let heartbeat_sys = sys.clone();
|
||||
let heartbeat = tokio::spawn(async move { heartbeat_sys.heartbeat_once().await });
|
||||
accepted.await.expect("heartbeat request should reach test server");
|
||||
|
||||
let replacement_endpoint = "https://replacement.example:9443".to_string();
|
||||
let replacement = target_client_for_test(
|
||||
"arn:replacement",
|
||||
replacement_endpoint.clone(),
|
||||
S3Client::from_conf(
|
||||
S3Config::builder()
|
||||
.endpoint_url(replacement_endpoint)
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.build(),
|
||||
),
|
||||
);
|
||||
sys.arn_remotes_map
|
||||
.write()
|
||||
.await
|
||||
.insert(replacement.arn.clone(), ArnTarget::with_client(replacement.clone()));
|
||||
sys.init_target_health(&replacement).await;
|
||||
release.send(()).expect("stale heartbeat response should be released");
|
||||
heartbeat.await.expect("heartbeat should finish");
|
||||
|
||||
assert!(!sys.is_target_offline(&replacement).await);
|
||||
server.join().expect("test HTTP server should stop");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_update_mutex_reuses_live_lock_and_reclaims_dead_entries() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let first = sys.target_update_mutex("first").await;
|
||||
let same = sys.target_update_mutex("first").await;
|
||||
assert!(Arc::ptr_eq(&first, &same));
|
||||
drop(first);
|
||||
drop(same);
|
||||
|
||||
let _second = sys.target_update_mutex("second").await;
|
||||
let mutexes = sys.target_update_mutexes.lock().await;
|
||||
assert!(!mutexes.contains_key("first"));
|
||||
assert!(mutexes.contains_key("second"));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn target_updates_serialize_client_build_through_publication_per_bucket() {
|
||||
let sys = Arc::new(BucketTargetSys::default());
|
||||
let started = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
let release = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
*sys.target_client_build_probe.lock().await = Some(TargetClientBuildProbe {
|
||||
arn: "arn:first".to_string(),
|
||||
started: started.clone(),
|
||||
release: release.clone(),
|
||||
});
|
||||
let target = |arn: &str| BucketTarget {
|
||||
arn: arn.to_string(),
|
||||
endpoint: "192.168.1.10:9000".to_string(),
|
||||
target_bucket: "target-bucket".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
credentials: Some(Credentials {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
session_token: None,
|
||||
expiration: None,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let first_targets = BucketTargets {
|
||||
targets: vec![target("arn:first")],
|
||||
};
|
||||
let second_targets = BucketTargets {
|
||||
targets: vec![target("arn:second")],
|
||||
};
|
||||
let first_sys = sys.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
first_sys.update_all_targets("bucket", Some(&first_targets)).await;
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(2), started.acquire())
|
||||
.await
|
||||
.expect("first client build should start")
|
||||
.expect("first started semaphore should remain open")
|
||||
.forget();
|
||||
let second_started = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
let second_release = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
*sys.target_client_build_probe.lock().await = Some(TargetClientBuildProbe {
|
||||
arn: "arn:second".to_string(),
|
||||
started: second_started.clone(),
|
||||
release: second_release.clone(),
|
||||
});
|
||||
let second_sys = sys.clone();
|
||||
let second = tokio::spawn(async move {
|
||||
second_sys.update_all_targets("bucket", Some(&second_targets)).await;
|
||||
});
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(50), second_started.acquire())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(!sys.targets_map.read().await.contains_key("bucket"));
|
||||
|
||||
release.add_permits(1);
|
||||
tokio::time::timeout(Duration::from_secs(2), first)
|
||||
.await
|
||||
.expect("first target update should not stall")
|
||||
.expect("first target update should finish");
|
||||
tokio::time::timeout(Duration::from_secs(1), second_started.acquire())
|
||||
.await
|
||||
.expect("second client build should start after first update publishes")
|
||||
.expect("second started semaphore should remain open")
|
||||
.forget();
|
||||
second_release.add_permits(1);
|
||||
tokio::time::timeout(Duration::from_secs(2), second)
|
||||
.await
|
||||
.expect("second target update should not stall")
|
||||
.expect("second target update should finish");
|
||||
let targets = sys.targets_map.read().await;
|
||||
assert_eq!(targets["bucket"][0].arn, "arn:second");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replication_trust_store_composes_system_global_and_target_roots_for_real_tls() {
|
||||
let tls_dir = tempfile::tempdir().expect("temporary TLS directory should be created");
|
||||
@@ -2996,8 +2465,8 @@ mod tests {
|
||||
);
|
||||
let http_client = build_aws_s3_http_client_with_trust_store(trust_store).expect("composed TLS client should build");
|
||||
|
||||
let (global_port, global_server) = spawn_https_server(&global_ca, 1);
|
||||
s3_client_for_test(global_port, Some(http_client.clone()))
|
||||
let (global_port, global_server) = spawn_single_request_https_server(&global_ca);
|
||||
s3_client_with_http_client(global_port, http_client.clone())
|
||||
.head_bucket()
|
||||
.bucket("test-bucket")
|
||||
.send()
|
||||
@@ -3005,8 +2474,8 @@ mod tests {
|
||||
.expect("global RUSTFS_TLS_PATH CA should authenticate its TLS server");
|
||||
global_server.join().expect("global CA TLS server should finish");
|
||||
|
||||
let (target_port, target_server) = spawn_https_server(&target_ca, 1);
|
||||
s3_client_for_test(target_port, Some(http_client))
|
||||
let (target_port, target_server) = spawn_single_request_https_server(&target_ca);
|
||||
s3_client_with_http_client(target_port, http_client)
|
||||
.head_bucket()
|
||||
.bucket("test-bucket")
|
||||
.send()
|
||||
|
||||
@@ -33,7 +33,8 @@ use crate::bucket::lifecycle::manual_transition_job::{
|
||||
};
|
||||
use crate::bucket::lifecycle::replication_sink;
|
||||
use crate::bucket::lifecycle::replication_sink::{
|
||||
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, ReplicationStatusType, replication_state_to_filemeta,
|
||||
ReplicateDecision, ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_state_to_filemeta,
|
||||
replication_statuses_map, version_purge_statuses_map,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_delete_journal::{process_tier_delete_journal_entry, run_tier_delete_journal_recovery_loop};
|
||||
use crate::bucket::lifecycle::tier_free_version_recovery::{
|
||||
@@ -42,7 +43,6 @@ use crate::bucket::lifecycle::tier_free_version_recovery::{
|
||||
use crate::bucket::lifecycle::tier_last_day_stats::{DailyAllTierStats, LastDayTierStats};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_tier_idempotent_with_manager_and_identity};
|
||||
use crate::bucket::lifecycle::transition_transaction::run_transition_transaction_recovery_loop;
|
||||
use crate::bucket::versioning::VersioningApi as _;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::client::object_api_utils::new_getobjectreader;
|
||||
use crate::disk::error::DiskError;
|
||||
@@ -4079,12 +4079,12 @@ pub async fn expire_transitioned_object(
|
||||
lc_event: &lifecycle::Event,
|
||||
_src: &LcEventSrc,
|
||||
) -> Result<ObjectInfo, std::io::Error> {
|
||||
let snapshot = lifecycle_delete_config_snapshot(&api, oi)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let (versioned, version_suspended) = snapshot.versioning_config().delete_state(&oi.name);
|
||||
let mut opts = transitioned_object_delete_opts(oi, lc_event.action, versioned, version_suspended);
|
||||
opts.delete_replication_config_snapshot = Some(Arc::new(snapshot));
|
||||
let opts = transitioned_object_delete_opts(
|
||||
oi,
|
||||
lc_event.action,
|
||||
BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
|
||||
BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await,
|
||||
);
|
||||
//let tags = LcAuditEvent::new(src, lcEvent).Tags();
|
||||
if lc_event.action.delete_restored() {
|
||||
return match api.delete_object(&oi.bucket, &oi.name, opts).await {
|
||||
@@ -4692,28 +4692,8 @@ pub async fn apply_expiry_on_non_transitioned_objects(
|
||||
lc_event: &lifecycle::Event,
|
||||
_src: &LcEventSrc,
|
||||
) -> bool {
|
||||
let snapshot = match lifecycle_delete_config_snapshot(&api, oi).await {
|
||||
Ok(snapshot) => snapshot,
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_LIFECYCLE_DELETE_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
operation = "load_delete_config_snapshot",
|
||||
error = ?err,
|
||||
"Lifecycle delete admission failed"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let (versioned, version_suspended) = snapshot.versioning_config().delete_state(&oi.name);
|
||||
let mut opts = ObjectOptions {
|
||||
versioned,
|
||||
version_suspended,
|
||||
expiration: ExpirationOptions { expire: true },
|
||||
delete_replication_config_snapshot: Some(Arc::new(snapshot)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -4721,6 +4701,9 @@ pub async fn apply_expiry_on_non_transitioned_objects(
|
||||
opts.version_id = oi.version_id.map(|v| v.to_string());
|
||||
}
|
||||
|
||||
opts.versioned = BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await;
|
||||
opts.version_suspended = BucketVersioningSys::prefix_suspended(&oi.bucket, &oi.name).await;
|
||||
|
||||
if lc_event.action.delete_all() {
|
||||
opts.delete_prefix = true;
|
||||
opts.delete_prefix_object = true;
|
||||
@@ -4782,17 +4765,12 @@ pub async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &
|
||||
}
|
||||
|
||||
fn lifecycle_deleted_object(oi: &ObjectInfo, dobj: &ObjectInfo) -> DeletedObject {
|
||||
let replication_state = dobj.replication_state();
|
||||
let replication_state = (!replication_state.targets.is_empty() || !replication_state.purge_targets.is_empty())
|
||||
.then(|| replication_state_to_filemeta(&replication_state));
|
||||
|
||||
if dobj.delete_marker {
|
||||
return DeletedObject {
|
||||
object_name: oi.name.clone(),
|
||||
delete_marker: true,
|
||||
delete_marker_version_id: dobj.version_id,
|
||||
delete_marker_mtime: dobj.mod_time.or(oi.mod_time),
|
||||
replication_state,
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
@@ -4803,7 +4781,6 @@ fn lifecycle_deleted_object(oi: &ObjectInfo, dobj: &ObjectInfo) -> DeletedObject
|
||||
delete_marker: false,
|
||||
delete_marker_version_id: oi.version_id,
|
||||
delete_marker_mtime: oi.mod_time,
|
||||
replication_state,
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
@@ -4813,21 +4790,106 @@ fn lifecycle_deleted_object(oi: &ObjectInfo, dobj: &ObjectInfo) -> DeletedObject
|
||||
delete_marker: false,
|
||||
version_id: oi.version_id,
|
||||
delete_marker_mtime: oi.mod_time,
|
||||
replication_state,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn schedule_lifecycle_replication_delete_if_needed(oi: &ObjectInfo, dobj: &ObjectInfo) {
|
||||
let delete_object = lifecycle_deleted_object(oi, dobj);
|
||||
if delete_object.replication_state.is_none() {
|
||||
let mut delete_object = lifecycle_deleted_object(oi, dobj);
|
||||
let version_id = if delete_object.delete_marker {
|
||||
None
|
||||
} else if delete_object.delete_marker_version_id.is_some() {
|
||||
delete_object.delete_marker_version_id
|
||||
} else {
|
||||
delete_object.version_id
|
||||
};
|
||||
|
||||
let replication_state = lifecycle_delete_replication_state(oi, version_id).await;
|
||||
if replication_state.is_none() {
|
||||
return;
|
||||
}
|
||||
|
||||
delete_object.replication_state = replication_state.as_ref().map(replication_state_to_filemeta);
|
||||
|
||||
replication_sink::schedule_delete(oi.bucket.clone(), delete_object).await;
|
||||
}
|
||||
|
||||
async fn lifecycle_delete_config_snapshot(api: &ECStore, oi: &ObjectInfo) -> Result<DeleteReplicationConfigSnapshot, Error> {
|
||||
ReplicationObjectBridge::delete_request_config(api, &oi.bucket).await
|
||||
fn should_reuse_lifecycle_delete_replication_state(oi: &ObjectInfo, version_delete: bool) -> bool {
|
||||
let state = oi.replication_state();
|
||||
if version_delete {
|
||||
oi.version_purge_status == VersionPurgeStatusType::Pending && !state.purge_targets.is_empty()
|
||||
} else {
|
||||
oi.replication_status == ReplicationStatusType::Pending && !state.targets.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
fn lifecycle_version_purge_state_from_completed_targets(oi: &ObjectInfo) -> Option<ReplicationState> {
|
||||
if oi.replication_status != ReplicationStatusType::Completed {
|
||||
return None;
|
||||
}
|
||||
|
||||
let targets = oi.replication_state().targets;
|
||||
if targets.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let pending_status = targets.keys().map(|arn| format!("{arn}=PENDING;")).collect::<String>();
|
||||
|
||||
Some(ReplicationState {
|
||||
replicate_decision_str: oi.replication_decision.clone(),
|
||||
version_purge_status_internal: Some(pending_status.clone()),
|
||||
purge_targets: version_purge_statuses_map(&pending_status),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
async fn lifecycle_delete_replication_state(oi: &ObjectInfo, version_id: Option<Uuid>) -> Option<ReplicationState> {
|
||||
if should_reuse_lifecycle_delete_replication_state(oi, version_id.is_some()) {
|
||||
return Some(oi.replication_state());
|
||||
}
|
||||
|
||||
if version_id.is_some()
|
||||
&& let Some(state) = lifecycle_version_purge_state_from_completed_targets(oi)
|
||||
{
|
||||
return Some(state);
|
||||
}
|
||||
|
||||
let dsc = replication_sink::check_delete_replication(
|
||||
&oi.bucket,
|
||||
ObjectToDelete {
|
||||
object_name: oi.name.clone(),
|
||||
version_id,
|
||||
..Default::default()
|
||||
},
|
||||
oi,
|
||||
&ObjectOptions {
|
||||
version_id: version_id.map(|v| v.to_string()),
|
||||
versioned: BucketVersioningSys::prefix_enabled(&oi.bucket, &oi.name).await,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
if !dsc.replicate_any() {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(replication_state_for_delete(dsc, version_id.is_some()))
|
||||
}
|
||||
|
||||
fn replication_state_for_delete(dsc: ReplicateDecision, version_delete: bool) -> ReplicationState {
|
||||
let pending_status = dsc.pending_status();
|
||||
let mut state = ReplicationState {
|
||||
replicate_decision_str: dsc.to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
if version_delete {
|
||||
state.version_purge_status_internal = pending_status.clone();
|
||||
state.purge_targets = version_purge_statuses_map(pending_status.as_deref().unwrap_or_default());
|
||||
} else {
|
||||
state.replication_status_internal = pending_status.clone();
|
||||
state.targets = replication_statuses_map(pending_status.as_deref().unwrap_or_default());
|
||||
}
|
||||
state
|
||||
}
|
||||
|
||||
pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
|
||||
@@ -4863,14 +4925,16 @@ mod tests {
|
||||
enqueue_transition_with_lifecycle, enqueue_transition_with_lifecycle_report, eval_action_from_lifecycle,
|
||||
jitter_tier_free_version_recovery_delay, lifecycle_action_blocked_by_replication,
|
||||
lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object, lifecycle_replication_blocks_action,
|
||||
lifecycle_rule_has_date_expiration, manual_transition_duration_elapsed, manual_transition_has_more_after_limit,
|
||||
manual_transition_recovery_progress_sink, manual_transition_version_marker, manual_transition_worker_failure_reason,
|
||||
lifecycle_rule_has_date_expiration, lifecycle_version_purge_state_from_completed_targets,
|
||||
manual_transition_duration_elapsed, manual_transition_has_more_after_limit, manual_transition_recovery_progress_sink,
|
||||
manual_transition_version_marker, manual_transition_worker_failure_reason,
|
||||
mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate,
|
||||
persist_manual_transition_job_progress, persist_manual_transition_page_checkpoint, recover_manual_transition_job,
|
||||
recover_manual_transition_jobs, resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity,
|
||||
resolve_transition_queue_send_timeout, resolve_transition_worker_count, resolve_transition_workers_absolute_max,
|
||||
run_tier_free_version_recovery_loop, select_restore_s3_location, set_lifecycle_observability_observer,
|
||||
set_recovered_free_version_enqueue_observer, should_defer_date_expiry_for_recent_config_update,
|
||||
recover_manual_transition_jobs, replication_state_for_delete, resolve_tier_free_version_recovery_enabled,
|
||||
resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, resolve_transition_worker_count,
|
||||
resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop, select_restore_s3_location,
|
||||
set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer,
|
||||
should_defer_date_expiry_for_recent_config_update, should_reuse_lifecycle_delete_replication_state,
|
||||
transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
@@ -4894,7 +4958,9 @@ mod tests {
|
||||
save_manual_transition_scope_admission_if_absent, save_manual_transition_scope_admission_if_current,
|
||||
save_manual_transition_task_if_absent, save_manual_transition_worker_result_if_absent,
|
||||
};
|
||||
use crate::bucket::lifecycle::replication_sink::{ReplicationStatusType, VersionPurgeStatusType};
|
||||
use crate::bucket::lifecycle::replication_sink::{
|
||||
ReplicateDecision, ReplicateTargetDecision, ReplicationStatusType, VersionPurgeStatusType,
|
||||
};
|
||||
use crate::bucket::lifecycle::runtime_boundary as runtime_sources;
|
||||
use crate::bucket::lifecycle::tier_free_version_recovery::{
|
||||
FreeVersionRecoveryStats, RecoveryWalkTestAction, list_tier_free_versions, recover_tier_free_versions_with_cancel,
|
||||
@@ -5928,8 +5994,6 @@ mod tests {
|
||||
backend_identity: Some([1; 32]),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
};
|
||||
|
||||
let err = state
|
||||
@@ -6042,8 +6106,6 @@ mod tests {
|
||||
backend_identity: Some([1; 32]),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
};
|
||||
|
||||
state
|
||||
@@ -7624,46 +7686,6 @@ mod tests {
|
||||
assert_eq!(deleted.object_name, "key");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_deleted_object_hands_off_only_persisted_delete_admission_state() {
|
||||
let source = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "key".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let marker_result = ObjectInfo {
|
||||
delete_marker: true,
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
replication_status_internal: Some("arn:target=PENDING;".to_string()),
|
||||
replication_decision: "arn:target=true".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let marker_delete = lifecycle_deleted_object(&source, &marker_result);
|
||||
let marker_state = marker_delete
|
||||
.replication_state
|
||||
.expect("persisted marker admission must be handed off");
|
||||
assert_eq!(marker_state.replication_status_internal.as_deref(), Some("arn:target=PENDING;"));
|
||||
assert!(marker_state.version_purge_status_internal.is_none());
|
||||
|
||||
let version_result = ObjectInfo {
|
||||
version_purge_status_internal: Some("arn:target=PENDING;".to_string()),
|
||||
replication_decision: "arn:target=true".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let version_delete = lifecycle_deleted_object(
|
||||
&ObjectInfo {
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
..source
|
||||
},
|
||||
&version_result,
|
||||
);
|
||||
let version_state = version_delete
|
||||
.replication_state
|
||||
.expect("persisted version purge admission must be handed off");
|
||||
assert!(version_state.replication_status_internal.is_none());
|
||||
assert_eq!(version_state.version_purge_status_internal.as_deref(), Some("arn:target=PENDING;"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_deleted_object_uses_version_id_for_noncurrent_version_purge() {
|
||||
let version_id = Uuid::new_v4();
|
||||
@@ -7699,6 +7721,77 @@ mod tests {
|
||||
assert_eq!(deleted.version_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_state_for_delete_uses_replication_targets_for_current_delete() {
|
||||
let arn = "arn:aws:s3:::target-bucket";
|
||||
let mut dsc = ReplicateDecision::default();
|
||||
dsc.set(ReplicateTargetDecision::new(arn.to_string(), true, false));
|
||||
|
||||
let state = replication_state_for_delete(dsc, false);
|
||||
|
||||
assert_eq!(state.replication_status_internal.as_deref(), Some(format!("{arn}=PENDING;").as_str()));
|
||||
assert!(state.version_purge_status_internal.is_none());
|
||||
assert!(state.targets.contains_key(arn));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_state_for_delete_uses_purge_targets_for_version_delete() {
|
||||
let arn = "arn:aws:s3:::target-bucket";
|
||||
let mut dsc = ReplicateDecision::default();
|
||||
dsc.set(ReplicateTargetDecision::new(arn.to_string(), true, false));
|
||||
|
||||
let state = replication_state_for_delete(dsc, true);
|
||||
|
||||
assert_eq!(state.version_purge_status_internal.as_deref(), Some(format!("{arn}=PENDING;").as_str()));
|
||||
assert!(state.replication_status_internal.is_none());
|
||||
assert!(state.purge_targets.contains_key(arn));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_delete_replication_state_reuses_only_pending_version_purge_state() {
|
||||
let oi = ObjectInfo {
|
||||
version_purge_status: VersionPurgeStatusType::Pending,
|
||||
version_purge_status_internal: Some("arn:aws:s3:::target=PENDING;".to_string()),
|
||||
replication_decision: "arn:aws:s3:::target=true;false;arn:aws:s3:::target;".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(should_reuse_lifecycle_delete_replication_state(&oi, true));
|
||||
assert!(!should_reuse_lifecycle_delete_replication_state(&oi, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_delete_replication_state_does_not_reuse_put_replication_for_version_delete() {
|
||||
let oi = ObjectInfo {
|
||||
replication_status: ReplicationStatusType::Completed,
|
||||
replication_status_internal: Some("arn:aws:s3:::target=COMPLETED;".to_string()),
|
||||
replication_decision: "arn:aws:s3:::target=true;false;arn:aws:s3:::target;".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!should_reuse_lifecycle_delete_replication_state(&oi, true),
|
||||
"version purges must not reuse plain object replication state from prior PUT/delete-marker replication"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_version_purge_state_from_completed_targets_derives_pending_purge_targets() {
|
||||
let oi = ObjectInfo {
|
||||
replication_status: ReplicationStatusType::Completed,
|
||||
replication_status_internal: Some("arn:aws:s3:::target=COMPLETED;".to_string()),
|
||||
replication_decision: "arn:aws:s3:::target=true;false;arn:aws:s3:::target;".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let state = lifecycle_version_purge_state_from_completed_targets(&oi)
|
||||
.expect("completed replication targets should be convertible into version-purge targets");
|
||||
|
||||
assert_eq!(state.version_purge_status_internal.as_deref(), Some("arn:aws:s3:::target=PENDING;"));
|
||||
assert!(state.purge_targets.contains_key("arn:aws:s3:::target"));
|
||||
assert_eq!(state.replicate_decision_str, oi.replication_decision);
|
||||
}
|
||||
|
||||
fn expired_delete_marker_lifecycle() -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
@@ -10150,8 +10243,6 @@ mod tests {
|
||||
backend_identity: Some(identity),
|
||||
version_id_exact: false,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
};
|
||||
|
||||
let err = crate::bucket::lifecycle::tier_delete_journal::process_tier_delete_journal_entry(ecstore, &je)
|
||||
@@ -10191,8 +10282,6 @@ mod tests {
|
||||
backend_identity: Some(identity),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
};
|
||||
|
||||
crate::set_disk::cleanup_rejected_transition_upload_durably(
|
||||
|
||||
@@ -15,14 +15,15 @@
|
||||
use rustfs_common::metrics::IlmAction;
|
||||
|
||||
use crate::bucket::lifecycle::lifecycle::ObjectOpts;
|
||||
pub(crate) use crate::bucket::replication::ReplicationStatusType;
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::replication::VersionPurgeStatusType;
|
||||
pub(crate) use crate::bucket::replication::ReplicateTargetDecision;
|
||||
pub(crate) use crate::bucket::replication::{
|
||||
DeleteReplicationConfigSnapshot, ReplicationObjectBridge, replication_state_to_filemeta,
|
||||
ReplicateDecision, ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_state_to_filemeta,
|
||||
replication_statuses_map, version_purge_statuses_map,
|
||||
};
|
||||
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationLifecycleConfig};
|
||||
use crate::storage_api_contracts::object::DeletedObject;
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::storage_api_contracts::object::{DeletedObject, ObjectToDelete};
|
||||
|
||||
pub(crate) type LifecycleReplicationConfig = ReplicationLifecycleConfig;
|
||||
|
||||
@@ -56,6 +57,15 @@ pub(crate) fn lifecycle_action_waits_for_replication(action: IlmAction) -> bool
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) async fn check_delete_replication(
|
||||
bucket: &str,
|
||||
object: ObjectToDelete,
|
||||
source: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
) -> ReplicateDecision {
|
||||
ReplicationLifecycleBridge::check_delete_replication(bucket, &object, source, opts).await
|
||||
}
|
||||
|
||||
pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject) {
|
||||
ReplicationLifecycleBridge::schedule_delete(bucket, delete_object).await;
|
||||
}
|
||||
@@ -64,16 +74,7 @@ pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
use rustfs_common::metrics::IlmAction;
|
||||
use s3s::dto::{
|
||||
BucketVersioningStatus, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication,
|
||||
DeleteReplicationStatus, Destination, ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -138,97 +139,4 @@ mod tests {
|
||||
assert!(lifecycle_action_waits_for_replication(IlmAction::TransitionVersionAction));
|
||||
assert!(!lifecycle_action_waits_for_replication(IlmAction::NoneAction));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_delete_admission_uses_marker_and_version_switches_for_all_purges() {
|
||||
for marker_enabled in [false, true] {
|
||||
for purge_enabled in [false, true] {
|
||||
let snapshot = DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
Some(ReplicationConfiguration {
|
||||
role: String::new(),
|
||||
rules: vec![ReplicationRule {
|
||||
delete_marker_replication: Some(DeleteMarkerReplication {
|
||||
status: Some(if marker_enabled {
|
||||
DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)
|
||||
} else {
|
||||
DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::DISABLED)
|
||||
}),
|
||||
}),
|
||||
delete_replication: Some(DeleteReplication {
|
||||
status: if purge_enabled {
|
||||
DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED)
|
||||
} else {
|
||||
DeleteReplicationStatus::from_static(DeleteReplicationStatus::DISABLED)
|
||||
},
|
||||
}),
|
||||
destination: Destination {
|
||||
bucket: "arn:rustfs:replication:target".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
existing_object_replication: None,
|
||||
filter: None,
|
||||
id: Some("lifecycle-delete-switches".to_string()),
|
||||
prefix: Some(String::new()),
|
||||
priority: Some(1),
|
||||
source_selection_criteria: None,
|
||||
status: ReplicationRuleStatus::from_static(ReplicationRuleStatus::ENABLED),
|
||||
}],
|
||||
}),
|
||||
);
|
||||
let source = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "logs/object".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let marker = ObjectToDelete {
|
||||
object_name: source.name.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let marker_opts = ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
ReplicationObjectBridge::check_delete_with_snapshot(&marker, &source, &marker_opts, false, &snapshot)
|
||||
.replicate_any(),
|
||||
marker_enabled
|
||||
);
|
||||
|
||||
for delete_marker in [false, true] {
|
||||
for version_id in [Uuid::new_v4(), Uuid::nil()] {
|
||||
let purge = ObjectToDelete {
|
||||
object_name: source.name.clone(),
|
||||
version_id: Some(version_id),
|
||||
..Default::default()
|
||||
};
|
||||
let purge_source = ObjectInfo {
|
||||
delete_marker,
|
||||
..source.clone()
|
||||
};
|
||||
let purge_opts = ObjectOptions {
|
||||
version_id: Some(version_id.to_string()),
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
ReplicationObjectBridge::check_delete_with_snapshot(
|
||||
&purge,
|
||||
&purge_source,
|
||||
&purge_opts,
|
||||
false,
|
||||
&snapshot,
|
||||
)
|
||||
.replicate_any(),
|
||||
purge_enabled,
|
||||
"delete marker={delete_marker}, version_id={version_id}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,10 +20,8 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::runtime_boundary;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
Jentry, TierDeleteJournalState, TierDeleteSourceIdentity,
|
||||
delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
Jentry, delete_confirmed_transition_candidate_exact_with_manager_and_identity,
|
||||
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
|
||||
};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
@@ -32,7 +30,7 @@ use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader
|
||||
use crate::services::tier::tier::tier_destination_id_from_metadata;
|
||||
use crate::storage_api_contracts::{
|
||||
list::ListOperations as _,
|
||||
object::{DeletedObject, HTTPPreconditions, ObjectIO, ObjectOperations, ObjectToDelete},
|
||||
object::{DeletedObject, ObjectIO, ObjectOperations, ObjectToDelete},
|
||||
range::HTTPRangeSpec,
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
@@ -48,7 +46,6 @@ const TIER_DELETE_JOURNAL_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
|
||||
const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
|
||||
const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
|
||||
const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
|
||||
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
@@ -64,22 +61,13 @@ struct PersistedTierDeleteJournalEntry {
|
||||
version_id_exact: Option<bool>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
version_state: Option<rustfs_filemeta::TransitionVersionState>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
state: Option<TierDeleteJournalState>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
source: Option<TierDeleteSourceIdentity>,
|
||||
}
|
||||
|
||||
impl PersistedTierDeleteJournalEntry {
|
||||
fn from_jentry(je: &Jentry) -> Result<Self> {
|
||||
validate_version_state(je.version_state, &je.version_id, je.version_id_exact)?;
|
||||
let legacy_unknown = je.version_state == rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
let version = if je.source.is_some() || je.state == TierDeleteJournalState::Prepared {
|
||||
if je.backend_identity.is_none() {
|
||||
return Err(Error::other("tier delete transaction is missing its backend identity"));
|
||||
}
|
||||
TIER_DELETE_JOURNAL_TRANSACTION_VERSION
|
||||
} else if legacy_unknown {
|
||||
let version = if legacy_unknown {
|
||||
if je.backend_identity.is_some() {
|
||||
TIER_DELETE_JOURNAL_VERSION
|
||||
} else {
|
||||
@@ -99,10 +87,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
backend_identity: je.backend_identity,
|
||||
version_id_exact: je.version_id_exact.then_some(true),
|
||||
version_state: (!legacy_unknown).then_some(je.version_state),
|
||||
state: (version == TIER_DELETE_JOURNAL_TRANSACTION_VERSION).then_some(je.state),
|
||||
source: (version == TIER_DELETE_JOURNAL_TRANSACTION_VERSION)
|
||||
.then(|| je.source.clone())
|
||||
.flatten(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -117,21 +101,14 @@ impl PersistedTierDeleteJournalEntry {
|
||||
}
|
||||
if self.version != TIER_DELETE_JOURNAL_EXACT_VERSION
|
||||
&& self.version != TIER_DELETE_JOURNAL_STATE_VERSION
|
||||
&& self.version != TIER_DELETE_JOURNAL_TRANSACTION_VERSION
|
||||
&& self.version_id_exact.unwrap_or(false)
|
||||
{
|
||||
return Err(Error::other(
|
||||
"legacy tier delete journal entry has an unsupported exact version constraint",
|
||||
));
|
||||
}
|
||||
let (backend_identity, version_id_exact, version_state, state, source) = match self.version {
|
||||
1 => (
|
||||
None,
|
||||
false,
|
||||
rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
TierDeleteJournalState::Committed,
|
||||
None,
|
||||
),
|
||||
let (backend_identity, version_id_exact, version_state) = match self.version {
|
||||
1 => (None, false, rustfs_filemeta::TransitionVersionState::Unknown),
|
||||
TIER_DELETE_JOURNAL_VERSION => (
|
||||
Some(
|
||||
self.backend_identity
|
||||
@@ -139,8 +116,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
),
|
||||
false,
|
||||
rustfs_filemeta::TransitionVersionState::Unknown,
|
||||
TierDeleteJournalState::Committed,
|
||||
None,
|
||||
),
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION => {
|
||||
if self.version_id.is_empty() || self.version_id_exact != Some(true) {
|
||||
@@ -153,8 +128,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
),
|
||||
true,
|
||||
rustfs_filemeta::TransitionVersionState::Exact,
|
||||
TierDeleteJournalState::Committed,
|
||||
None,
|
||||
)
|
||||
}
|
||||
TIER_DELETE_JOURNAL_STATE_VERSION => {
|
||||
@@ -170,31 +143,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
),
|
||||
exact,
|
||||
state,
|
||||
TierDeleteJournalState::Committed,
|
||||
None,
|
||||
)
|
||||
}
|
||||
TIER_DELETE_JOURNAL_TRANSACTION_VERSION => {
|
||||
let state = self
|
||||
.state
|
||||
.ok_or_else(|| Error::other("tier delete journal v5 entry is missing its state"))?;
|
||||
let source = self
|
||||
.source
|
||||
.ok_or_else(|| Error::other("tier delete journal v5 entry is missing its source identity"))?;
|
||||
let exact = self.version_id_exact.unwrap_or(false);
|
||||
let version_state = self
|
||||
.version_state
|
||||
.ok_or_else(|| Error::other("tier delete journal v5 entry is missing its version state"))?;
|
||||
validate_version_state(version_state, &self.version_id, exact)?;
|
||||
(
|
||||
Some(
|
||||
self.backend_identity
|
||||
.ok_or_else(|| Error::other("tier delete journal v5 entry is missing its backend identity"))?,
|
||||
),
|
||||
exact,
|
||||
version_state,
|
||||
state,
|
||||
Some(source),
|
||||
)
|
||||
}
|
||||
version => return Err(Error::other(format!("unsupported tier delete journal version {version}"))),
|
||||
@@ -206,8 +154,6 @@ impl PersistedTierDeleteJournalEntry {
|
||||
backend_identity,
|
||||
version_id_exact,
|
||||
version_state,
|
||||
state,
|
||||
source,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -255,20 +201,6 @@ pub(crate) fn tier_delete_journal_object_name(je: &Jentry) -> String {
|
||||
hasher.update([0]);
|
||||
hasher.update(b"exact-version-id");
|
||||
}
|
||||
if let Some(source) = &je.source {
|
||||
hasher.update([0]);
|
||||
hasher.update(source.bucket.as_bytes());
|
||||
hasher.update([0]);
|
||||
hasher.update(source.object.as_bytes());
|
||||
hasher.update([0]);
|
||||
hasher.update(source.version_id.as_deref().unwrap_or_default().as_bytes());
|
||||
hasher.update([0]);
|
||||
hasher.update(source.data_dir.as_deref().unwrap_or_default().as_bytes());
|
||||
hasher.update([0]);
|
||||
hasher.update(source.etag.as_deref().unwrap_or_default().as_bytes());
|
||||
hasher.update([0]);
|
||||
hasher.update(source.mod_time.as_deref().unwrap_or_default().as_bytes());
|
||||
}
|
||||
format!(
|
||||
"{TIER_DELETE_JOURNAL_PREFIX}{}.json",
|
||||
rustfs_utils::crypto::hex(hasher.finalize().as_slice())
|
||||
@@ -314,66 +246,6 @@ where
|
||||
.map_err(std::io::Error::other)
|
||||
}
|
||||
|
||||
pub async fn commit_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = http::HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
let mut committed = je.clone();
|
||||
committed.state = TierDeleteJournalState::Committed;
|
||||
persist_tier_delete_journal_entry(api, &committed).await
|
||||
}
|
||||
|
||||
pub async fn abort_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
remove_tier_delete_journal_entry(api, je).await
|
||||
}
|
||||
|
||||
pub async fn abort_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
let name = tier_delete_journal_object_name(je);
|
||||
let (data, metadata) = match config_boundary::read_config_with_metadata(api.clone(), &name, &ObjectOptions::default()).await {
|
||||
Ok(result) => result,
|
||||
Err(Error::ConfigNotFound) | Err(Error::FileNotFound) => return Ok(()),
|
||||
Err(err) => return Err(std::io::Error::other(err)),
|
||||
};
|
||||
let current = decode_tier_delete_journal_entry(&data).map_err(std::io::Error::other)?;
|
||||
if current.state != TierDeleteJournalState::Prepared {
|
||||
return Ok(());
|
||||
}
|
||||
let etag = metadata
|
||||
.etag
|
||||
.ok_or_else(|| std::io::Error::other("prepared tier delete journal has no entity tag"))?;
|
||||
match config_boundary::delete_config_if_match(api, &name, &etag).await {
|
||||
Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
|
||||
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal changed before abort",
|
||||
)),
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn enqueue_committed_tier_delete_journal_entry(je: &Jentry) -> std::io::Result<()> {
|
||||
let expiry_state = runtime_boundary::expiry_state_handle();
|
||||
expiry_state.write().await.enqueue_tier_journal_entry(je)
|
||||
}
|
||||
|
||||
pub async fn remove_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
@@ -392,13 +264,6 @@ where
|
||||
}
|
||||
|
||||
pub async fn process_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
if je.state == TierDeleteJournalState::Prepared {
|
||||
return reconcile_prepared_tier_delete_journal_entry(api, je).await;
|
||||
}
|
||||
process_committed_tier_delete_journal_entry(api, je).await
|
||||
}
|
||||
|
||||
async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
if je.version_state == rustfs_filemeta::TransitionVersionState::Unknown {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
@@ -431,87 +296,6 @@ async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jen
|
||||
remove_tier_delete_journal_entry(api, je).await
|
||||
}
|
||||
|
||||
async fn reconcile_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
let (data, metadata) =
|
||||
config_boundary::read_config_with_metadata(api.clone(), &tier_delete_journal_object_name(je), &ObjectOptions::default())
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let current = decode_tier_delete_journal_entry(&data).map_err(std::io::Error::other)?;
|
||||
if current.state != TierDeleteJournalState::Prepared {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal changed before reconciliation",
|
||||
));
|
||||
}
|
||||
let Some(etag) = metadata.etag else {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"prepared tier delete journal has no entity tag",
|
||||
));
|
||||
};
|
||||
let source = je
|
||||
.source
|
||||
.as_ref()
|
||||
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "prepared tier delete journal has no source"))?;
|
||||
match api
|
||||
.get_object_info(&source.bucket, &source.object, &source.lookup_options())
|
||||
.await
|
||||
{
|
||||
Ok(info) if source.matches(&info) => {
|
||||
match config_boundary::delete_config_if_match(api, &tier_delete_journal_object_name(¤t), &etag).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal changed before abort",
|
||||
)),
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
Ok(_info) if source.has_stable_identity() => {
|
||||
commit_prepared_tier_delete_journal_entry_if_current(api, current, etag).await
|
||||
}
|
||||
Ok(_) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal source identity is not sufficient to confirm deletion",
|
||||
)),
|
||||
Err(Error::ObjectNotFound(_, _)) | Err(Error::FileNotFound) | Err(Error::FileVersionNotFound) => {
|
||||
commit_prepared_tier_delete_journal_entry_if_current(api, current, etag).await
|
||||
}
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn commit_prepared_tier_delete_journal_entry_if_current(
|
||||
api: Arc<ECStore>,
|
||||
mut committed: Jentry,
|
||||
etag: String,
|
||||
) -> std::io::Result<()> {
|
||||
committed.state = TierDeleteJournalState::Committed;
|
||||
let data = encode_tier_delete_journal_entry(&committed).map_err(std::io::Error::other)?;
|
||||
match config_boundary::save_config_with_opts(
|
||||
api.clone(),
|
||||
&tier_delete_journal_object_name(&committed),
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(etag),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => process_committed_tier_delete_journal_entry(api, &committed).await,
|
||||
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal changed before commit",
|
||||
)),
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn recover_tier_delete_journal_entries(
|
||||
api: Arc<ECStore>,
|
||||
limit: usize,
|
||||
@@ -698,13 +482,10 @@ mod tests {
|
||||
decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
|
||||
tier_delete_journal_object_name,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{Jentry, TierDeleteJournalState, TierDeleteSourceIdentity};
|
||||
use crate::bucket::lifecycle::tier_sweeper::Jentry;
|
||||
use crate::error::Result;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn journal_entry() -> Jentry {
|
||||
Jentry {
|
||||
@@ -714,8 +495,6 @@ mod tests {
|
||||
backend_identity: Some([7; 32]),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
state: TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -734,55 +513,6 @@ mod tests {
|
||||
assert_eq!(decoded.version_state, je.version_state);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_transaction_roundtrips_prepared_source_identity() {
|
||||
let mut je = journal_entry();
|
||||
je.state = TierDeleteJournalState::Prepared;
|
||||
je.source = Some(TierDeleteSourceIdentity {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: Some("version".to_string()),
|
||||
versioned: true,
|
||||
version_suspended: false,
|
||||
data_dir: Some("data-dir".to_string()),
|
||||
etag: Some("etag".to_string()),
|
||||
mod_time: Some("mod-time".to_string()),
|
||||
});
|
||||
|
||||
let encoded = encode_tier_delete_journal_entry(&je).expect("prepared transaction should encode");
|
||||
let value: serde_json::Value = serde_json::from_slice(&encoded).expect("transaction should be JSON");
|
||||
assert_eq!(value["version"], serde_json::json!(5));
|
||||
assert_eq!(value["state"], serde_json::json!("Prepared"));
|
||||
assert!(value["source"].is_object());
|
||||
|
||||
let decoded = decode_tier_delete_journal_entry(&encoded).expect("prepared transaction should decode");
|
||||
assert_eq!(decoded.state, TierDeleteJournalState::Prepared);
|
||||
assert_eq!(decoded.source, je.source);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_source_identity_rejects_recreated_object() {
|
||||
let version_id = Uuid::from_u128(1);
|
||||
let data_dir = Uuid::from_u128(2);
|
||||
let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1);
|
||||
let info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(version_id),
|
||||
data_dir: Some(data_dir),
|
||||
mod_time: Some(mod_time),
|
||||
..Default::default()
|
||||
};
|
||||
let source = TierDeleteSourceIdentity::from_object_info("bucket", "object", &info, true, false);
|
||||
assert!(source.matches(&info));
|
||||
|
||||
let recreated = ObjectInfo {
|
||||
data_dir: Some(Uuid::from_u128(3)),
|
||||
..info
|
||||
};
|
||||
assert!(!source.matches(&recreated));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_roundtrips_exact_put_response_constraint() {
|
||||
let mut exact = journal_entry();
|
||||
|
||||
@@ -23,12 +23,10 @@ use crate::bucket::lifecycle::bucket_lifecycle_ops::ExpiryOp;
|
||||
use crate::bucket::lifecycle::lifecycle::{self, ObjectOpts};
|
||||
use crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry;
|
||||
use crate::client::signer_error::error_chain_contains_signer_header_marker;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease};
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
use crate::store::ECStore;
|
||||
use rustfs_utils::get_env_usize;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::any::Any;
|
||||
use std::collections::VecDeque;
|
||||
@@ -259,8 +257,6 @@ impl ObjSweeper {
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
|
||||
),
|
||||
version_state: self.transition_version_state,
|
||||
state: TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
});
|
||||
}
|
||||
None
|
||||
@@ -289,76 +285,6 @@ impl ObjSweeper {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub(crate) enum TierDeleteJournalState {
|
||||
Prepared,
|
||||
Committed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct TierDeleteSourceIdentity {
|
||||
pub(crate) bucket: String,
|
||||
pub(crate) object: String,
|
||||
pub(crate) version_id: Option<String>,
|
||||
pub(crate) versioned: bool,
|
||||
pub(crate) version_suspended: bool,
|
||||
pub(crate) data_dir: Option<String>,
|
||||
pub(crate) etag: Option<String>,
|
||||
pub(crate) mod_time: Option<String>,
|
||||
}
|
||||
|
||||
impl TierDeleteSourceIdentity {
|
||||
pub(crate) fn from_object_info(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
info: &ObjectInfo,
|
||||
versioned: bool,
|
||||
version_suspended: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: info.version_id.map(|id| id.to_string()),
|
||||
versioned,
|
||||
version_suspended,
|
||||
data_dir: info.data_dir.map(|id| id.to_string()),
|
||||
etag: info.etag.clone(),
|
||||
mod_time: info.mod_time.map(|time| time.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn lookup_options(&self) -> crate::object_api::ObjectOptions {
|
||||
crate::object_api::ObjectOptions {
|
||||
version_id: self.version_id.clone(),
|
||||
versioned: self.versioned,
|
||||
version_suspended: self.version_suspended,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn matches(&self, info: &ObjectInfo) -> bool {
|
||||
if self.bucket != info.bucket {
|
||||
return false;
|
||||
}
|
||||
if let Some(version_id) = &self.version_id {
|
||||
return info.version_id.map(|id| id.to_string()).as_deref() == Some(version_id.as_str())
|
||||
&& self.data_dir == info.data_dir.map(|id| id.to_string());
|
||||
}
|
||||
if self.data_dir.is_some() {
|
||||
return self.data_dir == info.data_dir.map(|id| id.to_string());
|
||||
}
|
||||
self.etag.is_some()
|
||||
&& self.etag == info.etag
|
||||
&& self.mod_time.is_some()
|
||||
&& self.mod_time == info.mod_time.map(|time| time.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn has_stable_identity(&self) -> bool {
|
||||
self.version_id.is_some() || self.data_dir.is_some() || (self.etag.is_some() && self.mod_time.is_some())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(unused_assignments)]
|
||||
pub struct Jentry {
|
||||
@@ -368,8 +294,6 @@ pub struct Jentry {
|
||||
pub(crate) backend_identity: Option<TierDestinationId>,
|
||||
pub(crate) version_id_exact: bool,
|
||||
pub(crate) version_state: rustfs_filemeta::TransitionVersionState,
|
||||
pub(crate) state: TierDeleteJournalState,
|
||||
pub(crate) source: Option<TierDeleteSourceIdentity>,
|
||||
}
|
||||
|
||||
impl ExpiryOp for Jentry {
|
||||
@@ -630,48 +554,9 @@ pub fn transitioned_force_delete_journal_entry(
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact
|
||||
),
|
||||
version_state: transition_version_state,
|
||||
state: TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn attach_tier_delete_source(
|
||||
je: &mut Jentry,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
info: &ObjectInfo,
|
||||
versioned: bool,
|
||||
version_suspended: bool,
|
||||
) {
|
||||
je.state = TierDeleteJournalState::Prepared;
|
||||
je.source = Some(TierDeleteSourceIdentity::from_object_info(
|
||||
bucket,
|
||||
object,
|
||||
info,
|
||||
versioned,
|
||||
version_suspended,
|
||||
));
|
||||
}
|
||||
|
||||
pub(crate) fn transitioned_delete_journal_entry_for_source(
|
||||
version_id: Option<Uuid>,
|
||||
versioned: bool,
|
||||
suspended: bool,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
source: &ObjectInfo,
|
||||
) -> Option<Jentry> {
|
||||
let mut je = transitioned_delete_journal_entry(
|
||||
version_id,
|
||||
versioned,
|
||||
suspended,
|
||||
&source.transitioned_object,
|
||||
source.transition_version_state,
|
||||
)?;
|
||||
attach_tier_delete_source(&mut je, bucket, object, source, versioned, suspended);
|
||||
Some(je)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use crate::client::signer_error::invalid_utf8_header_error;
|
||||
|
||||
@@ -45,12 +45,12 @@ mod runtime_boundary;
|
||||
|
||||
pub use datatypes::ResyncStatusType;
|
||||
pub use replication_config_boundary::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigurationExt, ReplicationTargetValidationError, invalid_replication_config_status_field,
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationTargetValidationError, invalid_replication_config_status_field,
|
||||
replication_target_arns, should_remove_replication_target, unsupported_replication_config_field,
|
||||
validate_replication_config_target_arns,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use replication_filemeta_boundary::ReplicateTargetDecision;
|
||||
pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
|
||||
pub use replication_filemeta_boundary::{
|
||||
MrfOpKind, MrfReplicateEntry, REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationState,
|
||||
@@ -70,17 +70,16 @@ pub use replication_object_decision_boundary::{
|
||||
should_use_existing_delete_replication_source,
|
||||
};
|
||||
pub use replication_pool::{
|
||||
DurableMrfBacklog, DynReplicationPool, ReplicationPoolTrait, commit_force_delete_intent, complete_force_delete_intent,
|
||||
get_global_replication_pool, get_global_replication_stats, init_background_replication, persist_force_delete_intent,
|
||||
read_durable_mrf_backlog, resync_start_conflict_id,
|
||||
DurableMrfBacklog, DynReplicationPool, ReplicationPoolTrait, get_global_replication_pool, get_global_replication_stats,
|
||||
init_background_replication, read_durable_mrf_backlog, resync_start_conflict_id,
|
||||
};
|
||||
pub use replication_queue_boundary::{
|
||||
DeletedObjectReplicationInfo, ReplicationBatchAdmission, ReplicationHealQueueResult, ReplicationOperation,
|
||||
ReplicationPriority, ReplicationQueueAdmission,
|
||||
DeletedObjectReplicationInfo, ReplicationHealQueueResult, ReplicationOperation, ReplicationPriority,
|
||||
ReplicationQueueAdmission,
|
||||
};
|
||||
pub use replication_resync_boundary::{BucketReplicationResyncStatus, ResyncOpts, TargetReplicationResyncStatus};
|
||||
pub use replication_scanner_bridge::ReplicationScannerBridge;
|
||||
pub use replication_state::{ReplicationStats, RuntimeReplicationTargetBacklog};
|
||||
pub use replication_stats_boundary::{BucketReplicationStats, BucketStats};
|
||||
pub use replication_stats_boundary::BucketStats;
|
||||
pub use replication_storage_boundary::{ReplicationObjectIO, ReplicationStorage};
|
||||
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
||||
|
||||
@@ -13,9 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::{
|
||||
ObjectOpts, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError, invalid_replication_config_status_field,
|
||||
replication_target_arns, should_remove_replication_target, unsupported_replication_config_field,
|
||||
validate_replication_config_target_arns,
|
||||
ObjectOpts, ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError,
|
||||
invalid_replication_config_status_field, replication_target_arns, should_remove_replication_target,
|
||||
unsupported_replication_config_field, validate_replication_config_target_arns,
|
||||
};
|
||||
|
||||
@@ -30,24 +30,10 @@ impl ReplicationConfigStore {
|
||||
com::read_config(api, file).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::read_config_no_lock(api, file).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::save_config(api, file, data).await
|
||||
}
|
||||
|
||||
pub(crate) async fn save_no_lock<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||
where
|
||||
S: ReplicationObjectIO,
|
||||
{
|
||||
com::save_config_no_lock(api, file, data).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ pub use rustfs_replication::{MrfOpKind, MrfReplicateEntry};
|
||||
pub(crate) use rustfs_replication::{
|
||||
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL_DELETE, ReplicateTargetDecision, ReplicatedInfos,
|
||||
ReplicatedTargetInfo, ReplicationAction, ReplicationWorkerOperation, ResyncDecision, get_replication_state,
|
||||
parse_replicate_decision, replicate_decision_for_admitted_targets, target_reset_header, version_purge_statuses_map,
|
||||
parse_replicate_decision, target_reset_header, version_purge_statuses_map,
|
||||
};
|
||||
pub use rustfs_replication::{
|
||||
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicationState, ReplicationStatusType, ReplicationType,
|
||||
|
||||
@@ -32,7 +32,6 @@ pub(crate) struct ReplicationMetadataStore;
|
||||
|
||||
impl ReplicationMetadataStore {
|
||||
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
|
||||
|
||||
pub(crate) async fn replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
metadata_sys::get_replication_config(bucket).await
|
||||
@@ -110,9 +109,5 @@ mod tests {
|
||||
"buckets/bucket-a/.replication/resync.bin"
|
||||
);
|
||||
assert_eq!(ReplicationMetadataStore::MRF_REPLICATION_FILE, "config/replication/mrf.bin");
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
|
||||
"config/replication/force-delete.bin"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
use super::replication_error_boundary::Result;
|
||||
use super::replication_filemeta_boundary::{ReplicateDecision, ReplicatedTargetInfo, ReplicationStatusType, ReplicationType};
|
||||
use super::replication_filemeta_boundary::{ReplicateDecision, ReplicationStatusType, ReplicationType};
|
||||
use super::replication_metadata_boundary::ReplicationInstanceContext;
|
||||
use super::replication_object_config::{
|
||||
DeleteReplicationConfigSnapshot, check_replicate_delete, check_replicate_delete_strict, check_replicate_delete_with_snapshot,
|
||||
@@ -89,13 +89,6 @@ impl ReplicationObjectBridge {
|
||||
snapshot.has_active_rule(object)
|
||||
}
|
||||
|
||||
pub fn force_delete_target_set(
|
||||
snapshot: &DeleteReplicationConfigSnapshot,
|
||||
prefix: &str,
|
||||
) -> Option<(Vec<String>, time::OffsetDateTime)> {
|
||||
snapshot.force_delete_target_set(prefix)
|
||||
}
|
||||
|
||||
pub fn check_delete_with_snapshot(
|
||||
object: &ObjectToDelete,
|
||||
source: &ObjectInfo,
|
||||
@@ -119,31 +112,6 @@ impl ReplicationObjectBridge {
|
||||
schedule_replication_delete(delete_object).await;
|
||||
}
|
||||
|
||||
pub async fn schedule_deletes(delete_objects: &[DeletedObjectReplicationInfo]) {
|
||||
if let Some(pool) = super::runtime_boundary::replication_pool() {
|
||||
let _ = pool.queue_replica_delete_batch(delete_objects).await;
|
||||
}
|
||||
|
||||
if let Some(stats) = super::runtime_boundary::replication_stats() {
|
||||
for delete_object in delete_objects {
|
||||
if let Some(rs) = &delete_object.delete_object.replication_state {
|
||||
for k in rs.targets.keys() {
|
||||
let ri = ReplicatedTargetInfo {
|
||||
arn: k.clone(),
|
||||
size: 0,
|
||||
duration: std::time::Duration::default(),
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
stats
|
||||
.update(&delete_object.bucket, &ri, ReplicationStatusType::Pending, ReplicationStatusType::Empty)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn schedule_storage_delete(delete_object: DeletedObject, bucket: String, event_type: String) {
|
||||
Self::schedule_delete(DeletedObjectReplicationInfo {
|
||||
delete_object: deleted_object_for_replication(delete_object),
|
||||
@@ -153,19 +121,6 @@ impl ReplicationObjectBridge {
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
pub async fn schedule_storage_deletes(delete_objects: Vec<DeletedObject>, bucket: String, event_type: String) {
|
||||
let delete_objects = delete_objects
|
||||
.into_iter()
|
||||
.map(|delete_object| DeletedObjectReplicationInfo {
|
||||
delete_object: deleted_object_for_replication(delete_object),
|
||||
bucket: bucket.clone(),
|
||||
event_type: event_type.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
Self::schedule_deletes(&delete_objects).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -18,7 +18,6 @@ use crate::bucket::metadata::BucketMetadata;
|
||||
use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
|
||||
use s3s::dto::{BucketVersioningStatus, ReplicationConfiguration, ReplicationRuleStatus, VersioningConfiguration};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use time::OffsetDateTime;
|
||||
use tracing::error;
|
||||
|
||||
use super::replication_config_boundary::{
|
||||
@@ -84,15 +83,6 @@ impl DeleteReplicationConfigSnapshot {
|
||||
.and_then(|metadata| metadata.replication_config.as_ref())
|
||||
}
|
||||
|
||||
pub(crate) fn force_delete_target_set(&self, prefix: &str) -> Option<(Vec<String>, OffsetDateTime)> {
|
||||
self.metadata.as_ref().and_then(|metadata| {
|
||||
metadata
|
||||
.replication_config
|
||||
.as_ref()
|
||||
.map(|config| (config.filter_force_delete_target_arns(prefix), metadata.replication_config_updated_at))
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn has_active_rule(&self, object: &str) -> bool {
|
||||
self.replication_config()
|
||||
.is_some_and(|config| config.has_active_rules(object, true))
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::{
|
||||
DeletedObjectReplicationInfo, ReplicationBatchAdmission, ReplicationHealQueueResult, ReplicationOperation,
|
||||
ReplicationPriority, ReplicationQueueAdmission,
|
||||
DeletedObjectReplicationInfo, ReplicationHealQueueResult, ReplicationOperation, ReplicationPriority,
|
||||
ReplicationQueueAdmission,
|
||||
};
|
||||
pub(crate) use rustfs_replication::{
|
||||
LARGE_WORKER_COUNT, ReplicationBackpressureRecommendation, ReplicationBackpressureState, ReplicationHealQueueAction,
|
||||
|
||||
@@ -19,7 +19,7 @@ use super::replication_error_boundary::{Result, is_err_object_not_found, is_err_
|
||||
use super::replication_event_sink::{EventArgs, send_event, send_local_event};
|
||||
use super::replication_filemeta_boundary::{
|
||||
NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedInfos,
|
||||
ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType, ReplicationType, VersionPurgeStatusType,
|
||||
ReplicatedTargetInfo, ReplicationAction, ReplicationStatusType, ReplicationType, VersionPurgeStatusType,
|
||||
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
};
|
||||
use super::replication_lock_boundary::ReplicationLockTiming;
|
||||
@@ -96,6 +96,7 @@ const EVENT_REPLICATION_FORCE_DELETE_SKIPPED: &str = "replication_force_delete_s
|
||||
const EVENT_RESYNC_TASK_FAILED: &str = "replication_resync_task_failed";
|
||||
const EVENT_RESYNC_TARGET_OPERATION_FAILED: &str = "replication_resync_target_operation_failed";
|
||||
const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_channel_failed";
|
||||
const ERR_REPLICATION_METADATA_COPY_UNSUPPORTED: &str = "metadata-only replication is not implemented";
|
||||
const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
|
||||
"dispatch failure",
|
||||
"timeouterror",
|
||||
@@ -209,7 +210,7 @@ fn is_replication_target_offline_error(err: &(impl Display + ?Sized)) -> bool {
|
||||
.any(|marker| message.contains(marker))
|
||||
}
|
||||
|
||||
async fn mark_replication_target_offline_if_needed(target_client: &Arc<TargetClient>, err: &(impl Display + ?Sized)) {
|
||||
async fn mark_replication_target_offline_if_needed(target_client: &TargetClient, err: &(impl Display + ?Sized)) {
|
||||
if is_replication_target_offline_error(err) {
|
||||
ReplicationTargetStore::mark_target_offline(target_client).await;
|
||||
}
|
||||
@@ -792,7 +793,6 @@ impl ReplicationResyncer {
|
||||
let storage = storage.clone();
|
||||
let results_tx = results_tx.clone();
|
||||
let bucket_name = opts.bucket.clone();
|
||||
let target_arn = opts.arn.clone();
|
||||
|
||||
let f = tokio::spawn(async move {
|
||||
while let Some(mut roi) = rx.recv().await {
|
||||
@@ -820,7 +820,6 @@ impl ReplicationResyncer {
|
||||
bucket: roi.bucket.clone(),
|
||||
event_type: REPLICATE_EXISTING_DELETE.to_string(),
|
||||
op_type: ReplicationType::ExistingObject,
|
||||
target_arn: target_arn.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
replicate_delete(doi, storage.clone()).await;
|
||||
@@ -1385,7 +1384,6 @@ pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicat
|
||||
let mut join_set = JoinSet::new();
|
||||
|
||||
// Process each target
|
||||
let target_arns = dobj.admitted_target_arns();
|
||||
for tgt_entry in dsc.targets_map.values() {
|
||||
// Skip targets that should not be replicated
|
||||
if !tgt_entry.replicate {
|
||||
@@ -1393,7 +1391,7 @@ pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicat
|
||||
}
|
||||
|
||||
// If dobj.TargetArn is not empty string, this is a case of specific target being re-synced.
|
||||
if !target_arns.is_empty() && !target_arns.iter().any(|arn| arn == &tgt_entry.arn) {
|
||||
if !dobj.target_arn.is_empty() && dobj.target_arn != tgt_entry.arn {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1620,8 +1618,7 @@ async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedOb
|
||||
if !tgt_entry.replicate {
|
||||
continue;
|
||||
}
|
||||
let target_arns = dobj.admitted_target_arns();
|
||||
if !target_arns.is_empty() && !target_arns.iter().any(|arn| arn == &tgt_entry.arn) {
|
||||
if !dobj.target_arn.is_empty() && dobj.target_arn != tgt_entry.arn {
|
||||
continue;
|
||||
}
|
||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &tgt_entry.arn).await else {
|
||||
@@ -1642,62 +1639,54 @@ async fn replicate_delete_marker_purge_to_targets(bucket: &str, dobj: &DeletedOb
|
||||
async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &DeletedObjectReplicationInfo, storage: Arc<S>) {
|
||||
let bucket = &dobj.bucket;
|
||||
let object_name = &dobj.delete_object.object_name;
|
||||
let admitted_target_arns = dobj.admitted_target_arns();
|
||||
|
||||
let legacy_target_arns = if admitted_target_arns.is_empty() {
|
||||
match get_replication_config(bucket).await {
|
||||
Ok(Some(config)) => config.filter_target_arns(&ObjectOpts {
|
||||
name: object_name.clone(),
|
||||
let rcfg = match get_replication_config(bucket).await {
|
||||
Ok(Some(config)) => config,
|
||||
Ok(None) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
reason = "replication_config_missing",
|
||||
"Skipping replication force-delete because replication config is missing"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
Ok(None) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
reason = "replication_config_missing",
|
||||
"Skipping replication force-delete because replication config is missing"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
Vec::new()
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
error = %err,
|
||||
reason = "replication_config_lookup_failed",
|
||||
"Skipping replication force-delete because replication config lookup failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
Vec::new()
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
error = %err,
|
||||
reason = "replication_config_lookup_failed",
|
||||
"Skipping replication force-delete because replication config lookup failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
let ns_lock = match storage
|
||||
@@ -1759,18 +1748,19 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
}
|
||||
};
|
||||
|
||||
let tgt_arns = if admitted_target_arns.is_empty() {
|
||||
legacy_target_arns
|
||||
let tgt_arns = if !dobj.target_arn.is_empty() {
|
||||
vec![dobj.target_arn.clone()]
|
||||
} else {
|
||||
admitted_target_arns
|
||||
rcfg.filter_target_arns(&ObjectOpts {
|
||||
name: object_name.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
};
|
||||
|
||||
let mut join_set = JoinSet::new();
|
||||
let mut all_succeeded = true;
|
||||
|
||||
for arn in tgt_arns {
|
||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(bucket, &arn).await else {
|
||||
all_succeeded = false;
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_FORCE_DELETE_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -1820,7 +1810,7 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(e) = tgt_client
|
||||
@@ -1849,46 +1839,24 @@ async fn replicate_force_delete_to_targets<S: ReplicationStorage>(dobj: &Deleted
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
true
|
||||
});
|
||||
}
|
||||
|
||||
while let Some(result) = join_set.join_next().await {
|
||||
match result {
|
||||
Ok(success) => all_succeeded &= success,
|
||||
Err(error) => {
|
||||
all_succeeded = false;
|
||||
error!(
|
||||
event = EVENT_RESYNC_TASK_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
object = %object_name,
|
||||
operation = "force_delete",
|
||||
error = %error,
|
||||
"Replication resync task failed"
|
||||
);
|
||||
}
|
||||
if let Err(e) = result {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TASK_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
object = %object_name,
|
||||
operation = "force_delete",
|
||||
error = %e,
|
||||
"Replication resync task failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if all_succeeded
|
||||
&& let Some(operation_id) = dobj.delete_object.force_delete_id
|
||||
&& let Err(error) = super::replication_pool::complete_force_delete_intent(storage, operation_id).await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
object = %object_name,
|
||||
operation_id = %operation_id,
|
||||
error = %error,
|
||||
"Force-delete replication completed but durable intent cleanup failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn target_delete_version_id(version_id: Uuid, version_purge: bool) -> Option<String> {
|
||||
@@ -2033,11 +2001,61 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
rinfo
|
||||
}
|
||||
|
||||
pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, storage: Arc<S>) -> ReplicationState {
|
||||
pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, storage: Arc<S>) {
|
||||
let bucket = roi.bucket.clone();
|
||||
let object = roi.name.clone();
|
||||
|
||||
let tgt_arns = roi.admitted_target_arns();
|
||||
let cfg = match get_replication_config(&bucket).await {
|
||||
Ok(Some(config)) => config,
|
||||
Ok(None) => {
|
||||
debug!(
|
||||
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
reason = "replication_config_missing",
|
||||
"Skipping replication object because replication config is missing"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: roi.to_object_info(),
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
reason = "replication_config_lookup_failed",
|
||||
error = %err,
|
||||
"Failed to look up replication config for object replication"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: roi.to_object_info(),
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let tgt_arns = cfg.filter_target_arns(&ObjectOpts {
|
||||
name: object.clone(),
|
||||
user_tags: roi.user_tags.clone(),
|
||||
ssec: roi.ssec,
|
||||
op_type: roi.op_type,
|
||||
// ExistingObject ops must respect per-rule ExistingObjectReplicationStatus.
|
||||
// Heal ops intentionally bypass it (repairing a past failure is not an initial sync).
|
||||
existing_object: roi.op_type == ReplicationType::ExistingObject,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
// Acquire a per-object namespace lock so that at most one worker (across all cluster
|
||||
// nodes and MRF retry goroutines) replicates this object version at a time.
|
||||
@@ -2062,7 +2080,7 @@ pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, s
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return roi.replication_state.unwrap_or_default();
|
||||
return;
|
||||
}
|
||||
};
|
||||
let _obj_lock_guard = match obj_ns_lock.get_write_lock(ReplicationLockTiming::acquire_timeout()).await {
|
||||
@@ -2085,7 +2103,7 @@ pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, s
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
return roi.replication_state.unwrap_or_default();
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2161,10 +2179,8 @@ pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, s
|
||||
}
|
||||
}
|
||||
|
||||
let previous_state = roi.replication_state.clone().unwrap_or_default();
|
||||
let merged_state = get_replication_state(&rinfos, &previous_state, roi.version_id.map(|v| v.to_string()));
|
||||
let replication_status = merged_state.composite_replication_status();
|
||||
let new_replication_internal = merged_state.replication_status_internal.clone();
|
||||
let replication_status = rinfos.replication_status();
|
||||
let new_replication_internal = rinfos.replication_status_internal();
|
||||
let mut object_info = roi.to_object_info();
|
||||
|
||||
if roi.replication_status_internal != new_replication_internal || rinfos.replication_resynced() {
|
||||
@@ -2233,8 +2249,6 @@ pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
merged_state
|
||||
}
|
||||
|
||||
trait ReplicateObjectInfoExt {
|
||||
@@ -2846,7 +2860,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
// The target already holds a matching object (reached here only via
|
||||
// the version-id fallback ETag match above) — there is nothing to
|
||||
// copy. Record it as synced and return, instead of falling into the
|
||||
// metadata propagation path below, which previously left
|
||||
// metadata-unsupported failure branch below, which previously left
|
||||
// AWS-style targets permanently FAILED and never converging
|
||||
// (backlog#860 / #799 B11).
|
||||
if self.op_type == ReplicationType::ExistingObject && !tgt_client.reset_id.is_empty() {
|
||||
@@ -2863,71 +2877,10 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
// The target client has no metadata-only operation. Reuse the existing
|
||||
// object transport so metadata changes carry tags and object-lock state
|
||||
// atomically with the source version.
|
||||
let (put_opts, is_multipart) = match replication_put_object_options(&tgt_client.storage_class, &object_info) {
|
||||
Ok((put_opts, is_mp)) => (put_opts, is_mp),
|
||||
Err(e) => {
|
||||
rinfo.error = Some(e.to_string());
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %tgt_client.arn,
|
||||
operation = "build_put_options",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
return rinfo;
|
||||
}
|
||||
};
|
||||
|
||||
let has_tagging_replication = !put_opts.user_tags.is_empty();
|
||||
if let Some(err) = if is_multipart {
|
||||
drop(gr);
|
||||
let result = replicate_object_with_multipart(MultipartReplicationContext {
|
||||
storage: storage.clone(),
|
||||
cli: tgt_client.clone(),
|
||||
src_bucket: &bucket,
|
||||
dst_bucket: &tgt_client.bucket,
|
||||
object: &object,
|
||||
object_info: &object_info,
|
||||
obj_opts: &obj_opts,
|
||||
arn: &rinfo.arn,
|
||||
put_opts,
|
||||
})
|
||||
.await;
|
||||
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
|
||||
if has_tagging_replication {
|
||||
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
|
||||
}
|
||||
result.err()
|
||||
} else {
|
||||
gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
|
||||
let byte_stream = async_read_to_bytestream(gr.stream);
|
||||
let result = tgt_client
|
||||
.put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()));
|
||||
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
|
||||
if has_tagging_replication {
|
||||
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
|
||||
}
|
||||
result.err()
|
||||
} {
|
||||
// action == Metadata: metadata-only replication is not implemented.
|
||||
if replication_action != ReplicationAction::All {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(err.to_string());
|
||||
rinfo.error = Some(ERR_REPLICATION_METADATA_COPY_UNSUPPORTED.to_string());
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -2935,14 +2888,98 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
bucket = %bucket,
|
||||
arn = %tgt_client.arn,
|
||||
object = %object,
|
||||
operation = "put_object",
|
||||
error = ?err,
|
||||
operation = "copy_object_metadata",
|
||||
error = ERR_REPLICATION_METADATA_COPY_UNSUPPORTED,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
|
||||
mark_replication_target_offline_if_needed(&tgt_client, &err).await;
|
||||
return rinfo;
|
||||
} else {
|
||||
let (put_opts, is_multipart) = match replication_put_object_options(&tgt_client.storage_class, &object_info) {
|
||||
Ok((put_opts, is_mp)) => (put_opts, is_mp),
|
||||
Err(e) => {
|
||||
rinfo.error = Some(e.to_string());
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %tgt_client.arn,
|
||||
operation = "build_put_options",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
send_local_event(EventArgs {
|
||||
event_name: EventName::ObjectReplicationNotTracked.to_string(),
|
||||
bucket_name: bucket.clone(),
|
||||
object: object_info,
|
||||
user_agent: "Internal: [Replication]".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
return rinfo;
|
||||
}
|
||||
};
|
||||
|
||||
let has_tagging_replication = !put_opts.user_tags.is_empty();
|
||||
if let Some(err) = if is_multipart {
|
||||
drop(gr);
|
||||
let result = replicate_object_with_multipart(MultipartReplicationContext {
|
||||
storage: storage.clone(),
|
||||
cli: tgt_client.clone(),
|
||||
src_bucket: &bucket,
|
||||
dst_bucket: &tgt_client.bucket,
|
||||
object: &object,
|
||||
object_info: &object_info,
|
||||
obj_opts: &obj_opts,
|
||||
arn: &rinfo.arn,
|
||||
put_opts,
|
||||
})
|
||||
.await;
|
||||
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
|
||||
if has_tagging_replication {
|
||||
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
|
||||
}
|
||||
result.err()
|
||||
} else {
|
||||
gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
|
||||
let byte_stream = async_read_to_bytestream(gr.stream);
|
||||
let result = tgt_client
|
||||
.put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()));
|
||||
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
|
||||
if has_tagging_replication {
|
||||
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
|
||||
}
|
||||
result.err()
|
||||
} {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
rinfo.error = Some(err.to_string());
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %tgt_client.arn,
|
||||
object = %object,
|
||||
operation = "put_object",
|
||||
error = ?err,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
|
||||
mark_replication_target_offline_if_needed(&tgt_client, &err).await;
|
||||
return rinfo;
|
||||
}
|
||||
}
|
||||
|
||||
rinfo
|
||||
@@ -3128,7 +3165,7 @@ mod tests {
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn test_target_client(endpoint: String) -> Arc<TargetClient> {
|
||||
fn test_target_client(endpoint: String) -> TargetClient {
|
||||
let config = aws_sdk_s3::Config::builder()
|
||||
.endpoint_url(endpoint.clone())
|
||||
.region(aws_sdk_s3::config::Region::new("us-east-1"))
|
||||
@@ -3138,7 +3175,7 @@ mod tests {
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.build();
|
||||
|
||||
Arc::new(TargetClient {
|
||||
TargetClient {
|
||||
endpoint,
|
||||
credentials: None,
|
||||
bucket: "target-bucket".to_string(),
|
||||
@@ -3150,11 +3187,7 @@ mod tests {
|
||||
health_check_duration: std::time::Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
|
||||
})
|
||||
}
|
||||
|
||||
async fn register_test_target(target: &Arc<TargetClient>) {
|
||||
ReplicationTargetStore::register_test_target(target).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3170,7 +3203,6 @@ mod tests {
|
||||
async fn replication_target_network_failure_marks_target_offline() {
|
||||
let endpoint = format!("http://network-failure-{}.example:9000", Uuid::new_v4());
|
||||
let target_client = test_target_client(endpoint);
|
||||
register_test_target(&target_client).await;
|
||||
|
||||
assert!(!ReplicationTargetStore::target_is_offline(&target_client).await);
|
||||
|
||||
@@ -3184,7 +3216,6 @@ mod tests {
|
||||
async fn replication_target_service_failure_keeps_target_online() {
|
||||
let endpoint = format!("http://service-failure-{}.example:9000", Uuid::new_v4());
|
||||
let target_client = test_target_client(endpoint);
|
||||
register_test_target(&target_client).await;
|
||||
|
||||
assert!(!ReplicationTargetStore::target_is_offline(&target_client).await);
|
||||
|
||||
|
||||
@@ -12,10 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::BucketStats;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_replication::FailStats;
|
||||
pub(crate) use rustfs_replication::{
|
||||
ActiveWorkerStat, BucketReplicationStat, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache, ReplicationMetricScope,
|
||||
SRMetricsSummary, XferStats,
|
||||
ActiveWorkerStat, BucketReplicationStat, BucketReplicationStats, InQueueMetric, ProxyMetric, ProxyStatsCache, QueueCache,
|
||||
ReplicationMetricScope, SRMetricsSummary, XferStats,
|
||||
};
|
||||
pub use rustfs_replication::{BucketReplicationStats, BucketStats};
|
||||
|
||||
@@ -105,9 +105,6 @@ pub(crate) fn deleted_object_for_replication(delete_object: DeletedObject) -> Re
|
||||
replication_state: delete_object.replication_state.as_ref().map(replication_state_from_filemeta),
|
||||
found: delete_object.found,
|
||||
force_delete: delete_object.force_delete,
|
||||
force_delete_id: delete_object.force_delete_id,
|
||||
force_delete_target_arns: delete_object.force_delete_target_arns,
|
||||
force_delete_generation: delete_object.force_delete_generation,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,11 +24,10 @@ use rustfs_replication::{
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE,
|
||||
AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID,
|
||||
AMZ_STORAGE_CLASS, AMZ_TAG_COUNT, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE,
|
||||
HeaderExt as _, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP,
|
||||
SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_TAGGING_TIMESTAMP, get_str, insert_header_map,
|
||||
is_internal_key,
|
||||
AMZ_OBJECT_TAGGING, AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID, AMZ_STORAGE_CLASS, AMZ_TAG_COUNT,
|
||||
CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_TYPE, HeaderExt as _,
|
||||
SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP, SUFFIX_OBJECTLOCK_RETENTION_TIMESTAMP, SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE,
|
||||
SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_TAGGING_TIMESTAMP, get_str, insert_header_map, is_internal_key,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
@@ -80,48 +79,6 @@ static VALID_SSE_REPLICATION_HEADERS: &[(&str, &str)] = &[
|
||||
];
|
||||
|
||||
const ERR_REPLICATION_MANAGED_SSE_UNSUPPORTED: &str = "managed SSE replication requires target encryption support";
|
||||
const ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED: &str = "replication source contains unsupported encryption metadata";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ReplicationSourceEncryption {
|
||||
Plaintext,
|
||||
SseS3,
|
||||
SseKms,
|
||||
SseC,
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
fn metadata_value<'a>(metadata: &'a HashMap<String, String>, name: &str) -> Option<&'a str> {
|
||||
metadata
|
||||
.iter()
|
||||
.find(|(key, _)| key.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.as_str())
|
||||
}
|
||||
|
||||
fn classify_replication_source_encryption(metadata: &HashMap<String, String>) -> ReplicationSourceEncryption {
|
||||
let is_ssec = replication_object_is_ssec_encrypted(metadata);
|
||||
let sse = metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION);
|
||||
let kms_key_id = metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID);
|
||||
let kms_context = metadata_value(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT);
|
||||
|
||||
if is_ssec {
|
||||
return if sse.is_some() || kms_key_id.is_some() || kms_context.is_some() {
|
||||
ReplicationSourceEncryption::Unsupported
|
||||
} else {
|
||||
ReplicationSourceEncryption::SseC
|
||||
};
|
||||
}
|
||||
|
||||
match sse.map(str::trim) {
|
||||
None if kms_key_id.is_none() && kms_context.is_none() => ReplicationSourceEncryption::Plaintext,
|
||||
Some(value) if value.eq_ignore_ascii_case("AES256") && kms_key_id.is_none() && kms_context.is_none() => {
|
||||
ReplicationSourceEncryption::SseS3
|
||||
}
|
||||
Some(value) if value.eq_ignore_ascii_case("aws:kms") => ReplicationSourceEncryption::SseKms,
|
||||
_ if kms_key_id.is_some() => ReplicationSourceEncryption::SseKms,
|
||||
_ => ReplicationSourceEncryption::Unsupported,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String, String>) -> bool {
|
||||
rustfs_replication::is_ssec_encrypted(user_defined)
|
||||
@@ -138,20 +95,12 @@ impl ReplicationTargetStore {
|
||||
BucketTargetSys::get().get_remote_target_client(bucket, arn).await
|
||||
}
|
||||
|
||||
pub(crate) async fn target_is_offline(target_client: &Arc<TargetClient>) -> bool {
|
||||
BucketTargetSys::get().is_target_offline(target_client).await
|
||||
pub(crate) async fn target_is_offline(target_client: &TargetClient) -> bool {
|
||||
BucketTargetSys::get().is_offline(&target_client.to_url()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn mark_target_offline(target_client: &Arc<TargetClient>) {
|
||||
BucketTargetSys::get().mark_target_offline(target_client).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn register_test_target(target_client: &Arc<TargetClient>) {
|
||||
BucketTargetSys::get().arn_remotes_map.write().await.insert(
|
||||
target_client.arn.clone(),
|
||||
crate::bucket::bucket_target_sys::ArnTarget::with_client(target_client.clone()),
|
||||
);
|
||||
pub(crate) async fn mark_target_offline(target_client: &TargetClient) {
|
||||
BucketTargetSys::get().mark_offline(&target_client.to_url()).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,18 +109,7 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
use rustfs_utils::http::{AMZ_CHECKSUM_TYPE, AMZ_CHECKSUM_TYPE_FULL_OBJECT};
|
||||
|
||||
let mut meta = HashMap::new();
|
||||
let source_encryption = classify_replication_source_encryption(&object_info.user_defined);
|
||||
let is_ssec = matches!(source_encryption, ReplicationSourceEncryption::SseC);
|
||||
|
||||
match source_encryption {
|
||||
ReplicationSourceEncryption::Plaintext | ReplicationSourceEncryption::SseC => {}
|
||||
ReplicationSourceEncryption::SseS3 | ReplicationSourceEncryption::SseKms => {
|
||||
return Err(Error::other(ERR_REPLICATION_MANAGED_SSE_UNSUPPORTED));
|
||||
}
|
||||
ReplicationSourceEncryption::Unsupported => {
|
||||
return Err(Error::other(ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED));
|
||||
}
|
||||
}
|
||||
let is_ssec = replication_object_is_ssec_encrypted(&object_info.user_defined);
|
||||
|
||||
for (key, value) in object_info.user_defined.iter() {
|
||||
let has_valid_sse_header = valid_sse_replication_header(key).is_some();
|
||||
@@ -297,6 +235,20 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
};
|
||||
}
|
||||
|
||||
let has_sse_s3 = object_info
|
||||
.user_defined
|
||||
.get(AMZ_SERVER_SIDE_ENCRYPTION)
|
||||
.is_some_and(|value| value.eq_ignore_ascii_case("AES256"));
|
||||
let has_sse_kms = object_info
|
||||
.user_defined
|
||||
.get(AMZ_SERVER_SIDE_ENCRYPTION)
|
||||
.is_some_and(|value| value.eq_ignore_ascii_case("aws:kms"))
|
||||
|| object_info.user_defined.contains_key(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID);
|
||||
|
||||
if has_sse_s3 || has_sse_kms {
|
||||
return Err(Error::other(ERR_REPLICATION_MANAGED_SSE_UNSUPPORTED));
|
||||
}
|
||||
|
||||
Ok((put_options, is_multipart))
|
||||
}
|
||||
|
||||
@@ -634,46 +586,6 @@ mod tests {
|
||||
assert!(get_header_map(&options.user_metadata, SUFFIX_REPLICATION_SSEC_CRC).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_source_encryption_classification_is_explicit_and_fail_closed() {
|
||||
assert_eq!(
|
||||
classify_replication_source_encryption(&HashMap::new()),
|
||||
ReplicationSourceEncryption::Plaintext
|
||||
);
|
||||
assert_eq!(
|
||||
classify_replication_source_encryption(&HashMap::from([(
|
||||
"x-amz-server-side-encryption".to_string(),
|
||||
"AES256".to_string()
|
||||
)])),
|
||||
ReplicationSourceEncryption::SseS3
|
||||
);
|
||||
assert_eq!(
|
||||
classify_replication_source_encryption(&HashMap::from([(
|
||||
"x-amz-server-side-encryption".to_string(),
|
||||
"AWS:KMS".to_string()
|
||||
)])),
|
||||
ReplicationSourceEncryption::SseKms
|
||||
);
|
||||
assert_eq!(
|
||||
classify_replication_source_encryption(&HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())])),
|
||||
ReplicationSourceEncryption::SseC
|
||||
);
|
||||
assert_eq!(
|
||||
classify_replication_source_encryption(&HashMap::from([(
|
||||
"x-amz-server-side-encryption".to_string(),
|
||||
"unsupported-algorithm".to_string(),
|
||||
)])),
|
||||
ReplicationSourceEncryption::Unsupported
|
||||
);
|
||||
assert_eq!(
|
||||
classify_replication_source_encryption(&HashMap::from([(
|
||||
AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT.to_string(),
|
||||
"opaque-context".to_string(),
|
||||
)])),
|
||||
ReplicationSourceEncryption::Unsupported
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_rejects_sse_s3_until_target_encryption_is_supported() {
|
||||
let object_info = ObjectInfo {
|
||||
@@ -707,25 +619,6 @@ mod tests {
|
||||
assert!(err.to_string().contains(ERR_REPLICATION_MANAGED_SSE_UNSUPPORTED));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_put_options_rejects_unknown_encryption_without_echoing_metadata() {
|
||||
let secret_like_value = "opaque-context-that-must-not-be-logged";
|
||||
let object_info = ObjectInfo {
|
||||
user_defined: Arc::new(HashMap::from([
|
||||
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "unsupported-algorithm".to_string()),
|
||||
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT.to_string(), secret_like_value.to_string()),
|
||||
])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = match replication_put_object_options("", &object_info) {
|
||||
Ok(_) => panic!("unknown encryption must fail closed"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(err.to_string().contains(ERR_REPLICATION_ENCRYPTION_METADATA_UNSUPPORTED));
|
||||
assert!(!err.to_string().contains(secret_like_value));
|
||||
}
|
||||
|
||||
// T3 (#1264): the outbound replication path forwards a stored object checksum into
|
||||
// user_metadata via decrypt_checksums, which is algorithm-agnostic. This locks that
|
||||
// the AWS 2026-04 additional algorithms (XXHash3/64/128, SHA-512, MD5) are forwarded
|
||||
|
||||
@@ -530,47 +530,6 @@ mod tests {
|
||||
assert!(redacted_json.contains(r#""session_token":null"#));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn historical_bucket_target_options_remain_readable() {
|
||||
let target: BucketTarget = serde_json::from_value(serde_json::json!({
|
||||
"endpoint": "legacy.example:9000",
|
||||
"credentials": {
|
||||
"accessKey": "legacy-access",
|
||||
"secretKey": "legacy-secret",
|
||||
"session_token": "legacy-session-token",
|
||||
"expiration": "2024-12-31T23:59:59Z"
|
||||
},
|
||||
"targetbucket": "legacy-bucket",
|
||||
"api": "s3v2",
|
||||
"healthCheckDuration": 30,
|
||||
"disableProxy": true,
|
||||
"edge": true,
|
||||
"edgeSyncBeforeExpiry": true,
|
||||
"type": "replication"
|
||||
}))
|
||||
.expect("historical remote target should remain readable");
|
||||
|
||||
assert_eq!(target.api, "s3v2");
|
||||
assert_eq!(target.health_check_duration, Duration::from_secs(30));
|
||||
assert!(target.disable_proxy);
|
||||
assert!(target.edge);
|
||||
assert!(target.edge_sync_before_expiry);
|
||||
assert_eq!(
|
||||
target
|
||||
.credentials
|
||||
.as_ref()
|
||||
.and_then(|credentials| credentials.session_token.as_deref()),
|
||||
Some("legacy-session-token")
|
||||
);
|
||||
assert!(
|
||||
target
|
||||
.credentials
|
||||
.as_ref()
|
||||
.and_then(|credentials| credentials.expiration)
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_target_type_json_deserialize() {
|
||||
// Test BucketTargetType JSON deserialization
|
||||
|
||||
@@ -21,15 +21,17 @@ const EVENT_LIFECYCLE_CLEANUP_SKIPPED: &str = "lifecycle_cleanup_skipped";
|
||||
const EVENT_LIFECYCLE_CLEANUP_FAILED: &str = "lifecycle_cleanup_failed";
|
||||
|
||||
use crate::bucket::lifecycle::lifecycle;
|
||||
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationObjectBridge};
|
||||
use crate::bucket::replication::{ReplicationLifecycleBridge, ReplicationState, replication_state_to_filemeta};
|
||||
use crate::bucket::versioning::VersioningApi;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::storage_api_contracts::object::{ObjectOperations as _, ObjectToDelete};
|
||||
use crate::store::ECStore;
|
||||
use rustfs_lock::MAX_DELETE_LIST;
|
||||
|
||||
pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
|
||||
let delete_config_snapshot = match ReplicationObjectBridge::delete_request_config(api, bucket).await {
|
||||
Ok(snapshot) => Arc::new(snapshot),
|
||||
let version_suspended = match BucketVersioningSys::get(bucket).await {
|
||||
Ok(vc) => vc.suspended(),
|
||||
Err(err) => {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_CLEANUP_SKIPPED,
|
||||
@@ -37,7 +39,7 @@ pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket,
|
||||
error = ?err,
|
||||
reason = "delete_config_snapshot_unavailable",
|
||||
reason = "versioning_config_unavailable",
|
||||
"Skipped lifecycle noncurrent version cleanup"
|
||||
);
|
||||
return;
|
||||
@@ -53,12 +55,45 @@ pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[
|
||||
remaining = &[];
|
||||
}
|
||||
|
||||
let mut replication_candidates: Vec<Option<ReplicationState>> = Vec::with_capacity(to_del.len());
|
||||
for object in to_del.iter() {
|
||||
let version_id = object.version_id.map(|vid| vid.to_string());
|
||||
let opts = ObjectOptions {
|
||||
version_id: version_id.clone(),
|
||||
versioned: true,
|
||||
version_suspended,
|
||||
..Default::default()
|
||||
};
|
||||
let candidate = match api.get_object_info(bucket, &object.object_name, &opts).await {
|
||||
Ok(info) => {
|
||||
let dsc = ReplicationLifecycleBridge::check_delete_replication(bucket, object, &info, &opts).await;
|
||||
dsc.replicate_any()
|
||||
.then(|| ReplicationLifecycleBridge::version_delete_replication_state(&dsc))
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_CLEANUP_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket,
|
||||
object = %object.object_name,
|
||||
version_id = ?version_id,
|
||||
error = ?err,
|
||||
reason = "object_info_unavailable",
|
||||
"Skipped lifecycle delete replication scheduling"
|
||||
);
|
||||
None
|
||||
}
|
||||
};
|
||||
replication_candidates.push(candidate);
|
||||
}
|
||||
|
||||
let (mut deleted_objs, errors) = api
|
||||
.delete_objects(
|
||||
bucket,
|
||||
to_del.to_vec(),
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(Arc::clone(&delete_config_snapshot)),
|
||||
version_suspended,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -73,9 +108,10 @@ pub async fn delete_object_versions(api: &Arc<ECStore>, bucket: &str, to_del: &[
|
||||
if let Some(target) = to_del.get(i) {
|
||||
crate::object_api::notify_object_mutation(bucket, &target.object_name).await;
|
||||
}
|
||||
if deleted_obj.replication_state.is_none() {
|
||||
let Some(replication_state) = replication_candidates.get(i).and_then(|c| c.clone()) else {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
deleted_obj.replication_state = Some(replication_state_to_filemeta(&replication_state));
|
||||
ReplicationLifecycleBridge::schedule_delete(bucket.to_string(), deleted_obj.clone()).await;
|
||||
}
|
||||
|
||||
|
||||
@@ -339,9 +339,7 @@ impl Sets {
|
||||
futures.push(set.delete_object(bucket, object, opt.clone()));
|
||||
}
|
||||
|
||||
if let Some(err) = join_all(futures).await.into_iter().find_map(Result::err) {
|
||||
return Err(err);
|
||||
}
|
||||
let _results = join_all(futures).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -142,20 +142,6 @@ impl Disk {
|
||||
Disk::Remote(remote_disk) => remote_disk.set_disk_id(id).await,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn local_health_tracker_epoch_for_reconnect(&self) -> Option<disk_store::ReconnectDiskHealthState> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => Some(local_disk.health_tracker_epoch_for_reconnect()),
|
||||
Disk::Remote(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cached_disk_id(&self) -> Option<Uuid> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.get_current_disk_id().await,
|
||||
Disk::Remote(remote_disk) => remote_disk.get_disk_id().await.ok().flatten(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -620,13 +606,6 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn metrics_snapshot(&self) -> Option<DiskMetrics> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => Some(local_disk.metrics_snapshot()),
|
||||
Disk::Remote(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn health_check_enabled_for_test(&self) -> bool {
|
||||
match self {
|
||||
@@ -685,21 +664,9 @@ impl Disk {
|
||||
}
|
||||
|
||||
pub async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> Result<DiskStore> {
|
||||
new_disk_with_health_tracker(ep, opt, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn new_disk_with_health_tracker(
|
||||
ep: &Endpoint,
|
||||
opt: &DiskOption,
|
||||
reconnect: Option<disk_store::ReconnectDiskHealthState>,
|
||||
) -> Result<DiskStore> {
|
||||
if ep.is_local {
|
||||
let s = LocalDisk::new(ep, opt.cleanup).await?;
|
||||
Ok(Arc::new(Disk::Local(Box::new(LocalDiskWrapper::new_with_reconnect_state(
|
||||
Arc::new(s),
|
||||
opt.health_check,
|
||||
reconnect,
|
||||
)))))
|
||||
Ok(Arc::new(Disk::Local(Box::new(LocalDiskWrapper::new(Arc::new(s), opt.health_check)))))
|
||||
} else {
|
||||
let data_transport = build_internode_data_transport_from_env();
|
||||
let remote_disk = RemoteDisk::new(ep, opt, data_transport?).await?;
|
||||
|
||||
@@ -677,22 +677,21 @@ pub(crate) type ExistingBaseDirectoryGuard = ();
|
||||
#[cfg(windows)]
|
||||
fn lock_windows_directory(path: &Path) -> io::Result<winapi_util::Handle> {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::Storage::FileSystem::{
|
||||
FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT,
|
||||
FILE_SHARE_READ, FILE_SHARE_WRITE,
|
||||
};
|
||||
|
||||
// Relative child publication requires write sharing on every guarded
|
||||
// ancestor. Omitting delete sharing still prevents any directory in the
|
||||
// resolved path from being renamed or removed before the commit finishes.
|
||||
const FILE_ATTRIBUTE_DIRECTORY: u64 = 0x10;
|
||||
const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400;
|
||||
const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
|
||||
const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
|
||||
const FILE_SHARE_READ: u32 = 0x1;
|
||||
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE)
|
||||
.share_mode(FILE_SHARE_READ)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(path)?;
|
||||
let handle = winapi_util::Handle::from_file(file);
|
||||
let info = winapi_util::file::information(&handle)?;
|
||||
if info.file_attributes() & u64::from(FILE_ATTRIBUTE_DIRECTORY) == 0
|
||||
if info.file_attributes() & FILE_ATTRIBUTE_DIRECTORY == 0
|
||||
|| info.file_attributes() & u64::from(FILE_ATTRIBUTE_REPARSE_POINT) != 0
|
||||
{
|
||||
return Err(io::Error::from(io::ErrorKind::NotADirectory));
|
||||
@@ -1224,53 +1223,23 @@ mod tests {
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_parent_guard_blocks_parent_replacement() {
|
||||
fn windows_parent_guard_blocks_base_and_intermediate_replacement() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
std::fs::create_dir(&base).expect("create destination base");
|
||||
let parent = base.join("object").join("nested");
|
||||
let guard = mkdir_all_below_existing_base_std(&parent, &base).expect("create and lock destination parents");
|
||||
|
||||
std::fs::read_dir(&parent).expect("the locked parent must remain readable");
|
||||
std::fs::rename(&base, temp_dir.path().join("replacement-base"))
|
||||
.expect_err("the locked base must not be replaceable before commit");
|
||||
std::fs::rename(base.join("object"), base.join("replacement-object"))
|
||||
.expect_err("a locked intermediate directory must not be replaceable before commit");
|
||||
std::fs::rename(&parent, base.join("replacement-parent"))
|
||||
.expect_err("the locked destination parent must not be replaceable before commit");
|
||||
assert!(parent.is_dir(), "failed replacement must leave the guarded parent in place");
|
||||
|
||||
drop(guard);
|
||||
std::fs::rename(base.join("object"), base.join("replacement-object"))
|
||||
.expect("replacement should succeed after the commit guard is released");
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn windows_guarded_parent_allows_same_and_descendant_publication() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let base = temp_dir.path().join("bucket");
|
||||
let parent = base.join("object");
|
||||
std::fs::create_dir_all(&parent).expect("create destination parent");
|
||||
let _guard = mkdir_all_below_existing_base_std(&parent, &base).expect("guard destination parent");
|
||||
let first_src = temp_dir.path().join("first-stage");
|
||||
let second_src = temp_dir.path().join("second-stage");
|
||||
std::fs::write(&first_src, b"first").expect("write first source");
|
||||
std::fs::write(&second_src, b"second").expect("write second source");
|
||||
|
||||
rename_all(&first_src, parent.join("first"), &base)
|
||||
.await
|
||||
.expect("same-parent rename must succeed while a guard is held");
|
||||
rename_all(&second_src, parent.join("nested").join("second"), &base)
|
||||
.await
|
||||
.expect("descendant-parent rename must succeed while an ancestor guard is held");
|
||||
assert_eq!(std::fs::read(parent.join("first")).expect("read first destination"), b"first");
|
||||
assert_eq!(
|
||||
std::fs::read(parent.join("nested").join("second")).expect("read second destination"),
|
||||
b"second"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn rename_parent_creation_rejects_symlinked_base() {
|
||||
|
||||
@@ -112,9 +112,6 @@ pub struct ObjectOptions {
|
||||
pub want_checksum: Option<Checksum>,
|
||||
pub skip_verify_bitrot: bool,
|
||||
pub capacity_scope_token: Option<Uuid>,
|
||||
/// Storage-owned journal writer used by the atomic delete path. This is
|
||||
/// populated only by the `ECStore` wrapper that holds the namespace locks.
|
||||
pub tier_delete_journal_api: Option<Arc<crate::store::ECStore>>,
|
||||
}
|
||||
|
||||
impl ObjectOptions {
|
||||
|
||||
@@ -9383,8 +9383,6 @@ mod tests {
|
||||
backend_identity: Some(current_identity),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
};
|
||||
journal_store
|
||||
.insert_config_object(
|
||||
|
||||
@@ -4358,15 +4358,8 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
|
||||
let runtime_state = disk.runtime_state();
|
||||
let offline_duration_seconds = disk.offline_duration_secs();
|
||||
let capacity_snapshot = disk.last_capacity_snapshot();
|
||||
let cached_disk_id = disk.cached_disk_id().await;
|
||||
if runtime_state.should_probe_for_admin() || runtime_state == disk::health_state::RuntimeDriveHealthState::Suspect {
|
||||
match disk
|
||||
.disk_info(&DiskInfoOptions {
|
||||
metrics: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
match disk.disk_info(&DiskInfoOptions::default()).await {
|
||||
Ok(res) => {
|
||||
disk.record_capacity_probe(res.total, res.used, res.free);
|
||||
ret.push(rustfs_madmin::Disk {
|
||||
@@ -4397,7 +4390,6 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
|
||||
utilization: utilization_percent(res.total, res.used),
|
||||
used_inodes: res.used_inodes,
|
||||
free_inodes: res.free_inodes,
|
||||
metrics: Some(res.metrics),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
@@ -4405,15 +4397,12 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
|
||||
let mut disk_info = rustfs_madmin::Disk {
|
||||
state: err.to_string(),
|
||||
endpoint: eps[i].to_string(),
|
||||
drive_path: eps[i].get_file_path(),
|
||||
local: eps[i].is_local,
|
||||
pool_index: eps[i].pool_idx,
|
||||
set_index: eps[i].set_idx,
|
||||
disk_index: eps[i].disk_idx,
|
||||
runtime_state: Some(runtime_state.as_str().to_string()),
|
||||
offline_duration_seconds,
|
||||
metrics: disk.metrics_snapshot(),
|
||||
uuid: cached_disk_id.map_or_else(String::new, |id| id.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
if let Some((total, used, free, _)) = capacity_snapshot {
|
||||
@@ -4432,16 +4421,16 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let mut disk_info =
|
||||
build_runtime_snapshot_disk(&eps[i], runtime_state, offline_duration_seconds, capacity_snapshot);
|
||||
disk_info.metrics = disk.metrics_snapshot();
|
||||
disk_info.uuid = cached_disk_id.map_or_else(String::new, |id| id.to_string());
|
||||
ret.push(disk_info);
|
||||
ret.push(build_runtime_snapshot_disk(
|
||||
&eps[i],
|
||||
runtime_state,
|
||||
offline_duration_seconds,
|
||||
capacity_snapshot,
|
||||
));
|
||||
}
|
||||
} else {
|
||||
ret.push(rustfs_madmin::Disk {
|
||||
endpoint: eps[i].to_string(),
|
||||
drive_path: eps[i].get_file_path(),
|
||||
local: eps[i].is_local,
|
||||
pool_index: eps[i].pool_idx,
|
||||
set_index: eps[i].set_idx,
|
||||
@@ -4467,7 +4456,6 @@ fn build_runtime_snapshot_disk(
|
||||
) -> rustfs_madmin::Disk {
|
||||
let mut disk = rustfs_madmin::Disk {
|
||||
endpoint: endpoint.to_string(),
|
||||
drive_path: endpoint.get_file_path(),
|
||||
local: endpoint.is_local,
|
||||
pool_index: endpoint.pool_idx,
|
||||
set_index: endpoint.set_idx,
|
||||
@@ -4710,7 +4698,6 @@ pub fn is_infrequent_access_class(storage_class: &str) -> bool {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::replication::{replication_statuses_map, version_purge_statuses_map};
|
||||
use crate::cluster::rpc::{RemoteDisk, TcpHttpInternodeDataTransport};
|
||||
use crate::disk::CHECK_PART_UNKNOWN;
|
||||
use crate::disk::CHECK_PART_VOLUME_NOT_FOUND;
|
||||
use crate::disk::DataDirDeleteStatus;
|
||||
@@ -5002,26 +4989,6 @@ mod tests {
|
||||
(dir, endpoint, disk)
|
||||
}
|
||||
|
||||
async fn make_remote_disk_for_info_test(disk_idx: usize) -> (Endpoint, DiskStore) {
|
||||
let endpoint_url = format!("http://remote-server:9000/data{disk_idx}");
|
||||
let mut endpoint = Endpoint::try_from(endpoint_url.as_str()).expect("remote endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_idx);
|
||||
let remote_disk = RemoteDisk::new(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
Arc::new(TcpHttpInternodeDataTransport),
|
||||
)
|
||||
.await
|
||||
.expect("remote disk should be created");
|
||||
|
||||
(endpoint, Arc::new(disk::Disk::Remote(Box::new(remote_disk))))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_rename_data_quorum_failure_rolls_back_destination_object() {
|
||||
let dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
@@ -7731,13 +7698,6 @@ mod tests {
|
||||
.as_ref()
|
||||
.expect("disk 1 should exist")
|
||||
.force_runtime_state_for_test(RuntimeDriveHealthState::Suspect);
|
||||
let offline_disk_id = Uuid::new_v4();
|
||||
disks[2]
|
||||
.as_ref()
|
||||
.expect("disk 2 should exist")
|
||||
.set_disk_id_state(Some(offline_disk_id))
|
||||
.await
|
||||
.expect("offline disk id should be cached");
|
||||
disks[2]
|
||||
.as_ref()
|
||||
.expect("disk 2 should exist")
|
||||
@@ -7749,82 +7709,14 @@ mod tests {
|
||||
assert_eq!(info[0].state, "ok");
|
||||
assert_eq!(info[0].runtime_state.as_deref(), Some("online"));
|
||||
assert!(!info[0].drive_path.is_empty(), "online disk should keep immediate disk_info probe");
|
||||
assert!(
|
||||
info[0]
|
||||
.metrics
|
||||
.as_ref()
|
||||
.and_then(|metrics| metrics.api_calls.get("disk_info"))
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
> 0,
|
||||
"online disk should expose disk_info operation metrics"
|
||||
);
|
||||
|
||||
assert_eq!(info[1].state, "ok");
|
||||
assert_eq!(info[1].runtime_state.as_deref(), Some("suspect"));
|
||||
assert!(!info[1].drive_path.is_empty(), "suspect disk should still probe for fresher disk info");
|
||||
assert!(
|
||||
info[1]
|
||||
.metrics
|
||||
.as_ref()
|
||||
.and_then(|metrics| metrics.last_minute.get("disk_info"))
|
||||
.map(|action| action.count)
|
||||
.unwrap_or_default()
|
||||
> 0,
|
||||
"suspect disk should expose last-minute disk_info latency"
|
||||
);
|
||||
|
||||
assert_eq!(info[2].state, "offline");
|
||||
assert_eq!(info[2].runtime_state.as_deref(), Some("offline"));
|
||||
assert_eq!(
|
||||
info[2].drive_path,
|
||||
endpoints[2].get_file_path(),
|
||||
"offline disk should keep stable endpoint path"
|
||||
);
|
||||
assert_eq!(info[2].uuid, offline_disk_id.to_string());
|
||||
assert!(
|
||||
info[2].metrics.is_some(),
|
||||
"offline runtime fallback should preserve disk metrics snapshot"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_disks_info_preserves_remote_cached_disk_id_when_offline() {
|
||||
let (endpoint, disk) = make_remote_disk_for_info_test(0).await;
|
||||
let remote_disk_id = Uuid::new_v4();
|
||||
disk.set_disk_id_state(Some(remote_disk_id))
|
||||
.await
|
||||
.expect("remote disk id should be cached");
|
||||
disk.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
|
||||
|
||||
let info = get_disks_info(&[Some(disk)], &[endpoint]).await;
|
||||
|
||||
assert_eq!(info.len(), 1);
|
||||
assert_eq!(info[0].state, "offline");
|
||||
assert_eq!(info[0].runtime_state.as_deref(), Some("offline"));
|
||||
assert_eq!(info[0].uuid, remote_disk_id.to_string());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_disks_info_preserves_cached_disk_id_after_failed_live_probe() {
|
||||
let format = FormatV3::new(1, 1);
|
||||
let (temp_dir, endpoint, disk) = make_formatted_local_disk_for_info_test(0, &format).await;
|
||||
let cached_disk_id = Uuid::new_v4();
|
||||
disk.set_disk_id_state(Some(cached_disk_id))
|
||||
.await
|
||||
.expect("disk id should be cached before the failed probe");
|
||||
disk.force_runtime_state_for_test(RuntimeDriveHealthState::Suspect);
|
||||
|
||||
let info = get_disks_info(&[Some(disk)], &[endpoint]).await;
|
||||
|
||||
assert_eq!(info.len(), 1);
|
||||
assert_eq!(info[0].runtime_state.as_deref(), Some("suspect"));
|
||||
assert_eq!(info[0].uuid, cached_disk_id.to_string());
|
||||
assert_eq!(
|
||||
info[0].drive_path,
|
||||
temp_dir.path().to_string_lossy(),
|
||||
"failed live probe should still keep the endpoint path"
|
||||
);
|
||||
assert!(info[2].drive_path.is_empty(), "offline disk should use runtime snapshot fallback");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -322,16 +322,7 @@ impl SetDisks {
|
||||
pub async fn renew_disk(&self, ep: &Endpoint) {
|
||||
debug!("renew_disk: start {:?}", ep);
|
||||
|
||||
let previous_health = {
|
||||
let disks = self.disks.read().await;
|
||||
disks
|
||||
.iter()
|
||||
.filter_map(|disk| disk.as_ref())
|
||||
.find(|disk| disk.endpoint() == *ep)
|
||||
.and_then(|disk| disk.local_health_tracker_epoch_for_reconnect())
|
||||
};
|
||||
|
||||
let (new_disk, fm) = match Self::connect_endpoint(ep, previous_health).await {
|
||||
let (new_disk, fm) = match Self::connect_endpoint(ep).await {
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
warn!("renew_disk: connect_endpoint err {:?}", &e);
|
||||
@@ -409,17 +400,13 @@ impl SetDisks {
|
||||
Err(Error::other("DriveID: not found"))
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) async fn connect_endpoint(
|
||||
ep: &Endpoint,
|
||||
reconnect: Option<disk::disk_store::ReconnectDiskHealthState>,
|
||||
) -> disk::error::Result<(DiskStore, FormatV3)> {
|
||||
let disk = crate::disk::new_disk_with_health_tracker(
|
||||
pub(in crate::set_disk) async fn connect_endpoint(ep: &Endpoint) -> disk::error::Result<(DiskStore, FormatV3)> {
|
||||
let disk = new_disk(
|
||||
ep,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: true,
|
||||
},
|
||||
reconnect,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -727,25 +714,6 @@ mod tests {
|
||||
renewed_disk.health_check_enabled_for_test(),
|
||||
"renewed disks must keep health monitoring enabled so later faulty marks can recover"
|
||||
);
|
||||
renewed_disk
|
||||
.disk_info(&DiskInfoOptions::default())
|
||||
.await
|
||||
.expect("renewed disk_info should record a drive API metric");
|
||||
renewed_disk.force_runtime_state_for_test(disk::health_state::RuntimeDriveHealthState::Offline);
|
||||
|
||||
set_disks.renew_disk(&endpoints[0]).await;
|
||||
|
||||
let disks = set_disks.get_disks_internal().await;
|
||||
let renewed_again = disks[0]
|
||||
.as_ref()
|
||||
.expect("second renew_disk should keep the recovered disk attached");
|
||||
assert_eq!(
|
||||
renewed_again
|
||||
.metrics_snapshot()
|
||||
.and_then(|metrics| metrics.api_calls.get("disk_info").copied()),
|
||||
Some(1),
|
||||
"disk reconnect must preserve the local drive metrics tracker epoch"
|
||||
);
|
||||
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
@@ -24,14 +24,10 @@ use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks
|
||||
use crate::set_disk::read::GetObjectDownstreamWriter;
|
||||
|
||||
use crate::bucket::lifecycle::{
|
||||
tier_delete_journal::{
|
||||
enqueue_committed_tier_delete_journal_entry, persist_tier_delete_journal_entry,
|
||||
record_tier_delete_journal_backend_identity, remove_tier_delete_journal_entry,
|
||||
},
|
||||
tier_delete_journal::{persist_tier_delete_journal_entry, remove_tier_delete_journal_entry},
|
||||
tier_sweeper::{
|
||||
Jentry, RemoteTierDeleteOutcome, TierDeleteJournalState,
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent, delete_object_from_remote_tier_with_lease_idempotent,
|
||||
transitioned_delete_journal_entry_for_source,
|
||||
Jentry, RemoteTierDeleteOutcome, delete_confirmed_transition_candidate_exact_with_lease_idempotent,
|
||||
delete_object_from_remote_tier_with_lease_idempotent,
|
||||
},
|
||||
transition_transaction::{
|
||||
TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
|
||||
@@ -1855,8 +1851,6 @@ pub(crate) async fn cleanup_rejected_transition_upload_durably(
|
||||
} else {
|
||||
rustfs_filemeta::TransitionVersionState::Exact
|
||||
},
|
||||
state: TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
};
|
||||
|
||||
let journal_error = if let Some(api) = api.as_ref() {
|
||||
@@ -2774,11 +2768,6 @@ impl SetDisks {
|
||||
let (mut fi, _, disks) = self.get_object_fileinfo_gated(bucket, object, opts, false, false).await?;
|
||||
|
||||
fi.metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_owned());
|
||||
if let Some(eval_metadata) = &opts.eval_metadata {
|
||||
for (key, value) in eval_metadata {
|
||||
fi.metadata.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pause_object_tagging_commit(bucket, object).await;
|
||||
@@ -3214,7 +3203,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
object_lock_delete_check_required(metadata_sys::get_in(&self.ctx, bucket).await.ok().as_deref());
|
||||
|
||||
let mut vers_map: HashMap<&String, FileInfoVersions> = HashMap::new();
|
||||
let mut journal_entries: Vec<(usize, Jentry)> = Vec::new();
|
||||
|
||||
for (i, dobj) in objects.iter().enumerate() {
|
||||
if del_errs[i].is_some() {
|
||||
@@ -3239,8 +3227,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let marker_delete = dobj.version_id.is_none() || dobj.synthetic_version_id;
|
||||
let replication_needs_source = replicate_delete
|
||||
&& (!marker_delete || delete_config_snapshot.active_delete_marker_rules_require_tags(&replication_object_name));
|
||||
let (goi, gerr) = if object_lock_checks_required || replication_needs_source || opts.tier_delete_journal_api.is_some()
|
||||
{
|
||||
let (goi, gerr) = if object_lock_checks_required || replication_needs_source {
|
||||
let (goi, _write_quorum, gerr) = self.get_object_info_and_quorum(bucket, &dobj.object_name, &check_opts).await;
|
||||
(goi, gerr)
|
||||
} else {
|
||||
@@ -3249,14 +3236,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let source_missing = gerr
|
||||
.as_ref()
|
||||
.is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err));
|
||||
let explicit_delete_marker = opts.tier_delete_journal_api.is_some()
|
||||
&& dobj.version_id.is_some()
|
||||
&& goi.delete_marker
|
||||
&& goi.version_id == version_id
|
||||
&& matches!(gerr.as_ref(), Some(StorageError::MethodNotAllowed));
|
||||
if let Some(err) = gerr.as_ref()
|
||||
&& !source_missing
|
||||
&& !explicit_delete_marker
|
||||
{
|
||||
del_errs[i] = Some(err.clone());
|
||||
continue;
|
||||
@@ -3269,23 +3250,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
continue;
|
||||
}
|
||||
|
||||
if opts.tier_delete_journal_api.is_some()
|
||||
&& let Some(mut je) = transitioned_delete_journal_entry_for_source(
|
||||
version_id,
|
||||
versioned,
|
||||
version_suspended,
|
||||
bucket,
|
||||
&replication_object_name,
|
||||
&goi,
|
||||
)
|
||||
{
|
||||
if let Err(err) = record_tier_delete_journal_backend_identity(&mut je, &goi.user_defined) {
|
||||
del_errs[i] = Some(Error::other(err));
|
||||
continue;
|
||||
}
|
||||
journal_entries.push((i, je));
|
||||
}
|
||||
|
||||
let mut admitted = dobj.clone();
|
||||
admitted.object_name = replication_object_name;
|
||||
if admitted.synthetic_version_id {
|
||||
@@ -3331,11 +3295,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if goi.delete_marker && dobj.version_id.is_some() && goi.version_id == version_id {
|
||||
vr.deleted = true;
|
||||
vr.mod_time = goi.mod_time;
|
||||
}
|
||||
|
||||
let v = {
|
||||
if vers_map.contains_key(&dobj.object_name) {
|
||||
let val = vers_map.get_mut(&dobj.object_name).unwrap();
|
||||
@@ -3408,23 +3367,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
return (del_objects, del_errs);
|
||||
}
|
||||
|
||||
let mut persisted_journal_entries = Vec::with_capacity(journal_entries.len());
|
||||
if let Some(api) = opts.tier_delete_journal_api.as_ref() {
|
||||
for (idx, mut je) in journal_entries {
|
||||
if let Err(err) = persist_tier_delete_journal_entry(Arc::clone(api), &je).await {
|
||||
del_errs[idx] = Some(Error::other(err));
|
||||
continue;
|
||||
}
|
||||
je.state = TierDeleteJournalState::Prepared;
|
||||
persisted_journal_entries.push((idx, je));
|
||||
}
|
||||
}
|
||||
|
||||
for fi_vers in &mut vers {
|
||||
fi_vers.versions.retain(|fi| del_errs[fi.idx].is_none());
|
||||
}
|
||||
vers.retain(|fi_vers| !fi_vers.versions.is_empty());
|
||||
|
||||
let rollback_dir = Uuid::new_v4();
|
||||
|
||||
let disks = self.disks.read().await;
|
||||
@@ -3588,37 +3530,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
// TODO: add_partial
|
||||
|
||||
if let Some(api) = opts.tier_delete_journal_api.as_ref() {
|
||||
for (idx, je) in persisted_journal_entries {
|
||||
if del_errs[idx].is_none() {
|
||||
let mut committed = je;
|
||||
committed.state = TierDeleteJournalState::Committed;
|
||||
if let Err(err) = persist_tier_delete_journal_entry(Arc::clone(api), &committed).await {
|
||||
warn!(
|
||||
object = %committed.obj_name,
|
||||
tier = %committed.tier_name,
|
||||
error = ?err,
|
||||
"batch tier delete committed locally but journal commit failed; recovery will retry"
|
||||
);
|
||||
} else if let Err(err) = enqueue_committed_tier_delete_journal_entry(&committed).await {
|
||||
warn!(
|
||||
object = %committed.obj_name,
|
||||
tier = %committed.tier_name,
|
||||
error = ?err,
|
||||
"batch tier delete journal committed but could not be queued; recovery will retry"
|
||||
);
|
||||
}
|
||||
} else if let Err(err) = remove_tier_delete_journal_entry(Arc::clone(api), &je).await {
|
||||
warn!(
|
||||
object = %je.obj_name,
|
||||
tier = %je.tier_name,
|
||||
error = ?err,
|
||||
"failed to remove aborted batch tier delete journal"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if dist_erasure {
|
||||
self.release_dist_delete_object_locks_batch(dist_batch_lock_ids).await;
|
||||
}
|
||||
|
||||
@@ -1379,8 +1379,6 @@ mod tests {
|
||||
backend_identity: Some(identity_a),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
};
|
||||
let entry_b = Jentry {
|
||||
obj_name: "remote-b".to_string(),
|
||||
@@ -1389,8 +1387,6 @@ mod tests {
|
||||
backend_identity: Some(identity_b),
|
||||
version_id_exact: true,
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
state: crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed,
|
||||
source: None,
|
||||
};
|
||||
let remove_a = backend_a.arm_failing_remove_barrier().await;
|
||||
persist_tier_delete_journal_entry(store_a.clone(), &entry_a)
|
||||
|
||||
@@ -1279,22 +1279,6 @@ mod tests {
|
||||
assert!(disks[2].is_none(), "the malformed outlier must be isolated");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fresh_format_load_initializes_all_disks() {
|
||||
let (_temp_dir, mut disks) = local_disks(3).await;
|
||||
|
||||
let format = connect_load_init_formats(true, &mut disks, 1, 3, None)
|
||||
.await
|
||||
.expect("fresh disks should receive a storage format");
|
||||
|
||||
let (formats, errors) = load_format_erasure_all(&disks, false).await;
|
||||
assert!(errors.iter().all(Option::is_none), "every disk should load its fresh format: {errors:?}");
|
||||
assert!(
|
||||
formats_match_reference_slots(&formats, &format, 0),
|
||||
"fresh format publication must preserve every disk slot"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fresh_format_load_does_not_initialize_with_a_missing_disk() {
|
||||
let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await;
|
||||
|
||||
@@ -13,16 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::{
|
||||
tier_delete_journal::{
|
||||
abort_prepared_tier_delete_journal_entry as abort_prepared_journal_entry_if_current, commit_tier_delete_journal_entry,
|
||||
enqueue_committed_tier_delete_journal_entry, persist_tier_delete_journal_entry,
|
||||
record_tier_delete_journal_backend_identity,
|
||||
},
|
||||
tier_sweeper::{
|
||||
Jentry, attach_tier_delete_source, transitioned_delete_journal_entry_for_source, transitioned_force_delete_journal_entry,
|
||||
},
|
||||
};
|
||||
use crate::bucket::replication::ReplicationObjectBridge;
|
||||
use crate::disk::OldCurrentSize;
|
||||
use crate::object_api::DeleteLockFence;
|
||||
@@ -43,181 +33,6 @@ use std::{
|
||||
};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
|
||||
const FORCE_DELETE_LIST_PAGE_SIZE: i32 = 1_000;
|
||||
|
||||
fn build_tier_delete_journal_entry(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
source: &ObjectInfo,
|
||||
) -> Result<Option<Jentry>> {
|
||||
let version_id = opts.version_id.as_deref().map(Uuid::parse_str).transpose()?;
|
||||
let source_object = decode_dir_object(object);
|
||||
let Some(mut je) = (if opts.delete_prefix {
|
||||
transitioned_force_delete_journal_entry(&source.transitioned_object, source.transition_version_state).map(|mut je| {
|
||||
attach_tier_delete_source(&mut je, bucket, source_object.as_str(), source, opts.versioned, opts.version_suspended);
|
||||
je
|
||||
})
|
||||
} else {
|
||||
transitioned_delete_journal_entry_for_source(
|
||||
version_id,
|
||||
opts.versioned,
|
||||
opts.version_suspended,
|
||||
bucket,
|
||||
source_object.as_str(),
|
||||
source,
|
||||
)
|
||||
}) else {
|
||||
return Ok(None);
|
||||
};
|
||||
record_tier_delete_journal_backend_identity(&mut je, &source.user_defined).map_err(Error::other)?;
|
||||
Ok(Some(je))
|
||||
}
|
||||
|
||||
async fn prepare_tier_delete_journal_entry(
|
||||
api: &Arc<ECStore>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
source: &ObjectInfo,
|
||||
) -> Result<Option<Jentry>> {
|
||||
let Some(je) = build_tier_delete_journal_entry(bucket, object, opts, source)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
persist_tier_delete_journal_entry(Arc::clone(api), &je)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
Ok(Some(je))
|
||||
}
|
||||
|
||||
async fn abort_prepared_tier_delete_journal_entry(api: &Arc<ECStore>, je: &Jentry) {
|
||||
if let Err(err) = abort_prepared_journal_entry_if_current(Arc::clone(api), je).await {
|
||||
warn!(
|
||||
object = %je.obj_name,
|
||||
tier = %je.tier_name,
|
||||
error = ?err,
|
||||
"failed to remove aborted tier delete journal"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn abort_prepared_tier_delete_journal_entries(api: &Arc<ECStore>, entries: &[Jentry]) {
|
||||
for entry in entries {
|
||||
abort_prepared_tier_delete_journal_entry(api, entry).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn commit_prepared_tier_delete_journal_entry(api: &Arc<ECStore>, je: &Jentry) {
|
||||
if let Err(err) = commit_tier_delete_journal_entry(Arc::clone(api), je).await {
|
||||
warn!(
|
||||
object = %je.obj_name,
|
||||
tier = %je.tier_name,
|
||||
error = ?err,
|
||||
"tier delete committed locally but journal commit failed; recovery will retry"
|
||||
);
|
||||
return;
|
||||
}
|
||||
let mut committed = je.clone();
|
||||
committed.state = crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState::Committed;
|
||||
if let Err(err) = enqueue_committed_tier_delete_journal_entry(&committed).await {
|
||||
warn!(
|
||||
object = %je.obj_name,
|
||||
tier = %je.tier_name,
|
||||
error = ?err,
|
||||
"tier delete journal committed but could not be queued; recovery will retry"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn commit_prepared_tier_delete_journal_entries(api: &Arc<ECStore>, entries: &[Jentry]) {
|
||||
for entry in entries {
|
||||
commit_prepared_tier_delete_journal_entry(api, entry).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn prepare_prefix_tier_delete_journal_entries(
|
||||
api: &Arc<ECStore>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<Vec<Jentry>> {
|
||||
let mut marker = None;
|
||||
let mut version_marker = None;
|
||||
let mut entries = Vec::new();
|
||||
|
||||
loop {
|
||||
let page = Arc::clone(api)
|
||||
.list_object_versions_for_lifecycle(
|
||||
bucket,
|
||||
prefix,
|
||||
marker.clone(),
|
||||
version_marker.clone(),
|
||||
None,
|
||||
FORCE_DELETE_LIST_PAGE_SIZE,
|
||||
)
|
||||
.await?;
|
||||
|
||||
for source in page.objects {
|
||||
if let Some(entry) = build_tier_delete_journal_entry(bucket, &source.name, opts, &source)? {
|
||||
entries.push(entry);
|
||||
}
|
||||
}
|
||||
|
||||
if !page.is_truncated {
|
||||
break;
|
||||
}
|
||||
|
||||
let next_marker = page
|
||||
.next_marker
|
||||
.ok_or_else(|| Error::other("truncated force delete listing has no next marker"))?;
|
||||
let next_version_marker = page.next_version_idmarker;
|
||||
if marker.as_deref() == Some(next_marker.as_str()) && version_marker == next_version_marker {
|
||||
return Err(Error::other("force delete listing marker did not advance"));
|
||||
}
|
||||
marker = Some(next_marker);
|
||||
version_marker = next_version_marker;
|
||||
}
|
||||
|
||||
let mut persisted = Vec::with_capacity(entries.len());
|
||||
for entry in entries {
|
||||
if let Err(err) = persist_tier_delete_journal_entry(Arc::clone(api), &entry).await {
|
||||
abort_prepared_tier_delete_journal_entries(api, &persisted).await;
|
||||
return Err(Error::other(err));
|
||||
}
|
||||
persisted.push(entry);
|
||||
}
|
||||
Ok(persisted)
|
||||
}
|
||||
async fn delete_prefix_with_tier_delete_journal(
|
||||
store: &ECStore,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
tier_journal_api: Option<&Arc<ECStore>>,
|
||||
) -> Result<()> {
|
||||
let journal_entry = if let Some(api) = tier_journal_api {
|
||||
Some(prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let result = store.delete_prefix(bucket, object, opts).await;
|
||||
match result {
|
||||
Ok(()) => {
|
||||
if let (Some(api), Some(entries)) = (tier_journal_api, journal_entry.as_ref()) {
|
||||
commit_prepared_tier_delete_journal_entries(api, entries).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(err) => {
|
||||
if let (Some(api), Some(entries)) = (tier_journal_api, journal_entry.as_ref()) {
|
||||
abort_prepared_tier_delete_journal_entries(api, entries).await;
|
||||
}
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A GET whose object identity has been resolved while its namespace read lock
|
||||
/// remains held, but whose body reader has not been constructed yet.
|
||||
///
|
||||
@@ -1154,15 +969,6 @@ impl ECStore {
|
||||
|
||||
if !dst_opts.versioned && src_opts.version_id.is_none() {
|
||||
if src_info.metadata_only {
|
||||
// Zero-copy update: only xl.meta is rewritten, the data blocks stay as they
|
||||
// are. The caller must therefore guarantee that the destination metadata
|
||||
// still describes the stored bytes. In particular a copy that re-derives
|
||||
// encryption material may NOT set metadata_only — that would leave a fresh
|
||||
// DEK beside ciphertext sealed under the old one, permanently destroying the
|
||||
// object. The S3 handler enforces this before calling in (see the
|
||||
// metadata_only decision in rustfs/src/app/object_usecase.rs); the sibling
|
||||
// versioned branch below resolves the same risk by rewriting through
|
||||
// put_object (issue #4238).
|
||||
return self.pools[pool_idx]
|
||||
.copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &dst_opts)
|
||||
.await;
|
||||
@@ -1277,49 +1083,8 @@ impl ECStore {
|
||||
purged
|
||||
}
|
||||
|
||||
pub async fn delete_object_with_tier_delete_journal(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
let result = self
|
||||
.handle_delete_object_with_journal(bucket, object, opts, Some(Arc::clone(self)))
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_list_objects_mutation(self, bucket).await;
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub async fn delete_objects_with_tier_delete_journal(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
objects: Vec<ObjectToDelete>,
|
||||
opts: ObjectOptions,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
|
||||
let result = self
|
||||
.handle_delete_objects_with_journal(bucket, objects, opts, Some(Arc::clone(self)))
|
||||
.await;
|
||||
let success_count = result.1.iter().filter(|err| err.is_none()).count();
|
||||
if success_count > 0 {
|
||||
list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
|
||||
self.handle_delete_object_with_journal(bucket, object, opts, None).await
|
||||
}
|
||||
|
||||
pub(super) async fn handle_delete_object_with_journal(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: ObjectOptions,
|
||||
tier_journal_api: Option<Arc<ECStore>>,
|
||||
) -> Result<ObjectInfo> {
|
||||
check_del_obj_args(bucket, object)?;
|
||||
|
||||
let object = if opts.delete_prefix && !opts.delete_prefix_object {
|
||||
@@ -1329,12 +1094,11 @@ impl ECStore {
|
||||
};
|
||||
let object = object.as_str();
|
||||
let mut opts = opts;
|
||||
opts.tier_delete_journal_api = tier_journal_api.clone();
|
||||
|
||||
if opts.delete_prefix && !opts.delete_prefix_object {
|
||||
// Prefix deletes cover multiple object keys; an exact lock on the prefix string
|
||||
// would not protect child objects.
|
||||
delete_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
|
||||
self.delete_prefix(bucket, object, &opts).await?;
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
|
||||
@@ -1346,7 +1110,7 @@ impl ECStore {
|
||||
};
|
||||
|
||||
if opts.delete_prefix {
|
||||
delete_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
|
||||
self.delete_prefix(bucket, object, &opts).await?;
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
|
||||
@@ -1442,25 +1206,8 @@ impl ECStore {
|
||||
));
|
||||
}
|
||||
|
||||
let journal_entry = if let Some(api) = tier_journal_api.as_ref() {
|
||||
prepare_tier_delete_journal_entry(api, bucket, object, &opts, &pinfo.object_info).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if !errs.is_empty() && !opts.versioned && !opts.version_suspended {
|
||||
let mut obj = match self.delete_object_from_all_pools(bucket, object, &opts, errs).await {
|
||||
Ok(obj) => obj,
|
||||
Err(err) => {
|
||||
if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) {
|
||||
abort_prepared_tier_delete_journal_entry(api, je).await;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) {
|
||||
commit_prepared_tier_delete_journal_entry(api, je).await;
|
||||
}
|
||||
let mut obj = self.delete_object_from_all_pools(bucket, object, &opts, errs).await?;
|
||||
obj.name = decode_dir_object(object);
|
||||
return Ok(obj);
|
||||
}
|
||||
@@ -1468,9 +1215,6 @@ impl ECStore {
|
||||
for pool in self.pools.iter() {
|
||||
match pool.delete_object(bucket, object, opts.clone()).await {
|
||||
Ok(res) => {
|
||||
if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) {
|
||||
commit_prepared_tier_delete_journal_entry(api, je).await;
|
||||
}
|
||||
let mut obj = res;
|
||||
obj.name = decode_dir_object(object);
|
||||
return Ok(obj);
|
||||
@@ -1483,10 +1227,6 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
|
||||
if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) {
|
||||
abort_prepared_tier_delete_journal_entry(api, je).await;
|
||||
}
|
||||
|
||||
if let Some(ver) = opts.version_id {
|
||||
return Err(StorageError::VersionNotFound(bucket.to_owned(), object.to_owned(), ver));
|
||||
}
|
||||
@@ -1500,16 +1240,6 @@ impl ECStore {
|
||||
bucket: &str,
|
||||
objects: Vec<ObjectToDelete>,
|
||||
opts: ObjectOptions,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
|
||||
self.handle_delete_objects_with_journal(bucket, objects, opts, None).await
|
||||
}
|
||||
|
||||
pub(super) async fn handle_delete_objects_with_journal(
|
||||
&self,
|
||||
bucket: &str,
|
||||
objects: Vec<ObjectToDelete>,
|
||||
opts: ObjectOptions,
|
||||
tier_journal_api: Option<Arc<ECStore>>,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
|
||||
// encode object name
|
||||
let objects: Vec<ObjectToDelete> = objects
|
||||
@@ -1530,7 +1260,6 @@ impl ECStore {
|
||||
}
|
||||
|
||||
let mut opts = opts;
|
||||
opts.tier_delete_journal_api = tier_journal_api;
|
||||
if opts.delete_replication_config_snapshot.is_none() {
|
||||
match ReplicationObjectBridge::delete_request_config_in(&self.ctx, bucket).await {
|
||||
Ok(snapshot) => opts.delete_replication_config_snapshot = Some(Arc::new(snapshot)),
|
||||
@@ -1887,7 +1616,6 @@ impl ECStore {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::core::TRANSITION_COMPLETE;
|
||||
use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState;
|
||||
use crate::bucket::replication::{
|
||||
ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_state_to_filemeta, replication_statuses_map,
|
||||
version_purge_statuses_map,
|
||||
@@ -1903,7 +1631,6 @@ mod tests {
|
||||
};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
use crate::storage_api_contracts::lifecycle::TransitionedObject;
|
||||
use bytes::Bytes;
|
||||
use std::io::Cursor;
|
||||
use std::sync::Arc;
|
||||
@@ -1924,55 +1651,6 @@ mod tests {
|
||||
|
||||
struct BodyCacheHookGuard;
|
||||
|
||||
#[test]
|
||||
fn tier_delete_entry_is_prepared_and_bound_to_source_generation() {
|
||||
let identity = [9_u8; 32];
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(identity),
|
||||
);
|
||||
let version_id = Uuid::from_u128(1);
|
||||
let data_dir = Uuid::from_u128(2);
|
||||
let source = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(version_id),
|
||||
data_dir: Some(data_dir),
|
||||
user_defined: Arc::new(metadata),
|
||||
transitioned_object: TransitionedObject {
|
||||
name: "remote/object".to_string(),
|
||||
version_id: "remote-version".to_string(),
|
||||
tier: "WARM".to_string(),
|
||||
status: TRANSITION_COMPLETE.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
..Default::default()
|
||||
};
|
||||
let entry = build_tier_delete_journal_entry(
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions {
|
||||
version_id: Some(version_id.to_string()),
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
&source,
|
||||
)
|
||||
.expect("transition source should produce a journal entry")
|
||||
.expect("completed transition should be journaled");
|
||||
|
||||
assert_eq!(entry.state, TierDeleteJournalState::Prepared);
|
||||
assert_eq!(entry.backend_identity, Some(identity));
|
||||
let data_dir_string = data_dir.to_string();
|
||||
assert_eq!(
|
||||
entry.source.as_ref().and_then(|source| source.data_dir.as_deref()),
|
||||
Some(data_dir_string.as_str())
|
||||
);
|
||||
}
|
||||
|
||||
impl Drop for BodyCacheHookGuard {
|
||||
fn drop(&mut self) {
|
||||
clear_get_object_body_cache_hook();
|
||||
|
||||
@@ -62,7 +62,6 @@ pub(super) fn authorization(oidc: &OidcSys, provider_id: String, claims: OidcCla
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::oidc::{make_test_sys, test_config};
|
||||
use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -100,43 +99,4 @@ mod tests {
|
||||
};
|
||||
assert!(string_list_claim(&ambiguous, "roles").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authorization_preserves_verified_claims_and_keeps_source_groups_distinct() {
|
||||
let mut config = test_config("corp");
|
||||
config.claim_prefix = "mapped-".to_string();
|
||||
config.roles_claim = "roles".to_string();
|
||||
let oidc = make_test_sys(vec![config]);
|
||||
let raw = HashMap::from([
|
||||
("iss".to_string(), json!("https://corp.example.test")),
|
||||
("department".to_string(), json!("engineering")),
|
||||
("roles".to_string(), json!(["reader", "admin"])),
|
||||
]);
|
||||
let authorization = authorization(
|
||||
&oidc,
|
||||
"corp".to_string(),
|
||||
OidcClaims {
|
||||
sub: " subject-123 ".to_string(),
|
||||
email: " user@example.test ".to_string(),
|
||||
username: " user ".to_string(),
|
||||
groups: vec![
|
||||
"source-ops".to_string(),
|
||||
"source-developers".to_string(),
|
||||
"source-ops".to_string(),
|
||||
],
|
||||
raw: raw.clone(),
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(authorization.provider_id, "corp");
|
||||
assert_eq!(authorization.claims.sub, " subject-123 ");
|
||||
assert_eq!(authorization.claims.email, " user@example.test ");
|
||||
assert_eq!(authorization.claims.username, " user ");
|
||||
assert_eq!(authorization.claims.groups, ["source-ops", "source-developers", "source-ops"]);
|
||||
assert_eq!(authorization.claims.raw, raw);
|
||||
assert_eq!(authorization.policies, ["mapped-source-developers", "mapped-source-ops"]);
|
||||
assert_eq!(authorization.groups, ["source-developers", "source-ops"]);
|
||||
assert_eq!(authorization.roles_claim_key.as_deref(), Some("roles"));
|
||||
assert_eq!(authorization.roles, ["reader", "admin"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -140,51 +140,29 @@ mod tests {
|
||||
};
|
||||
use crate::oidc::{OidcProviderConfig, OidcProviderSummary};
|
||||
use rustfs_credentials::Credentials;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum ProviderFailure {
|
||||
None,
|
||||
Exchange,
|
||||
Verification,
|
||||
Logout,
|
||||
}
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
struct TestProvider {
|
||||
with_policy: bool,
|
||||
with_group: bool,
|
||||
browser_provider_id: &'static str,
|
||||
web_provider_id: &'static str,
|
||||
failure: ProviderFailure,
|
||||
events: Arc<Mutex<Vec<&'static str>>>,
|
||||
expected_logout: (&'static str, &'static str),
|
||||
}
|
||||
|
||||
impl TestProvider {
|
||||
fn new(events: Arc<Mutex<Vec<&'static str>>>) -> Self {
|
||||
Self {
|
||||
with_policy: true,
|
||||
with_group: false,
|
||||
browser_provider_id: "default",
|
||||
web_provider_id: "default",
|
||||
failure: ProviderFailure::None,
|
||||
events,
|
||||
expected_logout: ("default", "id-token"),
|
||||
}
|
||||
}
|
||||
|
||||
fn record(&self, event: &'static str) {
|
||||
self.events.lock().expect("event log should not be poisoned").push(event);
|
||||
}
|
||||
|
||||
fn authorization(&self, provider_id: &str) -> FederatedAuthorization {
|
||||
fn authorization(&self) -> FederatedAuthorization {
|
||||
FederatedAuthorization {
|
||||
provider_id: provider_id.to_string(),
|
||||
provider_id: "default".to_string(),
|
||||
claims: FederatedClaims {
|
||||
sub: "subject".to_string(),
|
||||
email: String::new(),
|
||||
username: "user".to_string(),
|
||||
groups: vec!["source-group".to_string()],
|
||||
groups: Vec::new(),
|
||||
raw: Default::default(),
|
||||
},
|
||||
policies: if self.with_policy {
|
||||
@@ -192,11 +170,7 @@ mod tests {
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
groups: if self.with_group {
|
||||
vec!["developers".to_string()]
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
groups: Vec::new(),
|
||||
roles_claim_key: None,
|
||||
roles: Vec::new(),
|
||||
}
|
||||
@@ -232,11 +206,8 @@ mod tests {
|
||||
|
||||
async fn exchange_code(&self, _state: &str, _code: &str, _redirect_uri: &str) -> Result<FederatedCodeExchange> {
|
||||
self.record("exchange");
|
||||
if self.failure == ProviderFailure::Exchange {
|
||||
return Err(FederationError::CodeExchange("exchange failed".to_string()));
|
||||
}
|
||||
Ok(FederatedCodeExchange {
|
||||
authorization: self.authorization(self.browser_provider_id),
|
||||
authorization: self.authorization(),
|
||||
redirect_after: Some("/browser".to_string()),
|
||||
id_token: "id-token".to_string(),
|
||||
})
|
||||
@@ -244,18 +215,11 @@ mod tests {
|
||||
|
||||
async fn verify_web_identity_token(&self, _jwt: &str) -> Result<FederatedAuthorization> {
|
||||
self.record("verify");
|
||||
if self.failure == ProviderFailure::Verification {
|
||||
return Err(FederationError::TokenVerification("verification failed".to_string()));
|
||||
}
|
||||
Ok(self.authorization(self.web_provider_id))
|
||||
Ok(self.authorization())
|
||||
}
|
||||
|
||||
async fn create_logout_token(&self, provider_id: &str, id_token: &str) -> Result<String> {
|
||||
async fn create_logout_token(&self, _provider_id: &str, _id_token: &str) -> Result<String> {
|
||||
self.record("logout");
|
||||
assert_eq!((provider_id, id_token), self.expected_logout);
|
||||
if self.failure == ProviderFailure::Logout {
|
||||
return Err(FederationError::Logout("logout failed".to_string()));
|
||||
}
|
||||
Ok("logout-token".to_string())
|
||||
}
|
||||
|
||||
@@ -264,37 +228,19 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
struct RecordingBinding {
|
||||
fail: bool,
|
||||
struct CountingBinding {
|
||||
calls: AtomicUsize,
|
||||
events: Arc<Mutex<Vec<&'static str>>>,
|
||||
transactions: Mutex<Vec<(String, usize, Option<String>)>>,
|
||||
}
|
||||
|
||||
impl RecordingBinding {
|
||||
fn new(events: Arc<Mutex<Vec<&'static str>>>) -> Self {
|
||||
Self {
|
||||
fail: false,
|
||||
events,
|
||||
transactions: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl FederatedSessionBinding for RecordingBinding {
|
||||
impl FederatedSessionBinding for CountingBinding {
|
||||
async fn bind(
|
||||
&self,
|
||||
transaction: &FederatedSessionTransaction,
|
||||
) -> core::result::Result<Credentials, FederatedSessionBindingError> {
|
||||
self.calls.fetch_add(1, Ordering::Relaxed);
|
||||
self.events.lock().expect("event log should not be poisoned").push("bind");
|
||||
self.transactions.lock().expect("transactions should not be poisoned").push((
|
||||
transaction.authorization.provider_id.clone(),
|
||||
transaction.duration_seconds,
|
||||
transaction.session_policy.clone(),
|
||||
));
|
||||
if self.fail {
|
||||
return Err(FederatedSessionBindingError::Internal("binding failed".to_string()));
|
||||
}
|
||||
Ok(Credentials {
|
||||
access_key: transaction.authorization.claims.session_identity(),
|
||||
..Default::default()
|
||||
@@ -303,14 +249,16 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn callback_and_web_identity_preserve_provider_and_transaction_boundaries() {
|
||||
async fn callback_and_web_identity_share_session_binding() {
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut provider = TestProvider::new(events.clone());
|
||||
provider.browser_provider_id = "corp";
|
||||
provider.web_provider_id = "partner";
|
||||
provider.expected_logout = ("corp", "id-token");
|
||||
let provider = Arc::new(provider);
|
||||
let binding = Arc::new(RecordingBinding::new(events.clone()));
|
||||
let provider = Arc::new(TestProvider {
|
||||
with_policy: true,
|
||||
events: events.clone(),
|
||||
});
|
||||
let binding = Arc::new(CountingBinding {
|
||||
calls: AtomicUsize::new(0),
|
||||
events: events.clone(),
|
||||
});
|
||||
let service = FederatedIdentityService::new(FederatedIdentityRegistry::new(provider));
|
||||
|
||||
let login = service
|
||||
@@ -318,7 +266,6 @@ mod tests {
|
||||
.await
|
||||
.expect("callback flow should complete");
|
||||
assert_eq!(login.session.credentials.access_key, "user");
|
||||
assert_eq!(login.session.authorization.provider_id, "corp");
|
||||
assert_eq!(login.redirect_after.as_deref(), Some("/browser"));
|
||||
assert_eq!(login.logout_token, "logout-token");
|
||||
assert_eq!(
|
||||
@@ -328,32 +275,25 @@ mod tests {
|
||||
events.lock().expect("event log should not be poisoned").clear();
|
||||
|
||||
let web_identity = service
|
||||
.assume_role_with_web_identity("jwt", 7200, Some("session-policy".to_string()), binding.as_ref())
|
||||
.assume_role_with_web_identity("jwt", 3600, None, binding.as_ref())
|
||||
.await
|
||||
.expect("web identity flow should complete");
|
||||
assert_eq!(web_identity.credentials.access_key, "user");
|
||||
assert_eq!(web_identity.authorization.provider_id, "partner");
|
||||
assert_eq!(binding.calls.load(Ordering::Relaxed), 2);
|
||||
assert_eq!(events.lock().expect("event log should not be poisoned").as_slice(), ["verify", "bind"]);
|
||||
assert_eq!(
|
||||
binding
|
||||
.transactions
|
||||
.lock()
|
||||
.expect("transactions should not be poisoned")
|
||||
.as_slice(),
|
||||
[
|
||||
("corp".to_string(), 3600, None),
|
||||
("partner".to_string(), 7200, Some("session-policy".to_string())),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn web_identity_without_policy_or_group_is_not_bound() {
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut provider = TestProvider::new(events.clone());
|
||||
provider.with_policy = false;
|
||||
let provider = Arc::new(provider);
|
||||
let binding = Arc::new(RecordingBinding::new(events.clone()));
|
||||
let provider = Arc::new(TestProvider {
|
||||
with_policy: false,
|
||||
events: events.clone(),
|
||||
});
|
||||
let binding = Arc::new(CountingBinding {
|
||||
calls: AtomicUsize::new(0),
|
||||
events,
|
||||
});
|
||||
let service = FederatedIdentityService::new(FederatedIdentityRegistry::new(provider));
|
||||
|
||||
let error = service
|
||||
@@ -362,102 +302,6 @@ mod tests {
|
||||
.expect_err("authorization context is required");
|
||||
|
||||
assert!(matches!(error, FederationError::NoAuthorizationContext));
|
||||
assert_eq!(events.lock().expect("event log should not be poisoned").as_slice(), ["verify"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn web_identity_group_only_authorization_is_bound_once() {
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut provider = TestProvider::new(events.clone());
|
||||
provider.with_policy = false;
|
||||
provider.with_group = true;
|
||||
let provider = Arc::new(provider);
|
||||
let binding = Arc::new(RecordingBinding::new(events.clone()));
|
||||
let service = FederatedIdentityService::new(FederatedIdentityRegistry::new(provider));
|
||||
|
||||
let session = service
|
||||
.assume_role_with_web_identity("jwt", 3600, None, binding.as_ref())
|
||||
.await
|
||||
.expect("a mapped group is an authorization context");
|
||||
|
||||
assert!(session.authorization.policies.is_empty());
|
||||
assert_eq!(session.authorization.groups, ["developers"]);
|
||||
assert_eq!(events.lock().expect("event log should not be poisoned").as_slice(), ["verify", "bind"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn callback_failures_preserve_existing_side_effect_order() {
|
||||
for (provider_failure, binding_failure, expected_events) in [
|
||||
(ProviderFailure::Exchange, false, vec!["exchange"]),
|
||||
(ProviderFailure::None, true, vec!["exchange", "bind"]),
|
||||
(ProviderFailure::Logout, false, vec!["exchange", "bind", "logout"]),
|
||||
] {
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut provider = TestProvider::new(events.clone());
|
||||
provider.failure = provider_failure;
|
||||
let mut binding = RecordingBinding::new(events.clone());
|
||||
binding.fail = binding_failure;
|
||||
let binding = Arc::new(binding);
|
||||
let service = FederatedIdentityService::new(FederatedIdentityRegistry::new(Arc::new(provider)));
|
||||
|
||||
let error = service
|
||||
.complete_authorization_code("state", "code", "https://console.example/callback", 3600, binding.as_ref())
|
||||
.await
|
||||
.expect_err("the configured failure should be returned");
|
||||
|
||||
if provider_failure == ProviderFailure::Exchange {
|
||||
assert!(matches!(error, FederationError::CodeExchange(ref message) if message == "exchange failed"));
|
||||
} else if binding_failure {
|
||||
assert!(matches!(
|
||||
error,
|
||||
FederationError::Binding(FederatedSessionBindingError::Internal(ref message))
|
||||
if message == "binding failed"
|
||||
));
|
||||
} else {
|
||||
assert!(matches!(error, FederationError::Logout(ref message) if message == "logout failed"));
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
events.lock().expect("event log should not be poisoned").as_slice(),
|
||||
expected_events,
|
||||
"later callback steps must not run after a failure"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn web_identity_failures_preserve_existing_side_effect_order() {
|
||||
for (provider_failure, binding_failure, expected_events) in [
|
||||
(ProviderFailure::Verification, false, vec!["verify"]),
|
||||
(ProviderFailure::None, true, vec!["verify", "bind"]),
|
||||
] {
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut provider = TestProvider::new(events.clone());
|
||||
provider.failure = provider_failure;
|
||||
let mut binding = RecordingBinding::new(events.clone());
|
||||
binding.fail = binding_failure;
|
||||
let binding = Arc::new(binding);
|
||||
let service = FederatedIdentityService::new(FederatedIdentityRegistry::new(Arc::new(provider)));
|
||||
|
||||
let error = service
|
||||
.assume_role_with_web_identity("jwt", 3600, None, binding.as_ref())
|
||||
.await
|
||||
.expect_err("the configured failure should be returned");
|
||||
|
||||
if provider_failure == ProviderFailure::Verification {
|
||||
assert!(matches!(error, FederationError::TokenVerification(_)));
|
||||
} else {
|
||||
assert!(matches!(
|
||||
error,
|
||||
FederationError::Binding(FederatedSessionBindingError::Internal(ref message))
|
||||
if message == "binding failed"
|
||||
));
|
||||
}
|
||||
assert_eq!(
|
||||
events.lock().expect("event log should not be poisoned").as_slice(),
|
||||
expected_events,
|
||||
"later web identity steps must not run after a failure"
|
||||
);
|
||||
}
|
||||
assert_eq!(binding.calls.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
}
|
||||
|
||||
+38
-36
@@ -2007,42 +2007,6 @@ fn claim_value_type_for_log(value: Option<&serde_json::Value>) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn make_test_sys(configs: Vec<OidcProviderConfig>) -> OidcSys {
|
||||
let configs = configs.into_iter().map(|config| (config.id.clone(), config)).collect();
|
||||
OidcSys {
|
||||
configs,
|
||||
provider_states: RwLock::new(HashMap::new()),
|
||||
state_store: OidcStateStore::new(),
|
||||
http_client: ReqwestHttpClient::new().expect("failed to initialize OIDC HTTP clients"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test_config(id: &str) -> OidcProviderConfig {
|
||||
OidcProviderConfig {
|
||||
id: id.to_string(),
|
||||
enabled: true,
|
||||
config_url: format!("https://example.com/{id}/.well-known/openid-configuration"),
|
||||
issuer: None,
|
||||
client_id: "client-id".to_string(),
|
||||
client_secret: None,
|
||||
scopes: vec!["openid".to_string()],
|
||||
other_audiences: vec![],
|
||||
redirect_uri: None,
|
||||
redirect_uri_dynamic: true,
|
||||
claim_name: "groups".to_string(),
|
||||
claim_prefix: String::new(),
|
||||
role_policy: String::new(),
|
||||
display_name: id.to_string(),
|
||||
groups_claim: "groups".to_string(),
|
||||
roles_claim: String::new(),
|
||||
email_claim: "email".to_string(),
|
||||
username_claim: "preferred_username".to_string(),
|
||||
hide_from_ui: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -2994,6 +2958,44 @@ mod tests {
|
||||
handle.join().expect("mock body server thread should exit");
|
||||
}
|
||||
|
||||
/// Helper to create an OidcSys with configs only (no provider states needed).
|
||||
fn make_test_sys(configs: Vec<OidcProviderConfig>) -> OidcSys {
|
||||
let mut config_map = HashMap::new();
|
||||
for c in configs {
|
||||
config_map.insert(c.id.clone(), c);
|
||||
}
|
||||
OidcSys {
|
||||
configs: config_map,
|
||||
provider_states: RwLock::new(HashMap::new()),
|
||||
state_store: OidcStateStore::new(),
|
||||
http_client: ReqwestHttpClient::new().expect("failed to initialize OIDC HTTP clients"),
|
||||
}
|
||||
}
|
||||
|
||||
fn test_config(id: &str) -> OidcProviderConfig {
|
||||
OidcProviderConfig {
|
||||
id: id.to_string(),
|
||||
enabled: true,
|
||||
config_url: format!("https://example.com/{id}/.well-known/openid-configuration"),
|
||||
issuer: None,
|
||||
client_id: "client-id".to_string(),
|
||||
client_secret: None,
|
||||
scopes: vec!["openid".to_string()],
|
||||
other_audiences: vec![],
|
||||
redirect_uri: None,
|
||||
redirect_uri_dynamic: true,
|
||||
claim_name: "groups".to_string(),
|
||||
claim_prefix: "".to_string(),
|
||||
role_policy: "".to_string(),
|
||||
display_name: id.to_string(),
|
||||
groups_claim: "groups".to_string(),
|
||||
roles_claim: String::new(),
|
||||
email_claim: "email".to_string(),
|
||||
username_claim: "preferred_username".to_string(),
|
||||
hide_from_ui: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_oidc_provider_config_debug_redacts_client_secret() {
|
||||
let config = OidcProviderConfig {
|
||||
|
||||
@@ -138,8 +138,6 @@ impl Default for OidcStateStore {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Barrier;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_state_store_insert_and_take() {
|
||||
@@ -199,41 +197,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn auth_state_is_consumed_once_under_concurrent_take() {
|
||||
let store = OidcStateStore::new();
|
||||
store
|
||||
.insert(
|
||||
"state_once".to_string(),
|
||||
OidcAuthSession {
|
||||
provider_id: "corp".to_string(),
|
||||
pkce_verifier: "verifier".to_string(),
|
||||
nonce: "nonce".to_string(),
|
||||
redirect_after: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let first_store = store.clone();
|
||||
let second_store = store.clone();
|
||||
let barrier = Arc::new(Barrier::new(3));
|
||||
let first_barrier = Arc::clone(&barrier);
|
||||
let first = tokio::spawn(async move {
|
||||
first_barrier.wait().await;
|
||||
first_store.take("state_once").await
|
||||
});
|
||||
let second_barrier = Arc::clone(&barrier);
|
||||
let second = tokio::spawn(async move {
|
||||
second_barrier.wait().await;
|
||||
second_store.take("state_once").await
|
||||
});
|
||||
barrier.wait().await;
|
||||
let first = first.await.expect("first state consumer should finish");
|
||||
let second = second.await.expect("second state consumer should finish");
|
||||
|
||||
assert_eq!(first.is_some() as usize + second.is_some() as usize, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_logout_state_store_insert_and_take() {
|
||||
let store = OidcStateStore::new();
|
||||
|
||||
@@ -246,7 +246,7 @@ pub use process_lock_metrics::{
|
||||
record_write_lock_held_acquire, record_write_lock_held_release, snapshot_process_lock_counts, snapshot_process_lock_events,
|
||||
snapshot_process_platform_stats,
|
||||
};
|
||||
pub use s3_api_metrics::{S3OperationMetricSnapshot, init_s3_metrics, record_s3_op, s3_op_metrics_snapshot};
|
||||
pub use s3_api_metrics::{init_s3_metrics, record_s3_op};
|
||||
pub use sampler::{
|
||||
ProcessResourceSnapshot, ProcessSampler, ProcessStatusSnapshot, ProcessSystemSnapshot, snapshot_process_platform,
|
||||
snapshot_process_resource, snapshot_process_resource_and_system, snapshot_process_resource_and_system_with,
|
||||
|
||||
@@ -14,25 +14,8 @@
|
||||
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
const S3_OPS_METRIC: &str = "rustfs_s3_operations_total";
|
||||
static S3_OP_COUNTERS: OnceLock<Box<[AtomicU64]>> = OnceLock::new();
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct S3OperationMetricSnapshot {
|
||||
pub op: &'static str,
|
||||
pub total: u64,
|
||||
}
|
||||
|
||||
fn s3_op_counters() -> &'static [AtomicU64] {
|
||||
S3_OP_COUNTERS.get_or_init(|| {
|
||||
std::iter::repeat_with(|| AtomicU64::new(0))
|
||||
.take(S3Operation::ALL.len())
|
||||
.collect::<Vec<_>>()
|
||||
.into_boxed_slice()
|
||||
})
|
||||
}
|
||||
|
||||
/// Record a handled S3 API operation.
|
||||
///
|
||||
@@ -43,22 +26,9 @@ fn s3_op_counters() -> &'static [AtomicU64] {
|
||||
/// This mirrors MinIO, which never labels its default operation counters with
|
||||
/// bucket. The `op` dimension is bounded (<= 122 variants).
|
||||
pub fn record_s3_op(op: S3Operation) {
|
||||
if let Some(counter) = s3_op_counters().get(op.metric_index()) {
|
||||
counter.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
counter!(S3_OPS_METRIC, "op" => op.as_str()).increment(1);
|
||||
}
|
||||
|
||||
pub fn s3_op_metrics_snapshot() -> Vec<S3OperationMetricSnapshot> {
|
||||
S3Operation::ALL
|
||||
.iter()
|
||||
.filter_map(|op| {
|
||||
let total = s3_op_counters().get(op.metric_index())?.load(Ordering::Relaxed);
|
||||
(total > 0).then_some(S3OperationMetricSnapshot { op: op.as_str(), total })
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn init_s3_metrics() {
|
||||
static METRICS_DESC_INIT: OnceLock<()> = OnceLock::new();
|
||||
METRICS_DESC_INIT.get_or_init(|| {
|
||||
@@ -72,9 +42,6 @@ mod tests {
|
||||
use metrics::with_local_recorder;
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Mutex;
|
||||
|
||||
static S3_OP_TEST_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
/// Collect the label-key sets recorded against `rustfs_s3_operations_total`.
|
||||
fn ops_metric_label_key_sets(recorder: &DebuggingRecorder) -> Vec<HashSet<String>> {
|
||||
@@ -93,7 +60,6 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn record_s3_op_labels_by_op_only_no_bucket() {
|
||||
let _guard = S3_OP_TEST_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let label_key_sets = ops_metric_label_key_sets(&recorder);
|
||||
|
||||
@@ -109,7 +75,6 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn record_s3_op_cardinality_bounded_by_distinct_ops() {
|
||||
let _guard = S3_OP_TEST_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
|
||||
@@ -148,24 +113,4 @@ mod tests {
|
||||
"series count must equal the number of distinct ops, never the bucket count"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s3_op_metrics_snapshot_reports_recorded_totals() {
|
||||
let _guard = S3_OP_TEST_LOCK.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let before = s3_op_metrics_snapshot()
|
||||
.into_iter()
|
||||
.find(|snapshot| snapshot.op == S3Operation::GetObject.as_str())
|
||||
.map(|snapshot| snapshot.total)
|
||||
.unwrap_or_default();
|
||||
|
||||
record_s3_op(S3Operation::GetObject);
|
||||
record_s3_op(S3Operation::GetObject);
|
||||
|
||||
let after = s3_op_metrics_snapshot()
|
||||
.into_iter()
|
||||
.find(|snapshot| snapshot.op == S3Operation::GetObject.as_str())
|
||||
.map(|snapshot| snapshot.total)
|
||||
.expect("GetObject snapshot should be present after recording");
|
||||
assert_eq!(after, before + 2);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,9 +99,6 @@ tokio = { workspace = true, features = ["net", "test-util"] }
|
||||
# Replays canned AWS KMS HTTP exchanges so the AWS backend tests stay offline.
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["test-util"] }
|
||||
http = { workspace = true }
|
||||
# Captures warning events in format-compatibility tests without installing a
|
||||
# process-wide subscriber.
|
||||
tracing-subscriber = { workspace = true, features = ["fmt"] }
|
||||
|
||||
[features]
|
||||
default = []
|
||||
|
||||
+90
-25
@@ -21,6 +21,7 @@ use crate::config::{
|
||||
redacted_secret, redacted_secret_option,
|
||||
};
|
||||
use crate::service_manager::KmsServiceStatus;
|
||||
use crate::types::KeyMetadata;
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
@@ -1198,19 +1199,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// The shapes this crate owns, and only those.
|
||||
///
|
||||
/// The first four are served verbatim by the dynamic-configuration admin
|
||||
/// handlers, so pinning them here pins the wire. The last three never
|
||||
/// reach a socket: `ObjectEncryptionService` returns them and the admin
|
||||
/// layer answers with its own `KmsKeyMetadataResponse` instead, so what
|
||||
/// they pin is this crate's public API, not the wire.
|
||||
///
|
||||
/// No key-management response belongs in this test. Those endpoints are
|
||||
/// served from types defined in the `rustfs` crate, and a copy here could
|
||||
/// only ever agree with them by accident — see
|
||||
/// `kms_key_admin_responses_have_stable_json_shapes` in
|
||||
/// `rustfs/src/admin/handlers/kms_keys.rs`.
|
||||
#[test]
|
||||
fn kms_management_responses_have_stable_json_shapes() {
|
||||
insta::assert_json_snapshot!(
|
||||
@@ -1246,6 +1234,41 @@ mod tests {
|
||||
config_summary: None,
|
||||
})
|
||||
);
|
||||
insta::assert_json_snapshot!(
|
||||
"kms_delete_key_response",
|
||||
stable_json_value(DeleteKeyResponse {
|
||||
success: true,
|
||||
message: "key scheduled for deletion".to_string(),
|
||||
key_id: "key-a".to_string(),
|
||||
deletion_date: Some("2026-07-01T00:00:00Z".to_string()),
|
||||
})
|
||||
);
|
||||
insta::assert_json_snapshot!(
|
||||
"kms_list_keys_response",
|
||||
stable_json_value(ListKeysResponse {
|
||||
success: true,
|
||||
message: "keys listed".to_string(),
|
||||
keys: vec!["key-a".to_string(), "key-b".to_string()],
|
||||
truncated: true,
|
||||
next_marker: Some("key-b".to_string()),
|
||||
})
|
||||
);
|
||||
insta::assert_json_snapshot!(
|
||||
"kms_describe_key_response_missing",
|
||||
stable_json_value(DescribeKeyResponse {
|
||||
success: false,
|
||||
message: "key not found".to_string(),
|
||||
key_metadata: None,
|
||||
})
|
||||
);
|
||||
insta::assert_json_snapshot!(
|
||||
"kms_cancel_key_deletion_response",
|
||||
stable_json_value(CancelKeyDeletionResponse {
|
||||
success: true,
|
||||
message: "key deletion canceled".to_string(),
|
||||
key_id: "key-a".to_string(),
|
||||
})
|
||||
);
|
||||
insta::assert_json_snapshot!(
|
||||
"kms_update_key_description_response",
|
||||
stable_json_value(UpdateKeyDescriptionResponse {
|
||||
@@ -1276,18 +1299,60 @@ mod tests {
|
||||
// ========================================
|
||||
// Key Management API Types
|
||||
// ========================================
|
||||
//
|
||||
// What remains here is the key-metadata trio, and nothing else belongs.
|
||||
// Create, delete, list, describe and cancel-deletion are served from types
|
||||
// defined in the `rustfs` crate (`rustfs/src/admin/handlers/kms_keys.rs`)
|
||||
// carrying fields this crate knows nothing about, so a copy here would shadow
|
||||
// `crate::types` under the same name while agreeing with the wire only by
|
||||
// accident.
|
||||
//
|
||||
// The same holds for `DeleteKeyRequest`: it lives in `crate::types` alone, so
|
||||
// the immediate-deletion gate (`force_immediate` + `confirm_key_id`) has
|
||||
// exactly one definition and cannot be silently dropped by deserializing into
|
||||
// a copy that lacks it.
|
||||
|
||||
/// JSON shape returned by the admin delete-key endpoint.
|
||||
///
|
||||
/// The delete *request* shape lives in [`crate::types::DeleteKeyRequest`] —
|
||||
/// there is deliberately no copy here, because the immediate-deletion gate
|
||||
/// (`force_immediate` + `confirm_key_id`) must have exactly one definition.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DeleteKeyResponse {
|
||||
/// Success flag
|
||||
pub success: bool,
|
||||
/// Status message
|
||||
pub message: String,
|
||||
/// Key ID that was deleted or scheduled for deletion
|
||||
pub key_id: String,
|
||||
/// Deletion date (if scheduled)
|
||||
pub deletion_date: Option<String>,
|
||||
}
|
||||
|
||||
/// Response from list keys operation
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ListKeysResponse {
|
||||
/// Success flag
|
||||
pub success: bool,
|
||||
/// Status message
|
||||
pub message: String,
|
||||
/// List of key IDs
|
||||
pub keys: Vec<String>,
|
||||
/// Whether more keys are available
|
||||
pub truncated: bool,
|
||||
/// Next marker for pagination
|
||||
pub next_marker: Option<String>,
|
||||
}
|
||||
|
||||
/// Response from describe key operation
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DescribeKeyResponse {
|
||||
/// Success flag
|
||||
pub success: bool,
|
||||
/// Status message
|
||||
pub message: String,
|
||||
/// Key metadata
|
||||
pub key_metadata: Option<KeyMetadata>,
|
||||
}
|
||||
|
||||
/// Response from cancel key deletion operation
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CancelKeyDeletionResponse {
|
||||
/// Success flag
|
||||
pub success: bool,
|
||||
/// Status message
|
||||
pub message: String,
|
||||
/// Key ID
|
||||
pub key_id: String,
|
||||
}
|
||||
|
||||
/// Request to update key description
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
|
||||
@@ -309,11 +309,11 @@ impl AwsKmsBackend {
|
||||
loader = loader.region(aws_sdk_kms::config::Region::new(region.clone()));
|
||||
}
|
||||
let sdk_config = loader.load().await;
|
||||
let Some(region) = sdk_config.region() else {
|
||||
if sdk_config.region().is_none() {
|
||||
return Err(KmsError::configuration_error(
|
||||
"AWS KMS backend could not resolve a region; set the backend region or AWS_REGION",
|
||||
));
|
||||
};
|
||||
}
|
||||
|
||||
let mut builder = aws_sdk_kms::config::Builder::from(&sdk_config)
|
||||
// `crate::policy` owns retries and timeouts; leaving the SDK's own
|
||||
@@ -324,18 +324,13 @@ impl AwsKmsBackend {
|
||||
builder = builder.endpoint_url(endpoint_url);
|
||||
}
|
||||
|
||||
Ok(Self::with_client(
|
||||
aws_sdk_kms::Client::from_conf(builder.build()),
|
||||
&config,
|
||||
aws_backend_config.endpoint_url.as_deref().unwrap_or_default(),
|
||||
region.as_ref(),
|
||||
))
|
||||
Ok(Self::with_client(aws_sdk_kms::Client::from_conf(builder.build()), &config))
|
||||
}
|
||||
|
||||
fn with_client(client: aws_sdk_kms::Client, config: &KmsConfig, endpoint: &str, region: &str) -> Self {
|
||||
fn with_client(client: aws_sdk_kms::Client, config: &KmsConfig) -> Self {
|
||||
Self {
|
||||
client,
|
||||
retry: RetryPolicy::for_backend(config, "aws", endpoint, Some(region), "operations"),
|
||||
retry: RetryPolicy::from_config(config),
|
||||
cancel: CancellationToken::new(),
|
||||
}
|
||||
}
|
||||
@@ -827,11 +822,10 @@ impl KmsBackend for AwsKmsBackend {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_sdk_kms::config::{BehaviorVersion, Credentials, Region};
|
||||
use aws_smithy_http_client::test_util::{NeverClient, ReplayEvent, StaticReplayClient};
|
||||
use aws_smithy_http_client::test_util::{ReplayEvent, StaticReplayClient};
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
/// AWS KMS speaks awsJson1_1; every request goes to `/` on the regional
|
||||
/// endpoint, so the replayed request side carries no useful assertion.
|
||||
@@ -863,15 +857,6 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
fn scripted_endpoint() -> String {
|
||||
static NEXT_SCRIPTED_BACKEND_ID: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
format!(
|
||||
"https://scripted-{}.example.invalid",
|
||||
NEXT_SCRIPTED_BACKEND_ID.fetch_add(1, Ordering::Relaxed)
|
||||
)
|
||||
}
|
||||
|
||||
fn scripted_backend(events: Vec<ReplayEvent>) -> (StaticReplayClient, AwsKmsBackend) {
|
||||
let http_client = StaticReplayClient::new(events);
|
||||
let sdk_config = aws_sdk_kms::Config::builder()
|
||||
@@ -882,20 +867,10 @@ mod tests {
|
||||
.retry_config(aws_sdk_kms::config::retry::RetryConfig::disabled())
|
||||
.build();
|
||||
let kms_config = KmsConfig::aws(Some("us-east-1".to_string()));
|
||||
let endpoint = scripted_endpoint();
|
||||
let backend = AwsKmsBackend::with_client(aws_sdk_kms::Client::from_conf(sdk_config), &kms_config, &endpoint, "us-east-1");
|
||||
let backend = AwsKmsBackend::with_client(aws_sdk_kms::Client::from_conf(sdk_config), &kms_config);
|
||||
(http_client, backend)
|
||||
}
|
||||
|
||||
fn aws_config(endpoint: &str, region: &str) -> KmsConfig {
|
||||
let mut config = KmsConfig::aws(Some(region.to_owned()));
|
||||
let BackendConfig::Aws(aws) = &mut config.backend_config else {
|
||||
panic!("AWS constructor must create an AWS backend configuration");
|
||||
};
|
||||
aws.endpoint_url = Some(endpoint.to_owned());
|
||||
config
|
||||
}
|
||||
|
||||
fn key_metadata_json(key_id: &str, state: &str) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"KeyMetadata": {
|
||||
@@ -925,27 +900,6 @@ mod tests {
|
||||
.expect("capabilities should deserialize into a flat bool map")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn aws_backend_new_identity_includes_endpoint_and_region() {
|
||||
let endpoint = scripted_endpoint();
|
||||
let first = AwsKmsBackend::new(aws_config(&endpoint, "us-east-1"))
|
||||
.await
|
||||
.expect("first AWS backend");
|
||||
let matching = AwsKmsBackend::new(aws_config(&endpoint, "us-east-1"))
|
||||
.await
|
||||
.expect("matching AWS backend");
|
||||
let other_region = AwsKmsBackend::new(aws_config(&endpoint, "us-west-2"))
|
||||
.await
|
||||
.expect("other-region AWS backend");
|
||||
let other_endpoint = AwsKmsBackend::new(aws_config(&scripted_endpoint(), "us-east-1"))
|
||||
.await
|
||||
.expect("other-endpoint AWS backend");
|
||||
|
||||
assert!(first.retry.shares_active_capacity_with(&matching.retry));
|
||||
assert!(!first.retry.shares_active_capacity_with(&other_region.retry));
|
||||
assert!(!first.retry.shares_active_capacity_with(&other_endpoint.retry));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn aws_backend_capabilities_golden() {
|
||||
let (_http, backend) = scripted_backend(Vec::new());
|
||||
@@ -1054,34 +1008,6 @@ mod tests {
|
||||
assert_eq!(http_client.actual_requests().count(), 3, "both throttled attempts should be replayed");
|
||||
}
|
||||
|
||||
/// A connector that never responds must be cut off by the backend's
|
||||
/// per-attempt timeout rather than hanging the KMS operation indefinitely.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn stalled_aws_request_is_cut_off_by_the_attempt_timeout() {
|
||||
let never_client = NeverClient::new();
|
||||
let sdk_config = aws_sdk_kms::Config::builder()
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.region(Region::new("us-east-1"))
|
||||
.credentials_provider(Credentials::new("AKIDTEST", "secret", None, None, "scripted"))
|
||||
.http_client(never_client.clone())
|
||||
.retry_config(aws_sdk_kms::config::retry::RetryConfig::disabled())
|
||||
.build();
|
||||
let mut kms_config = KmsConfig::aws(Some("us-east-1".to_string()));
|
||||
kms_config.timeout = std::time::Duration::from_millis(5_000);
|
||||
kms_config.retry_attempts = 1;
|
||||
let backend = AwsKmsBackend::with_client(aws_sdk_kms::Client::from_conf(sdk_config), &kms_config, "", "us-east-1");
|
||||
|
||||
let error = backend
|
||||
.describe_key(DescribeKeyRequest {
|
||||
key_id: "stalled-key".to_string(),
|
||||
})
|
||||
.await
|
||||
.expect_err("a stalled AWS request must be cut off by the attempt timeout");
|
||||
|
||||
assert!(matches!(error, KmsError::OperationTimedOut { .. }), "unexpected error: {error:?}");
|
||||
assert_eq!(never_client.num_calls(), 1, "one configured attempt must reach the connector");
|
||||
}
|
||||
|
||||
/// Access denial is deterministic: replaying it cannot help and would only
|
||||
/// multiply the audit trail of denied calls.
|
||||
#[tokio::test(start_paused = true)]
|
||||
@@ -1194,58 +1120,6 @@ mod tests {
|
||||
assert_eq!(http_client.actual_requests().count(), 0, "no key may be created in AWS");
|
||||
}
|
||||
|
||||
/// AWS intentionally does not use the shared lifecycle contract driver.
|
||||
/// The driver requires disabled/pending keys to decrypt, expects cancelling
|
||||
/// deletion to re-enable a key, and creates keys by caller-assigned name;
|
||||
/// AWS rejects the decryption assumption, leaves a cancelled key
|
||||
/// disabled, and cannot honour the third.
|
||||
#[tokio::test]
|
||||
async fn aws_backend_shared_contract_exemption_is_pinned() {
|
||||
let (_http, backend) = scripted_backend(vec![error_event(400, "DisabledException", "key is disabled")]);
|
||||
let error = backend
|
||||
.decrypt(DecryptRequest {
|
||||
ciphertext: b"blob".to_vec(),
|
||||
encryption_context: HashMap::new(),
|
||||
grant_tokens: Vec::new(),
|
||||
})
|
||||
.await
|
||||
.expect_err("AWS must reject decrypt with a disabled key");
|
||||
assert!(matches!(error, KmsError::InvalidOperation { .. }), "unexpected error: {error:?}");
|
||||
|
||||
let (_http, backend) = scripted_backend(vec![error_event(400, "KMSInvalidStateException", "key is pending deletion")]);
|
||||
let error = backend
|
||||
.decrypt(DecryptRequest {
|
||||
ciphertext: b"blob".to_vec(),
|
||||
encryption_context: HashMap::new(),
|
||||
grant_tokens: Vec::new(),
|
||||
})
|
||||
.await
|
||||
.expect_err("AWS must reject decrypt with a pending-deletion key");
|
||||
assert!(matches!(error, KmsError::InvalidOperation { .. }), "unexpected error: {error:?}");
|
||||
|
||||
let (_http, backend) = scripted_backend(vec![
|
||||
ok_event(serde_json::json!({})),
|
||||
ok_event(key_metadata_json("test-key", "Disabled")),
|
||||
]);
|
||||
let response = backend
|
||||
.cancel_key_deletion(CancelKeyDeletionRequest {
|
||||
key_id: "test-key".to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("AWS cancellation should complete");
|
||||
assert_eq!(response.key_metadata.key_state, KeyState::Disabled);
|
||||
|
||||
let (_http, backend) = scripted_backend(Vec::new());
|
||||
let error = backend
|
||||
.create_key(CreateKeyRequest {
|
||||
key_name: Some("contract-key".to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("AWS cannot create a key under a caller-assigned name");
|
||||
assert!(matches!(error, KmsError::UnsupportedCapability { .. }), "unexpected error: {error:?}");
|
||||
}
|
||||
|
||||
/// AWS rejects `Limit: 0` outright, so the request cannot be forwarded as
|
||||
/// written; clamping it up to one would answer a caller that asked for no
|
||||
/// keys with a key. The empty page is served locally instead.
|
||||
|
||||
@@ -22,7 +22,6 @@ use crate::config::KmsConfig;
|
||||
use crate::config::LocalConfig;
|
||||
use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummary};
|
||||
use crate::types::*;
|
||||
use aes_gcm::{
|
||||
Aes256Gcm, Key, Nonce,
|
||||
@@ -33,16 +32,11 @@ use async_trait::async_trait;
|
||||
use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
|
||||
use jiff::Zoned;
|
||||
use rand::RngExt;
|
||||
use serde::de::{self, IgnoredAny, MapAccess, Visitor};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
use tokio::fs;
|
||||
use tracing::{debug, warn};
|
||||
@@ -486,32 +480,9 @@ pub(crate) enum StoredKeyProtection {
|
||||
PlaintextDevOnly,
|
||||
}
|
||||
|
||||
pub(crate) const UNKNOWN_STORED_KEY_PROTECTION: &str = "unknown-at-rest-protection";
|
||||
const MAX_PROTECTION_MARKER_RAW_BYTES: usize = 128;
|
||||
|
||||
impl UnknownFieldSummary {
|
||||
fn record_for_local_key(&self) {
|
||||
let Some((field, field_name_truncated, field_count)) = self.record("local-key-record") else {
|
||||
return;
|
||||
};
|
||||
|
||||
static RECORDS_WITH_UNKNOWN_FIELDS: AtomicU64 = AtomicU64::new(0);
|
||||
let observed_records = RECORDS_WITH_UNKNOWN_FIELDS.fetch_add(1, Ordering::Relaxed).saturating_add(1);
|
||||
if observed_records.is_power_of_two() {
|
||||
tracing::warn!(
|
||||
field = ?field,
|
||||
field_name_truncated,
|
||||
field_count,
|
||||
observed_records,
|
||||
"Local KMS key record contains unknown fields"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reports whether the record's `at_rest_protection` value is unknown to this
|
||||
/// build. `false` means the marker is absent (pre-beta.9 records), null, or
|
||||
/// names a protection mode this build implements.
|
||||
/// The record's `at_rest_protection` value when this build cannot interpret
|
||||
/// it, rendered for diagnostics. `Ok(None)` means the marker is absent
|
||||
/// (pre-beta.9 records) or names a protection mode this build implements.
|
||||
///
|
||||
/// Every reader of a stored key record must consult this before its own
|
||||
/// schema parse. Letting a strict [`StoredKeyProtection`] field fail inside a
|
||||
@@ -519,38 +490,29 @@ impl UnknownFieldSummary {
|
||||
/// operator who reads corruption starts a disaster recovery instead of a
|
||||
/// version rollback. The probe deliberately ignores every other field, so the
|
||||
/// verdict is available even for records whose schema this build cannot
|
||||
/// satisfy. The raw marker is borrowed and length-bounded before enum parsing;
|
||||
/// it is never propagated into a caller-visible error or diagnostic.
|
||||
/// satisfy, and no key material is copied out of the caller's buffer.
|
||||
///
|
||||
/// `Err` carries the JSON error so callers can keep their own classification
|
||||
/// for bytes that are not a record at all.
|
||||
pub(crate) fn has_unknown_protection_marker(record: &[u8]) -> serde_json::Result<bool> {
|
||||
pub(crate) fn unknown_protection_marker(record: &[u8]) -> serde_json::Result<Option<String>> {
|
||||
#[derive(Deserialize)]
|
||||
struct MarkerProbe<'a> {
|
||||
struct MarkerProbe {
|
||||
#[serde(default)]
|
||||
#[serde(borrow)]
|
||||
at_rest_protection: Option<&'a serde_json::value::RawValue>,
|
||||
at_rest_protection: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
let Some(marker) = serde_json::from_slice::<MarkerProbe>(record)?.at_rest_protection else {
|
||||
return Ok(false);
|
||||
return Ok(None);
|
||||
};
|
||||
if marker.get().len() > MAX_PROTECTION_MARKER_RAW_BYTES {
|
||||
return Ok(true);
|
||||
if serde_json::from_value::<StoredKeyProtection>(marker.clone()).is_ok() {
|
||||
return Ok(None);
|
||||
}
|
||||
if serde_json::from_str::<StoredKeyProtection>(marker.get()).is_ok() {
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(true)
|
||||
Ok(Some(marker.as_str().map(str::to_owned).unwrap_or_else(|| marker.to_string())))
|
||||
}
|
||||
|
||||
/// Serializable representation of a master key stored on disk
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct StoredMasterKey {
|
||||
/// Persisted record schema version. Records written before this field was
|
||||
/// introduced default to version 1 during deserialization.
|
||||
#[serde(default = "default_stored_master_key_format_version")]
|
||||
format_version: u32,
|
||||
key_id: String,
|
||||
version: u32,
|
||||
algorithm: String,
|
||||
@@ -575,205 +537,6 @@ struct StoredMasterKey {
|
||||
at_rest_protection: StoredKeyProtection,
|
||||
}
|
||||
|
||||
pub(crate) const STORED_MASTER_KEY_FORMAT_VERSION: u32 = 1;
|
||||
|
||||
fn default_stored_master_key_format_version() -> u32 {
|
||||
STORED_MASTER_KEY_FORMAT_VERSION
|
||||
}
|
||||
|
||||
/// Read only the schema marker before attempting the complete key-record
|
||||
/// decode. A future record may add or remove required fields, but its version
|
||||
/// still needs to be reported as unsupported rather than as generic corruption.
|
||||
pub(crate) fn stored_master_key_format_version(record: &[u8]) -> serde_json::Result<u32> {
|
||||
#[derive(Deserialize)]
|
||||
struct FormatProbe {
|
||||
#[serde(default = "default_stored_master_key_format_version")]
|
||||
format_version: u32,
|
||||
}
|
||||
|
||||
Ok(serde_json::from_slice::<FormatProbe>(record)?.format_version)
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for StoredMasterKey {
|
||||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
enum Field {
|
||||
FormatVersion,
|
||||
KeyId,
|
||||
Version,
|
||||
Algorithm,
|
||||
Usage,
|
||||
Status,
|
||||
Description,
|
||||
Metadata,
|
||||
CreatedAt,
|
||||
RotatedAt,
|
||||
CreatedBy,
|
||||
DeletionDate,
|
||||
EncryptedKeyMaterial,
|
||||
Nonce,
|
||||
AtRestProtection,
|
||||
Unknown(BoundedUnknownFieldName),
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for Field {
|
||||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
struct FieldVisitor;
|
||||
|
||||
impl Visitor<'_> for FieldVisitor {
|
||||
type Value = Field;
|
||||
|
||||
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("a Local KMS key record field name")
|
||||
}
|
||||
|
||||
fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E>
|
||||
where
|
||||
E: de::Error,
|
||||
{
|
||||
Ok(match value {
|
||||
"format_version" => Field::FormatVersion,
|
||||
"key_id" => Field::KeyId,
|
||||
"version" => Field::Version,
|
||||
"algorithm" => Field::Algorithm,
|
||||
"usage" => Field::Usage,
|
||||
"status" => Field::Status,
|
||||
"description" => Field::Description,
|
||||
"metadata" => Field::Metadata,
|
||||
"created_at" => Field::CreatedAt,
|
||||
"rotated_at" => Field::RotatedAt,
|
||||
"created_by" => Field::CreatedBy,
|
||||
"deletion_date" => Field::DeletionDate,
|
||||
"encrypted_key_material" => Field::EncryptedKeyMaterial,
|
||||
"nonce" => Field::Nonce,
|
||||
"at_rest_protection" => Field::AtRestProtection,
|
||||
_ => Field::Unknown(BoundedUnknownFieldName::new(value)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_identifier(FieldVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] Zoned);
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct OptionalZonedValue(#[serde(with = "crate::time_serde::option_zoned")] Option<Zoned>);
|
||||
|
||||
struct StoredMasterKeyVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for StoredMasterKeyVisitor {
|
||||
type Value = StoredMasterKey;
|
||||
|
||||
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("a Local KMS key record")
|
||||
}
|
||||
|
||||
fn visit_map<A>(self, mut map: A) -> std::result::Result<Self::Value, A::Error>
|
||||
where
|
||||
A: MapAccess<'de>,
|
||||
{
|
||||
macro_rules! read_field {
|
||||
($slot:ident, $name:literal) => {{
|
||||
if $slot.is_some() {
|
||||
return Err(de::Error::duplicate_field($name));
|
||||
}
|
||||
$slot = Some(map.next_value()?);
|
||||
}};
|
||||
}
|
||||
|
||||
let mut format_version = None;
|
||||
let mut key_id = None;
|
||||
let mut version = None;
|
||||
let mut algorithm = None;
|
||||
let mut usage = None;
|
||||
let mut status = None;
|
||||
let mut description = None;
|
||||
let mut metadata = None;
|
||||
let mut created_at: Option<ZonedValue> = None;
|
||||
let mut rotated_at: Option<OptionalZonedValue> = None;
|
||||
let mut created_by = None;
|
||||
let mut deletion_date: Option<OptionalZonedValue> = None;
|
||||
let mut encrypted_key_material = None;
|
||||
let mut nonce = None;
|
||||
let mut at_rest_protection = None;
|
||||
let mut unknown_fields = UnknownFieldSummary::default();
|
||||
|
||||
while let Some(field) = map.next_key()? {
|
||||
match field {
|
||||
Field::FormatVersion => read_field!(format_version, "format_version"),
|
||||
Field::KeyId => read_field!(key_id, "key_id"),
|
||||
Field::Version => read_field!(version, "version"),
|
||||
Field::Algorithm => read_field!(algorithm, "algorithm"),
|
||||
Field::Usage => read_field!(usage, "usage"),
|
||||
Field::Status => read_field!(status, "status"),
|
||||
Field::Description => read_field!(description, "description"),
|
||||
Field::Metadata => read_field!(metadata, "metadata"),
|
||||
Field::CreatedAt => read_field!(created_at, "created_at"),
|
||||
Field::RotatedAt => read_field!(rotated_at, "rotated_at"),
|
||||
Field::CreatedBy => read_field!(created_by, "created_by"),
|
||||
Field::DeletionDate => read_field!(deletion_date, "deletion_date"),
|
||||
Field::EncryptedKeyMaterial => read_field!(encrypted_key_material, "encrypted_key_material"),
|
||||
Field::Nonce => read_field!(nonce, "nonce"),
|
||||
Field::AtRestProtection => read_field!(at_rest_protection, "at_rest_protection"),
|
||||
Field::Unknown(field) => {
|
||||
let _: IgnoredAny = map.next_value()?;
|
||||
unknown_fields.observe(field);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let key = StoredMasterKey {
|
||||
format_version: format_version.unwrap_or_else(default_stored_master_key_format_version),
|
||||
key_id: key_id.ok_or_else(|| de::Error::missing_field("key_id"))?,
|
||||
version: version.ok_or_else(|| de::Error::missing_field("version"))?,
|
||||
algorithm: algorithm.ok_or_else(|| de::Error::missing_field("algorithm"))?,
|
||||
usage: usage.ok_or_else(|| de::Error::missing_field("usage"))?,
|
||||
status: status.ok_or_else(|| de::Error::missing_field("status"))?,
|
||||
description: description.unwrap_or(None),
|
||||
metadata: metadata.ok_or_else(|| de::Error::missing_field("metadata"))?,
|
||||
created_at: created_at.ok_or_else(|| de::Error::missing_field("created_at"))?.0,
|
||||
rotated_at: rotated_at.map(|value: OptionalZonedValue| value.0).unwrap_or(None),
|
||||
created_by: created_by.unwrap_or(None),
|
||||
deletion_date: deletion_date.map(|value: OptionalZonedValue| value.0).unwrap_or(None),
|
||||
encrypted_key_material: encrypted_key_material
|
||||
.ok_or_else(|| de::Error::missing_field("encrypted_key_material"))?,
|
||||
nonce: nonce.ok_or_else(|| de::Error::missing_field("nonce"))?,
|
||||
at_rest_protection: at_rest_protection.unwrap_or_default(),
|
||||
};
|
||||
unknown_fields.record_for_local_key();
|
||||
Ok(key)
|
||||
}
|
||||
}
|
||||
|
||||
const FIELDS: &[&str] = &[
|
||||
"format_version",
|
||||
"key_id",
|
||||
"version",
|
||||
"algorithm",
|
||||
"usage",
|
||||
"status",
|
||||
"description",
|
||||
"metadata",
|
||||
"created_at",
|
||||
"rotated_at",
|
||||
"created_by",
|
||||
"deletion_date",
|
||||
"encrypted_key_material",
|
||||
"nonce",
|
||||
"at_rest_protection",
|
||||
];
|
||||
deserializer.deserialize_struct("StoredMasterKey", FIELDS, StoredMasterKeyVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
impl LocalKmsClient {
|
||||
/// Create a new local KMS client
|
||||
pub async fn new(config: LocalConfig) -> Result<Self> {
|
||||
@@ -1041,39 +804,6 @@ impl LocalKmsClient {
|
||||
path.display()
|
||||
))
|
||||
})?;
|
||||
let format_version = stored_master_key_format_version(&content).map_err(|error| {
|
||||
KmsError::configuration_error(format!(
|
||||
"Local KMS master key salt at {} is missing and key record {} is not interpretable by this build ({error}); \
|
||||
refusing to generate a replacement salt — restore the salt file from backup, or run a build that \
|
||||
understands the record",
|
||||
Self::master_key_salt_path(config).display(),
|
||||
path.display()
|
||||
))
|
||||
})?;
|
||||
if format_version > STORED_MASTER_KEY_FORMAT_VERSION {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"Local KMS master key salt at {} is missing and key record {} declares unsupported format version {format_version}; \
|
||||
refusing to generate a replacement salt",
|
||||
Self::master_key_salt_path(config).display(),
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
let has_unknown_marker = has_unknown_protection_marker(&content).map_err(|error| {
|
||||
KmsError::configuration_error(format!(
|
||||
"Local KMS master key salt at {} is missing and key record {} is not a readable JSON object ({error}); \
|
||||
refusing to generate a replacement salt",
|
||||
Self::master_key_salt_path(config).display(),
|
||||
path.display()
|
||||
))
|
||||
})?;
|
||||
if has_unknown_marker {
|
||||
return Err(KmsError::configuration_error(format!(
|
||||
"Local KMS master key salt at {} is missing and key record {} uses {UNKNOWN_STORED_KEY_PROTECTION}; \
|
||||
refusing to generate a replacement salt",
|
||||
Self::master_key_salt_path(config).display(),
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
let probe = serde_json::from_slice::<ProtectionProbe>(&content).map_err(|error| {
|
||||
KmsError::configuration_error(format!(
|
||||
"Local KMS master key salt at {} is missing and key record {} is not interpretable by this build ({error}); \
|
||||
@@ -1121,19 +851,13 @@ impl LocalKmsClient {
|
||||
|
||||
let content = fs::read(&key_path).await?;
|
||||
|
||||
let format_version = stored_master_key_format_version(&content)
|
||||
.map_err(|e| KmsError::material_corrupt(key_id, format!("stored key record is not a readable JSON object: {e}")))?;
|
||||
if format_version > STORED_MASTER_KEY_FORMAT_VERSION {
|
||||
return Err(KmsError::unsupported_format_version(key_id, format_version.to_string()));
|
||||
}
|
||||
|
||||
// Two-stage parse so an unrecognised protection marker is reported as an
|
||||
// unsupported format (a newer build may still read the key) instead of being
|
||||
// folded into generic corruption with every other malformed record.
|
||||
let has_unknown_marker = has_unknown_protection_marker(&content)
|
||||
let unknown_marker = unknown_protection_marker(&content)
|
||||
.map_err(|e| KmsError::material_corrupt(key_id, format!("stored key record is not a readable JSON object: {e}")))?;
|
||||
if has_unknown_marker {
|
||||
return Err(KmsError::unsupported_format_version(key_id, UNKNOWN_STORED_KEY_PROTECTION));
|
||||
if let Some(version) = unknown_marker {
|
||||
return Err(KmsError::unsupported_format_version(key_id, version));
|
||||
}
|
||||
let stored_key: StoredMasterKey = serde_json::from_slice(&content)
|
||||
.map_err(|e| KmsError::material_corrupt(key_id, format!("stored key record does not deserialize: {e}")))?;
|
||||
@@ -1299,7 +1023,6 @@ impl LocalKmsClient {
|
||||
};
|
||||
|
||||
let stored_key = StoredMasterKey {
|
||||
format_version: STORED_MASTER_KEY_FORMAT_VERSION,
|
||||
key_id: master_key.key_id.clone(),
|
||||
version: master_key.version,
|
||||
algorithm: master_key.algorithm.clone(),
|
||||
@@ -2180,8 +1903,6 @@ impl KmsBackend for LocalKmsBackend {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::test_support::{deserialize_with_ignored_only_unknown, unknown_field_metric};
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
use std::collections::HashMap;
|
||||
use tempfile::TempDir;
|
||||
|
||||
@@ -2394,8 +2115,8 @@ mod tests {
|
||||
),
|
||||
(
|
||||
"unknown protection marker",
|
||||
with_field("at_rest_protection", serde_json::json!("secret-marker-value-must-not-leak")),
|
||||
|e| matches!(e, KmsError::UnsupportedFormatVersion { version, .. } if version == UNKNOWN_STORED_KEY_PROTECTION),
|
||||
with_field("at_rest_protection", serde_json::json!("post-quantum-v2")),
|
||||
|e| matches!(e, KmsError::UnsupportedFormatVersion { version, .. } if version == "post-quantum-v2"),
|
||||
),
|
||||
];
|
||||
|
||||
@@ -2707,230 +2428,6 @@ mod tests {
|
||||
assert_eq!(key_info.created_at.time_zone().iana_name(), Some("UTC"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stored_master_key_format_version_is_explicit_and_legacy_defaults_to_v1() {
|
||||
let (client, _temp_dir) = create_dev_mode_client().await;
|
||||
client.create_key("format-key", "AES_256", None).await.expect("create key");
|
||||
|
||||
let key_path = client.master_key_path("format-key").expect("valid key id");
|
||||
let current_record = fs::read(&key_path).await.expect("read key record");
|
||||
let mut record: serde_json::Value = serde_json::from_slice(¤t_record).expect("decode key record");
|
||||
assert_eq!(record.get("format_version"), Some(&serde_json::json!(STORED_MASTER_KEY_FORMAT_VERSION)));
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct LegacyStoredMasterKeyProbe {
|
||||
key_id: String,
|
||||
version: u32,
|
||||
algorithm: String,
|
||||
usage: KeyUsage,
|
||||
status: KeyStatus,
|
||||
description: Option<String>,
|
||||
metadata: HashMap<String, String>,
|
||||
#[serde(with = "crate::time_serde::zoned")]
|
||||
created_at: Zoned,
|
||||
#[serde(with = "crate::time_serde::option_zoned")]
|
||||
rotated_at: Option<Zoned>,
|
||||
created_by: Option<String>,
|
||||
#[serde(default, with = "crate::time_serde::option_zoned")]
|
||||
deletion_date: Option<Zoned>,
|
||||
encrypted_key_material: String,
|
||||
nonce: Vec<u8>,
|
||||
#[serde(default)]
|
||||
at_rest_protection: StoredKeyProtection,
|
||||
}
|
||||
let legacy: LegacyStoredMasterKeyProbe =
|
||||
serde_json::from_slice(¤t_record).expect("the pre-format-version reader must accept a v1 record");
|
||||
let LegacyStoredMasterKeyProbe {
|
||||
key_id,
|
||||
version,
|
||||
algorithm,
|
||||
usage: _usage,
|
||||
status: _status,
|
||||
description: _description,
|
||||
metadata: _metadata,
|
||||
created_at: _created_at,
|
||||
rotated_at: _rotated_at,
|
||||
created_by: _created_by,
|
||||
deletion_date: _deletion_date,
|
||||
encrypted_key_material,
|
||||
nonce,
|
||||
at_rest_protection,
|
||||
} = legacy;
|
||||
assert_eq!(key_id, "format-key");
|
||||
assert_eq!(version, 1);
|
||||
assert_eq!(algorithm, "AES_256");
|
||||
assert!(!encrypted_key_material.is_empty());
|
||||
assert!(nonce.is_empty());
|
||||
assert_eq!(at_rest_protection, StoredKeyProtection::PlaintextDevOnly);
|
||||
|
||||
// A record from before the explicit field was added remains readable.
|
||||
record
|
||||
.as_object_mut()
|
||||
.expect("key record is an object")
|
||||
.remove("format_version")
|
||||
.expect("current records carry format_version");
|
||||
fs::write(&key_path, serde_json::to_vec_pretty(&record).expect("encode legacy key record"))
|
||||
.await
|
||||
.expect("write legacy key record");
|
||||
let info = client
|
||||
.describe_key("format-key", None)
|
||||
.await
|
||||
.expect("legacy key record should load");
|
||||
assert_eq!(info.key_id, "format-key");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stored_master_key_accepts_an_older_numeric_format_version() {
|
||||
let (client, _temp_dir) = create_dev_mode_client().await;
|
||||
client
|
||||
.create_key("older-format-key", "AES_256", None)
|
||||
.await
|
||||
.expect("create key");
|
||||
|
||||
let key_path = client.master_key_path("older-format-key").expect("valid key id");
|
||||
let mut record: serde_json::Value =
|
||||
serde_json::from_slice(&fs::read(&key_path).await.expect("read key record")).expect("decode key record");
|
||||
record["format_version"] = serde_json::json!(0);
|
||||
fs::write(&key_path, serde_json::to_vec_pretty(&record).expect("encode older key record"))
|
||||
.await
|
||||
.expect("write older key record");
|
||||
|
||||
let info = client
|
||||
.describe_key("older-format-key", None)
|
||||
.await
|
||||
.expect("older format version should remain readable");
|
||||
assert_eq!(info.key_id, "older-format-key");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stored_master_key_rejects_a_newer_format_version_before_decrypting() {
|
||||
let (client, _temp_dir) = create_dev_mode_client().await;
|
||||
client
|
||||
.create_key("future-format-key", "AES_256", None)
|
||||
.await
|
||||
.expect("create key");
|
||||
|
||||
let key_path = client.master_key_path("future-format-key").expect("valid key id");
|
||||
let mut record: serde_json::Value =
|
||||
serde_json::from_slice(&fs::read(&key_path).await.expect("read key record")).expect("decode key record");
|
||||
record["format_version"] = serde_json::json!(99);
|
||||
record.as_object_mut().expect("key record is an object").remove("usage");
|
||||
fs::write(&key_path, serde_json::to_vec_pretty(&record).expect("encode future key record"))
|
||||
.await
|
||||
.expect("write future key record");
|
||||
|
||||
let error = client
|
||||
.describe_key("future-format-key", None)
|
||||
.await
|
||||
.expect_err("a newer key format must fail closed");
|
||||
assert!(matches!(
|
||||
error,
|
||||
KmsError::UnsupportedFormatVersion { key_id, version }
|
||||
if key_id == "future-format-key" && version == "99"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_protection_marker_is_static_for_every_unknown_shape() {
|
||||
for marker in [
|
||||
serde_json::json!("secret-string-must-not-leak"),
|
||||
serde_json::json!({"future_mode": "secret-object-must-not-leak"}),
|
||||
serde_json::json!(["secret-array-must-not-leak"]),
|
||||
serde_json::json!(99),
|
||||
] {
|
||||
let record =
|
||||
serde_json::to_vec(&serde_json::json!({"at_rest_protection": marker})).expect("encode protection marker");
|
||||
assert!(has_unknown_protection_marker(&record).expect("probe protection marker"));
|
||||
}
|
||||
let long_marker = "secret-marker-must-not-be-copied".repeat(1024);
|
||||
let record =
|
||||
serde_json::to_vec(&serde_json::json!({"at_rest_protection": long_marker})).expect("encode long protection marker");
|
||||
assert!(has_unknown_protection_marker(&record).expect("probe long protection marker"));
|
||||
|
||||
for marker in [
|
||||
serde_json::Value::Null,
|
||||
serde_json::json!("legacy-unspecified"),
|
||||
serde_json::json!("encrypted-master-key"),
|
||||
serde_json::json!("plaintext-dev-only"),
|
||||
] {
|
||||
let record =
|
||||
serde_json::to_vec(&serde_json::json!({"at_rest_protection": marker})).expect("encode protection marker");
|
||||
assert!(!has_unknown_protection_marker(&record).expect("probe protection marker"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stored_master_key_unknown_fields_remain_readable() {
|
||||
const UNKNOWN_FIELD_VALUE: &str = "field value must not be logged";
|
||||
let (client, _temp_dir) = create_dev_mode_client().await;
|
||||
client
|
||||
.create_key("unknown-field-key", "AES_256", None)
|
||||
.await
|
||||
.expect("create key");
|
||||
|
||||
let key_path = client.master_key_path("unknown-field-key").expect("valid key id");
|
||||
let record: serde_json::Value =
|
||||
serde_json::from_slice(&fs::read(&key_path).await.expect("read key record")).expect("decode key record");
|
||||
let long_field = format!("{}界", "a".repeat(126));
|
||||
let long_prefix = "a".repeat(126);
|
||||
let injection_field = "b\n\u{1b}[31m";
|
||||
let record_with_unknown = |field: &str| {
|
||||
let mut record = record.clone();
|
||||
let object = record.as_object_mut().expect("key record is an object");
|
||||
object.insert(field.to_owned(), serde_json::json!(UNKNOWN_FIELD_VALUE));
|
||||
object.insert("zeta_extension".to_owned(), serde_json::json!("another value must not be logged"));
|
||||
serde_json::to_vec_pretty(&record).expect("encode key record with unknown fields")
|
||||
};
|
||||
let long_record = record_with_unknown(&long_field);
|
||||
let mut injection_record: serde_json::Value =
|
||||
serde_json::from_slice(&record_with_unknown(injection_field)).expect("decode injection record");
|
||||
injection_record["key_id"] = serde_json::json!("tenant-secret-or-untrusted-record-id");
|
||||
let injection_record = serde_json::to_vec(&injection_record).expect("encode injection record");
|
||||
let logs = crate::test_support::CapturedLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_ansi(false)
|
||||
.with_max_level(tracing::Level::WARN)
|
||||
.with_writer(logs.clone())
|
||||
.finish();
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let parse = |record: &[u8]| {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let stored = metrics::with_local_recorder(&recorder, || {
|
||||
tracing::dispatcher::with_default(&dispatch, || {
|
||||
serde_json::from_slice(record).expect("unknown fields must remain forward-compatible")
|
||||
})
|
||||
});
|
||||
assert_eq!(unknown_field_metric(&recorder, "local-key-record"), 2);
|
||||
stored
|
||||
};
|
||||
let stored: StoredMasterKey = parse(&long_record);
|
||||
let _: StoredMasterKey = parse(&long_record);
|
||||
let _: StoredMasterKey = parse(&injection_record);
|
||||
let _: StoredMasterKey = parse(&injection_record);
|
||||
assert_eq!(stored.key_id, "unknown-field-key");
|
||||
|
||||
let output = logs.output();
|
||||
assert!(output.contains("WARN"));
|
||||
assert_eq!(output.matches("Local KMS key record contains unknown fields").count(), 3);
|
||||
assert!(output.contains(&long_prefix));
|
||||
assert!(!output.contains(&long_field));
|
||||
assert!(output.contains("field_name_truncated=true"));
|
||||
assert!(output.contains(r#"\n\u{1b}[31m"#));
|
||||
assert!(!output.contains("zeta_extension"));
|
||||
assert!(output.contains("field_count=2"));
|
||||
for observed_records in [1, 2, 4] {
|
||||
assert!(output.contains(&format!("observed_records={observed_records}")));
|
||||
}
|
||||
assert!(!output.contains("observed_records=3"));
|
||||
assert!(!output.contains("tenant-secret-or-untrusted-record-id"));
|
||||
assert!(!output.contains(UNKNOWN_FIELD_VALUE));
|
||||
assert!(!output.contains("another value must not be logged"));
|
||||
|
||||
let streamed: StoredMasterKey = deserialize_with_ignored_only_unknown(record, "stream_only_extension")
|
||||
.expect("unknown values must be consumed through deserialize_ignored_any");
|
||||
assert_eq!(streamed.key_id, "unknown-field-key");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_load_master_key_accepts_legacy_encrypted_record_without_protection_field() {
|
||||
let (client, temp_dir) = create_test_client().await;
|
||||
@@ -3536,26 +3033,17 @@ mod tests {
|
||||
serde_json::from_slice(&fs::read(&key_path).await.expect("read key file")).expect("decode record");
|
||||
drop(client);
|
||||
|
||||
const UNKNOWN_MARKER_VALUE: &str = "secret-marker-value-must-not-leak";
|
||||
let mut newer_build_record = pristine.clone();
|
||||
newer_build_record["at_rest_protection"] = serde_json::json!(UNKNOWN_MARKER_VALUE);
|
||||
newer_build_record["at_rest_protection"] = serde_json::json!("post-quantum-v2");
|
||||
let newer_build_record = serde_json::to_vec_pretty(&newer_build_record).expect("encode record");
|
||||
let mut future_format_record = pristine.clone();
|
||||
future_format_record["format_version"] = serde_json::json!(99);
|
||||
let future_format_record = serde_json::to_vec_pretty(&future_format_record).expect("encode record");
|
||||
let truncated = {
|
||||
let bytes = serde_json::to_vec_pretty(&pristine).expect("encode record");
|
||||
bytes[..bytes.len() / 2].to_vec()
|
||||
};
|
||||
|
||||
for (name, content, expected_error) in [
|
||||
("record from a newer build", newer_build_record, Some(UNKNOWN_STORED_KEY_PROTECTION)),
|
||||
(
|
||||
"record with a future format version",
|
||||
future_format_record,
|
||||
Some("unsupported format version 99"),
|
||||
),
|
||||
("record that does not decode", truncated, None),
|
||||
for (name, content) in [
|
||||
("record from a newer build", newer_build_record),
|
||||
("record that does not decode", truncated),
|
||||
] {
|
||||
fs::write(&key_path, &content).await.expect("write record");
|
||||
fs::remove_file(&salt_path).await.ok();
|
||||
@@ -3572,16 +3060,6 @@ mod tests {
|
||||
"{name}: expected a salt-specific configuration error, got {error:?}"
|
||||
);
|
||||
assert!(error.to_string().contains("salt"), "{name}: error must point at the salt: {error}");
|
||||
assert!(
|
||||
!error.to_string().contains(UNKNOWN_MARKER_VALUE),
|
||||
"{name}: raw marker values must stay redacted"
|
||||
);
|
||||
if let Some(expected_error) = expected_error {
|
||||
assert!(
|
||||
error.to_string().contains(expected_error),
|
||||
"{name}: error must explain the incompatibility: {error}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
sorted_dir_file_names(temp_dir.path()).await,
|
||||
vec!["sealed-key.key".to_string()],
|
||||
@@ -3607,9 +3085,7 @@ mod tests {
|
||||
let key_path = client.master_key_path("beta").expect("valid key id");
|
||||
let mut record: serde_json::Value =
|
||||
serde_json::from_slice(&fs::read(&key_path).await.expect("read record")).expect("decode record");
|
||||
record["at_rest_protection"] = serde_json::json!({
|
||||
"future_mode": ["secret-marker-value-must-not-leak"]
|
||||
});
|
||||
record["at_rest_protection"] = serde_json::json!("post-quantum-v2");
|
||||
fs::write(&key_path, serde_json::to_vec_pretty(&record).expect("encode record"))
|
||||
.await
|
||||
.expect("write record");
|
||||
@@ -3620,10 +3096,29 @@ mod tests {
|
||||
.expect_err("a listing must not quietly omit a key it cannot read");
|
||||
assert!(
|
||||
matches!(&error, KmsError::UnsupportedFormatVersion { key_id, version }
|
||||
if key_id == "beta" && version == UNKNOWN_STORED_KEY_PROTECTION),
|
||||
if key_id == "beta" && version == "post-quantum-v2"),
|
||||
"got {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_keys_fails_closed_on_a_corrupt_record() {
|
||||
let (client, _temp_dir) = create_test_client().await;
|
||||
client.create_key("corrupt", "AES_256", None).await.expect("create key");
|
||||
|
||||
let key_path = client.master_key_path("corrupt").expect("valid key id");
|
||||
fs::write(&key_path, b"{\"key_id\":\"corrupt\"")
|
||||
.await
|
||||
.expect("write corrupt record");
|
||||
|
||||
let error = client
|
||||
.list_keys(&ListKeysRequest::default(), None)
|
||||
.await
|
||||
.expect_err("a corrupt record must not disappear from a listing");
|
||||
assert!(
|
||||
matches!(error, KmsError::MaterialCorrupt { key_id, .. } if key_id == "corrupt"),
|
||||
"got {error:?}"
|
||||
);
|
||||
assert!(!error.to_string().contains("secret-marker-value-must-not-leak"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -160,7 +160,7 @@ pub(crate) fn ensure_rewrap_context_matches(
|
||||
}
|
||||
|
||||
/// Page size used when a [`ListKeysRequest`] does not ask for one.
|
||||
pub(crate) const DEFAULT_LIST_KEYS_PAGE_SIZE: u32 = 100;
|
||||
pub(crate) const DEFAULT_LIST_KEYS_PAGE_SIZE: u32 = DEFAULT_LIST_KEYS_LIMIT;
|
||||
|
||||
/// One page of a key set the backend has to slice itself.
|
||||
pub(crate) struct KeyPage<'a, T> {
|
||||
@@ -231,7 +231,7 @@ pub(crate) fn paginate_keys<'a, T>(sorted: &'a [T], request: &ListKeysRequest, k
|
||||
pub(crate) fn list_keys_page_size(limit: Option<u32>) -> Option<usize> {
|
||||
match limit.unwrap_or(DEFAULT_LIST_KEYS_PAGE_SIZE) {
|
||||
0 => None,
|
||||
size => Some(size as usize),
|
||||
size => Some(size.min(MAX_LIST_KEYS_LIMIT) as usize),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -860,6 +860,21 @@ mod tests {
|
||||
assert_eq!(page_of(&keys, Some(u32::MAX), Some("key-01")), (vec![keys[2].clone()], None, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_limit_is_bounded_at_the_contract_maximum() {
|
||||
assert_eq!(list_keys_page_size(Some(MAX_LIST_KEYS_LIMIT)), Some(MAX_LIST_KEYS_LIMIT as usize));
|
||||
assert_eq!(
|
||||
list_keys_page_size(Some(MAX_LIST_KEYS_LIMIT + 1)),
|
||||
Some(MAX_LIST_KEYS_LIMIT as usize),
|
||||
"one past the maximum must not expand a backend page"
|
||||
);
|
||||
assert_eq!(
|
||||
list_keys_page_size(Some(u32::MAX)),
|
||||
Some(MAX_LIST_KEYS_LIMIT as usize),
|
||||
"the largest wire value must use the same bounded page"
|
||||
);
|
||||
}
|
||||
|
||||
/// The cursor is an identifier, so a marker naming a key that no longer
|
||||
/// exists resumes after where it would have been instead of restarting.
|
||||
#[test]
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
//! assertions. It intentionally implements just enough HTTP/1.1 for the
|
||||
//! `vaultrs` reqwest client: no keep-alive, no chunked bodies.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
@@ -68,7 +67,6 @@ pub(crate) struct ScriptedVault {
|
||||
/// Base address (`http://127.0.0.1:port`) to point a Vault client at.
|
||||
pub(crate) address: String,
|
||||
requests: Arc<Mutex<Vec<(String, String)>>>,
|
||||
kv2_state: Option<Arc<Mutex<Kv2State>>>,
|
||||
}
|
||||
|
||||
impl ScriptedVault {
|
||||
@@ -85,16 +83,13 @@ impl ScriptedVault {
|
||||
tokio::spawn(async move {
|
||||
let mut responses = responses.into_iter();
|
||||
loop {
|
||||
let Ok((stream, _)) = listener.accept().await else {
|
||||
let Ok((mut stream, _)) = listener.accept().await else {
|
||||
return;
|
||||
};
|
||||
let Some((request_line, body, mut stream)) = read_request(stream).await else {
|
||||
let Some(request) = read_request(&mut stream).await else {
|
||||
continue;
|
||||
};
|
||||
recorded
|
||||
.lock()
|
||||
.expect("scripted vault request log poisoned")
|
||||
.push((request_line, body));
|
||||
recorded.lock().expect("scripted vault request log poisoned").push(request);
|
||||
let response = responses
|
||||
.next()
|
||||
.unwrap_or_else(|| ScriptedResponse::error(599, "scripted vault: script exhausted"));
|
||||
@@ -109,59 +104,7 @@ impl ScriptedVault {
|
||||
}
|
||||
});
|
||||
|
||||
Self {
|
||||
address,
|
||||
requests,
|
||||
kv2_state: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Bind a small stateful KV2 responder for concurrency tests.
|
||||
///
|
||||
/// Unlike Self::serve, this responder evaluates CAS writes against a
|
||||
/// shared in-memory record and handles connections concurrently. It models
|
||||
/// only the KV2 data and metadata paths used by the rotation protocol; an
|
||||
/// unknown request receives a 599 response so a test cannot silently
|
||||
/// under-specify the Vault exchange.
|
||||
pub(crate) async fn serve_kv2(key_path: &str, key_data: serde_json::Value) -> Self {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind scripted vault listener");
|
||||
let address = format!("http://{}", listener.local_addr().expect("scripted vault local addr"));
|
||||
let requests = Arc::new(Mutex::new(Vec::new()));
|
||||
let state = Arc::new(Mutex::new(Kv2State::new(key_data)));
|
||||
let recorded = Arc::clone(&requests);
|
||||
let state_for_server = Arc::clone(&state);
|
||||
let key_path = key_path.to_string();
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let Ok((stream, _)) = listener.accept().await else {
|
||||
return;
|
||||
};
|
||||
let recorded = Arc::clone(&recorded);
|
||||
let state = Arc::clone(&state_for_server);
|
||||
let key_path = key_path.clone();
|
||||
tokio::spawn(async move {
|
||||
let Some((request_line, body, stream)) = read_request(stream).await else {
|
||||
return;
|
||||
};
|
||||
recorded
|
||||
.lock()
|
||||
.expect("scripted vault request log poisoned")
|
||||
.push((request_line.clone(), body.clone()));
|
||||
let response = state
|
||||
.lock()
|
||||
.expect("scripted KV2 state poisoned")
|
||||
.respond(&key_path, &request_line, &body);
|
||||
write_response(stream, response).await;
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
Self {
|
||||
address,
|
||||
requests,
|
||||
kv2_state: Some(state),
|
||||
}
|
||||
Self { address, requests }
|
||||
}
|
||||
|
||||
/// The `METHOD /path` lines of every request served so far, in order.
|
||||
@@ -185,195 +128,13 @@ impl ScriptedVault {
|
||||
.map(|(_, body)| body.clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Snapshot the in-memory KV2 state used by Self::serve_kv2.
|
||||
pub(crate) fn kv2_snapshot(&self) -> Option<Kv2Snapshot> {
|
||||
self.kv2_state.as_ref().map(|state| {
|
||||
let state = state.lock().expect("scripted KV2 state poisoned");
|
||||
Kv2Snapshot {
|
||||
current_data: state.current_data.clone(),
|
||||
current_secret_version: state.current_secret_version,
|
||||
version_records: state.version_records.clone(),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// State captured by the stateful KV2 responder for assertions in wiring tests.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct Kv2Snapshot {
|
||||
pub(crate) current_data: serde_json::Value,
|
||||
pub(crate) current_secret_version: u64,
|
||||
pub(crate) version_records: BTreeMap<u32, serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Kv2State {
|
||||
current_data: serde_json::Value,
|
||||
current_secret_version: u64,
|
||||
history: BTreeMap<u64, serde_json::Value>,
|
||||
version_records: BTreeMap<u32, serde_json::Value>,
|
||||
}
|
||||
|
||||
impl Kv2State {
|
||||
fn new(current_data: serde_json::Value) -> Self {
|
||||
let mut version_records = BTreeMap::new();
|
||||
if current_data["baseline_version"].as_u64() == Some(1) {
|
||||
version_records.insert(1, current_data.clone());
|
||||
}
|
||||
Self {
|
||||
history: BTreeMap::from([(1, current_data.clone())]),
|
||||
current_data,
|
||||
current_secret_version: 1,
|
||||
version_records,
|
||||
}
|
||||
}
|
||||
|
||||
fn respond(&mut self, key_path: &str, request_line: &str, body: &str) -> ScriptedResponse {
|
||||
let Some((method, path)) = request_line.split_once(' ') else {
|
||||
return ScriptedResponse::error(599, "scripted KV2: malformed request line");
|
||||
};
|
||||
let data_path = format!("/v1/secret/data/{key_path}");
|
||||
let metadata_path = format!("/v1/secret/metadata/{key_path}");
|
||||
let version_data_prefix = format!("{data_path}/versions/");
|
||||
let version_metadata_path = format!("{metadata_path}/versions");
|
||||
|
||||
if method == "GET" && path == metadata_path {
|
||||
return ScriptedResponse::ok(metadata_data(self.current_secret_version));
|
||||
}
|
||||
|
||||
if method == "LIST" && path == version_metadata_path {
|
||||
let keys = self.version_records.keys().map(u32::to_string).collect::<Vec<_>>();
|
||||
return ScriptedResponse::ok(serde_json::json!({ "keys": keys }));
|
||||
}
|
||||
|
||||
if let Some(version) = path
|
||||
.strip_prefix(&version_data_prefix)
|
||||
.and_then(|value| value.parse::<u32>().ok())
|
||||
{
|
||||
return match method {
|
||||
"GET" => self
|
||||
.version_records
|
||||
.get(&version)
|
||||
.cloned()
|
||||
.map(read_data)
|
||||
.unwrap_or_else(|| ScriptedResponse::error(404, "not found")),
|
||||
"POST" => self.create_version_record(version, body),
|
||||
_ => ScriptedResponse::error(599, "scripted KV2: unsupported version request"),
|
||||
};
|
||||
}
|
||||
|
||||
if method == "GET" && path.strip_prefix(&data_path).is_some() {
|
||||
let version = path
|
||||
.split_once("?version=")
|
||||
.and_then(|(_, value)| value.parse::<u64>().ok())
|
||||
.unwrap_or(self.current_secret_version);
|
||||
return self
|
||||
.history
|
||||
.get(&version)
|
||||
.cloned()
|
||||
.map(read_data)
|
||||
.unwrap_or_else(|| ScriptedResponse::error(404, "not found"));
|
||||
}
|
||||
|
||||
if method == "POST" && path == data_path {
|
||||
return self.write_current_record(body);
|
||||
}
|
||||
|
||||
ScriptedResponse::error(599, "scripted KV2: unexpected request")
|
||||
}
|
||||
|
||||
fn create_version_record(&mut self, version: u32, body: &str) -> ScriptedResponse {
|
||||
let body: serde_json::Value = match serde_json::from_str(body) {
|
||||
Ok(body) => body,
|
||||
Err(_) => return ScriptedResponse::error(400, "invalid JSON"),
|
||||
};
|
||||
if body["options"]["cas"].as_u64() != Some(0) {
|
||||
return ScriptedResponse::error(400, "version record requires create-only CAS");
|
||||
}
|
||||
if self.version_records.contains_key(&version) {
|
||||
return ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE);
|
||||
}
|
||||
let Some(data) = body.get("data").cloned() else {
|
||||
return ScriptedResponse::error(400, "missing data");
|
||||
};
|
||||
self.version_records.insert(version, data);
|
||||
write_ack(1)
|
||||
}
|
||||
|
||||
fn write_current_record(&mut self, body: &str) -> ScriptedResponse {
|
||||
let body: serde_json::Value = match serde_json::from_str(body) {
|
||||
Ok(body) => body,
|
||||
Err(_) => return ScriptedResponse::error(400, "invalid JSON"),
|
||||
};
|
||||
if body["options"]["cas"].as_u64() != Some(self.current_secret_version) {
|
||||
return ScriptedResponse::error(400, CAS_CONFLICT_MESSAGE);
|
||||
}
|
||||
let Some(data) = body.get("data").cloned() else {
|
||||
return ScriptedResponse::error(400, "missing data");
|
||||
};
|
||||
self.current_secret_version += 1;
|
||||
self.current_data = data.clone();
|
||||
self.history.insert(self.current_secret_version, data);
|
||||
write_ack(self.current_secret_version)
|
||||
}
|
||||
}
|
||||
|
||||
const CAS_CONFLICT_MESSAGE: &str = "check-and-set parameter did not match the current version";
|
||||
|
||||
fn metadata_data(current_version: u64) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"cas_required": false,
|
||||
"created_time": "2026-01-01T00:00:00Z",
|
||||
"current_version": current_version,
|
||||
"delete_version_after": "0s",
|
||||
"max_versions": 0,
|
||||
"oldest_version": 0,
|
||||
"updated_time": "2026-01-01T00:00:00Z",
|
||||
"custom_metadata": null,
|
||||
"versions": {},
|
||||
})
|
||||
}
|
||||
|
||||
fn read_data(data: serde_json::Value) -> ScriptedResponse {
|
||||
ScriptedResponse::ok(serde_json::json!({
|
||||
"data": data,
|
||||
"metadata": {
|
||||
"created_time": "2026-01-01T00:00:00Z",
|
||||
"deletion_time": "",
|
||||
"custom_metadata": null,
|
||||
"destroyed": false,
|
||||
"version": 1,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
fn write_ack(version: u64) -> ScriptedResponse {
|
||||
ScriptedResponse::ok(serde_json::json!({
|
||||
"created_time": "2026-01-01T00:00:00Z",
|
||||
"custom_metadata": null,
|
||||
"deletion_time": "",
|
||||
"destroyed": false,
|
||||
"version": version,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn write_response(mut stream: TcpStream, response: ScriptedResponse) {
|
||||
if let ScriptedResponse::Http { status, body } = response {
|
||||
let payload = format!(
|
||||
"HTTP/1.1 {status} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
|
||||
body.len(),
|
||||
);
|
||||
let _ = stream.write_all(payload.as_bytes()).await;
|
||||
let _ = stream.shutdown().await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Read one HTTP/1.1 request (head plus content-length body) and return its
|
||||
/// `METHOD /path` line together with the body. Draining the body before
|
||||
/// responding keeps the client from seeing a connection reset while it is
|
||||
/// still writing.
|
||||
async fn read_request(mut stream: TcpStream) -> Option<(String, String, TcpStream)> {
|
||||
async fn read_request(stream: &mut TcpStream) -> Option<(String, String)> {
|
||||
let mut buffer = Vec::new();
|
||||
let mut chunk = [0u8; 4096];
|
||||
let head_end = loop {
|
||||
@@ -417,5 +178,5 @@ async fn read_request(mut stream: TcpStream) -> Option<(String, String, TcpStrea
|
||||
}
|
||||
body.truncate(content_length);
|
||||
|
||||
Some((format!("{method} {path}"), String::from_utf8_lossy(&body).into_owned(), stream))
|
||||
Some((format!("{method} {path}"), String::from_utf8_lossy(&body).into_owned()))
|
||||
}
|
||||
|
||||
@@ -59,6 +59,12 @@ pub struct StaticKmsBackend {
|
||||
key_id: String,
|
||||
/// The raw 32-byte AES-256 key material (zeroed on drop).
|
||||
key: Zeroizing<[u8; KEY_SIZE]>,
|
||||
/// The time this configured key became visible to the backend.
|
||||
///
|
||||
/// Static KMS has no persisted key record, so this startup timestamp is
|
||||
/// the only stable creation date it can report across list and describe
|
||||
/// calls. It must not be regenerated for every response.
|
||||
created_at: Zoned,
|
||||
}
|
||||
|
||||
impl StaticKmsBackend {
|
||||
@@ -82,6 +88,7 @@ impl StaticKmsBackend {
|
||||
Ok(Self {
|
||||
key_id: static_config.key_id.clone(),
|
||||
key: key.into(),
|
||||
created_at: Zoned::now(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -92,7 +99,7 @@ impl StaticKmsBackend {
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: Some("Static single-key KMS backend".to_string()),
|
||||
creation_date: Zoned::now(),
|
||||
creation_date: self.created_at.clone(),
|
||||
deletion_date: None,
|
||||
origin: "EXTERNAL".to_string(),
|
||||
key_manager: "STATIC".to_string(),
|
||||
@@ -281,7 +288,7 @@ impl StaticKmsBackend {
|
||||
version: 1,
|
||||
metadata: HashMap::new(),
|
||||
tags: HashMap::new(),
|
||||
created_at: Zoned::now(),
|
||||
created_at: self.created_at.clone(),
|
||||
rotated_at: None,
|
||||
created_by: None,
|
||||
};
|
||||
@@ -677,6 +684,23 @@ mod tests {
|
||||
assert!(!response.truncated);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_key_creation_date_is_stable_for_the_backend_lifetime() {
|
||||
let (backend, key_id, _key) = create_test_backend().await;
|
||||
|
||||
let first = backend
|
||||
.list_configured_key(&ListKeysRequest::default())
|
||||
.expect("first list should succeed");
|
||||
tokio::task::yield_now().await;
|
||||
let second = backend
|
||||
.list_configured_key(&ListKeysRequest::default())
|
||||
.expect("second list should succeed");
|
||||
let described = backend.configured_key_info(&key_id).expect("describe should succeed");
|
||||
|
||||
assert_eq!(first.keys[0].created_at, second.keys[0].created_at);
|
||||
assert_eq!(first.keys[0].created_at, described.created_at);
|
||||
}
|
||||
|
||||
/// A zero limit means zero keys, even for a backend whose whole key set is
|
||||
/// a single configured key.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -227,13 +227,7 @@ impl VaultKmsClient {
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
};
|
||||
let source = token_source_for(&config.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
kms_config,
|
||||
&config.auth_method,
|
||||
"vault-kv2",
|
||||
&config.address,
|
||||
config.namespace.as_deref(),
|
||||
);
|
||||
let policy = VaultCredentialPolicy::from_kms_config(kms_config, &config.auth_method);
|
||||
let credentials = Arc::new(VaultCredentialProvider::new(settings, source, policy).await?);
|
||||
|
||||
info!(address = %config.address, "Vault KMS backend connected");
|
||||
@@ -243,7 +237,7 @@ impl VaultKmsClient {
|
||||
kv_mount: config.kv_mount.clone(),
|
||||
key_path_prefix: config.key_path_prefix.clone(),
|
||||
dek_crypto: AesDekCrypto::new(),
|
||||
retry: RetryPolicy::for_backend(kms_config, "vault-kv2", &config.address, config.namespace.as_deref(), "operations"),
|
||||
retry: RetryPolicy::from_config(kms_config),
|
||||
cancel: CancellationToken::new(),
|
||||
})
|
||||
}
|
||||
@@ -1219,8 +1213,13 @@ impl VaultKmsClient {
|
||||
// A key that disappeared between the listing and the read is
|
||||
// dropped from the page rather than failing it; the cursor comes
|
||||
// from the identifier list, so the listing still advances past it.
|
||||
let Ok(key_info) = self.describe_key(key_id, None).await else {
|
||||
continue;
|
||||
// Any other read error means the record is present but this build
|
||||
// cannot interpret it. Hiding that entry would make the listing
|
||||
// claim a complete key set while silently omitting corrupt data.
|
||||
let key_info = match self.describe_key(key_id, None).await {
|
||||
Ok(key_info) => key_info,
|
||||
Err(KmsError::KeyNotFound { .. }) => continue,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if request
|
||||
.status_filter
|
||||
@@ -1901,19 +1900,6 @@ mod tests {
|
||||
(vault, client)
|
||||
}
|
||||
|
||||
async fn scripted_kv2_client(key_data: &VaultKeyData) -> (ScriptedVault, VaultKmsClient) {
|
||||
let vault = ScriptedVault::serve_kv2(
|
||||
"rustfs/kms/keys/wired-key",
|
||||
serde_json::to_value(key_data).expect("serialize scripted KV2 key"),
|
||||
)
|
||||
.await;
|
||||
let (vault_config, kms_config) = scripted_configs(&vault.address);
|
||||
let client = VaultKmsClient::new(vault_config, &kms_config)
|
||||
.await
|
||||
.expect("scripted Vault client");
|
||||
(vault, client)
|
||||
}
|
||||
|
||||
fn healthy_key_data() -> VaultKeyData {
|
||||
VaultKeyData {
|
||||
algorithm: "AES_256".to_string(),
|
||||
@@ -1974,6 +1960,32 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_keys_fails_closed_on_an_unreadable_kv2_record() {
|
||||
let malformed_record = serde_json::json!({
|
||||
"data": {"algorithm": "AES_256"},
|
||||
"metadata": {
|
||||
"created_time": "2026-01-01T00:00:00Z",
|
||||
"deletion_time": "",
|
||||
"custom_metadata": null,
|
||||
"destroyed": false,
|
||||
"version": 1
|
||||
}
|
||||
});
|
||||
let (vault, client) = scripted_client(vec![
|
||||
ScriptedResponse::ok(serde_json::json!({ "keys": ["poisoned"] })),
|
||||
ScriptedResponse::ok(malformed_record),
|
||||
])
|
||||
.await;
|
||||
|
||||
let error = client
|
||||
.list_keys(&ListKeysRequest::default(), None)
|
||||
.await
|
||||
.expect_err("a malformed KV2 record must not disappear from a listing");
|
||||
assert!(matches!(error, KmsError::BackendError { .. }), "got {error:?}");
|
||||
assert_eq!(vault.requests().len(), 2, "listing must read the listed record before failing");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wired_read_retries_transient_status_then_succeeds() {
|
||||
let (vault, client) = scripted_client(vec![
|
||||
@@ -3242,102 +3254,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Concurrent rotations use KV2 create-only records and a CAS pointer
|
||||
/// switch. The stateful scripted Vault applies those preconditions to real
|
||||
/// HTTP requests, so this test proves committed versions are unique and
|
||||
/// contiguous instead of only checking that several calls returned.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn wired_concurrent_kv2_rotations_commit_unique_monotonic_versions() {
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
const ATTEMPTS: usize = 8;
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.baseline_version = Some(1);
|
||||
let (vault, client) = scripted_kv2_client(&key_data).await;
|
||||
let client = Arc::new(client);
|
||||
let barrier = Arc::new(tokio::sync::Barrier::new(ATTEMPTS));
|
||||
let tasks: Vec<_> = (0..ATTEMPTS)
|
||||
.map(|_| {
|
||||
let client = Arc::clone(&client);
|
||||
let barrier = Arc::clone(&barrier);
|
||||
tokio::spawn(async move {
|
||||
barrier.wait().await;
|
||||
client.rotate_key("wired-key", None).await
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut committed_versions = Vec::new();
|
||||
let mut errors = Vec::new();
|
||||
for task in tasks {
|
||||
match task.await.expect("join concurrent rotation task") {
|
||||
Ok(result) => committed_versions.push(result.version),
|
||||
Err(error) => errors.push(error),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
!committed_versions.is_empty(),
|
||||
"at least one concurrent rotation must commit; errors: {errors:?}"
|
||||
);
|
||||
assert!(
|
||||
errors.iter().all(
|
||||
|error| matches!(error, KmsError::InvalidOperation { message } if message.contains("Concurrent modification"))
|
||||
),
|
||||
"a lost CAS race must be the only expected failure: {errors:?}"
|
||||
);
|
||||
|
||||
let mut sorted_versions = committed_versions.clone();
|
||||
sorted_versions.sort_unstable();
|
||||
let unique_versions: HashSet<_> = sorted_versions.iter().copied().collect();
|
||||
assert_eq!(
|
||||
unique_versions.len(),
|
||||
sorted_versions.len(),
|
||||
"concurrent rotations must never return a version twice: {committed_versions:?}"
|
||||
);
|
||||
|
||||
let successful_rotations = u32::try_from(sorted_versions.len()).expect("test attempts fit u32");
|
||||
let current_version = 1u32 + successful_rotations;
|
||||
assert_eq!(
|
||||
sorted_versions,
|
||||
(2..=current_version).collect::<Vec<_>>(),
|
||||
"committed versions must form one monotonic sequence: {sorted_versions:?}"
|
||||
);
|
||||
|
||||
let snapshot = vault.kv2_snapshot().expect("stateful KV2 snapshot");
|
||||
let persisted_version = snapshot.current_data["version"]
|
||||
.as_u64()
|
||||
.and_then(|version| u32::try_from(version).ok())
|
||||
.expect("current KV2 record must carry a u32 key version");
|
||||
assert_eq!(persisted_version, current_version);
|
||||
assert!(
|
||||
snapshot.current_secret_version >= u64::from(current_version),
|
||||
"the KV2 secret version must advance with each committed pointer switch"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot.version_records.keys().copied().collect::<Vec<_>>(),
|
||||
(1..=current_version).collect::<Vec<_>>(),
|
||||
"every committed KMS version must have exactly one immutable record"
|
||||
);
|
||||
|
||||
let mut materials = HashSet::new();
|
||||
for (version, record) in &snapshot.version_records {
|
||||
let material = record["encrypted_key_material"]
|
||||
.as_str()
|
||||
.expect("version record must carry encrypted key material");
|
||||
assert!(materials.insert(material), "version {version} reuses another version's material");
|
||||
}
|
||||
assert_eq!(
|
||||
snapshot.current_data["encrypted_key_material"],
|
||||
snapshot
|
||||
.version_records
|
||||
.get(¤t_version)
|
||||
.expect("current version record")["encrypted_key_material"],
|
||||
"the top-level fast path must match the current immutable version record"
|
||||
);
|
||||
}
|
||||
|
||||
/// The tags write-back after a create is a check-and-set read-modify-write
|
||||
/// that carries the key material over from the freshly read record.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -520,10 +520,8 @@ impl VaultConnectionSettings {
|
||||
/// Refresh and fail-closed tuning for a [`VaultCredentialProvider`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct VaultCredentialPolicy {
|
||||
/// Retry budget for one login cycle.
|
||||
/// Retry budget for one login/renewal cycle.
|
||||
pub(crate) retry: RetryPolicy,
|
||||
/// Retry budget for one token-renewal cycle.
|
||||
pub(crate) renew_retry: RetryPolicy,
|
||||
/// Fail-closed margin: once the current token is within this window of
|
||||
/// expiry without a successful refresh, [`VaultCredentialProvider::current`]
|
||||
/// refuses to hand it out.
|
||||
@@ -538,14 +536,8 @@ impl VaultCredentialPolicy {
|
||||
/// The default safety window equals the per-attempt timeout: a request
|
||||
/// issued now can stay in flight for up to one attempt timeout, so the
|
||||
/// token must outlive at least that.
|
||||
pub(crate) fn from_kms_config(
|
||||
config: &KmsConfig,
|
||||
auth_method: &VaultAuthMethod,
|
||||
backend: &'static str,
|
||||
endpoint: &str,
|
||||
namespace: Option<&str>,
|
||||
) -> Self {
|
||||
let retry = RetryPolicy::for_credentials(config, backend, endpoint, namespace, "credentials-login");
|
||||
pub(crate) fn from_kms_config(config: &KmsConfig, auth_method: &VaultAuthMethod) -> Self {
|
||||
let retry = RetryPolicy::from_config(config);
|
||||
let safety_window = match auth_method {
|
||||
VaultAuthMethod::AppRole {
|
||||
refresh_safety_window_secs: Some(secs),
|
||||
@@ -559,7 +551,6 @@ impl VaultCredentialPolicy {
|
||||
};
|
||||
Self {
|
||||
retry,
|
||||
renew_retry: RetryPolicy::for_credentials(config, backend, endpoint, namespace, "credentials-renew"),
|
||||
safety_window,
|
||||
retry_interval: DEFAULT_REFRESH_RETRY_INTERVAL,
|
||||
}
|
||||
@@ -726,7 +717,7 @@ impl VaultCredentialProvider {
|
||||
|
||||
let renewable = current.lease.map(|lease| lease.renewable).unwrap_or(false);
|
||||
let renewed = if renewable {
|
||||
match policy::execute("vault_token_renew", OpClass::Auth, &self.policy.renew_retry, cancel, || {
|
||||
match policy::execute("vault_token_renew", OpClass::Auth, &self.policy.retry, cancel, || {
|
||||
self.source.renew(¤t.client)
|
||||
})
|
||||
.await
|
||||
@@ -877,56 +868,19 @@ mod tests {
|
||||
/// Tight retry budget so paused-clock tests stay deterministic: one
|
||||
/// attempt per cycle, failed cycles spaced by `retry_interval`.
|
||||
fn test_policy(safety_window: Duration, retry_interval: Duration) -> VaultCredentialPolicy {
|
||||
let retry = RetryPolicy::for_test(
|
||||
Duration::from_secs(1),
|
||||
Duration::from_secs(1),
|
||||
1,
|
||||
Duration::from_millis(10),
|
||||
Duration::from_millis(10),
|
||||
);
|
||||
let renew_retry = RetryPolicy::for_test(
|
||||
Duration::from_secs(1),
|
||||
Duration::from_secs(1),
|
||||
1,
|
||||
Duration::from_millis(10),
|
||||
Duration::from_millis(10),
|
||||
);
|
||||
VaultCredentialPolicy {
|
||||
retry,
|
||||
renew_retry,
|
||||
retry: RetryPolicy {
|
||||
attempt_timeout: Duration::from_secs(1),
|
||||
op_deadline: Duration::from_secs(1),
|
||||
max_attempts: 1,
|
||||
base_backoff: Duration::from_millis(10),
|
||||
max_backoff: Duration::from_millis(10),
|
||||
},
|
||||
safety_window,
|
||||
retry_interval,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configured_login_and_renewal_use_reserved_credential_capacity() {
|
||||
let config = KmsConfig::default();
|
||||
let auth_method = VaultAuthMethod::Token {
|
||||
token: TEST_TOKEN.to_string(),
|
||||
};
|
||||
let credentials = VaultCredentialPolicy::from_kms_config(
|
||||
&config,
|
||||
&auth_method,
|
||||
"vault-kv2",
|
||||
"https://credential-policy.example.invalid",
|
||||
Some("team-namespace"),
|
||||
);
|
||||
let operations = RetryPolicy::for_backend(
|
||||
&config,
|
||||
"vault-kv2",
|
||||
"https://credential-policy.example.invalid",
|
||||
Some("team-namespace"),
|
||||
"operations",
|
||||
);
|
||||
|
||||
assert!(credentials.retry.uses_credential_reserve());
|
||||
assert!(credentials.renew_retry.uses_credential_reserve());
|
||||
assert!(!operations.uses_credential_reserve());
|
||||
assert!(credentials.retry.shares_active_capacity_with(&credentials.renew_retry));
|
||||
assert!(credentials.retry.shares_active_capacity_with(&operations));
|
||||
}
|
||||
|
||||
/// Shared observable state of a [`ScriptedSource`].
|
||||
#[derive(Debug, Default)]
|
||||
struct ScriptedState {
|
||||
@@ -1130,12 +1084,11 @@ mod tests {
|
||||
"expected CredentialsUnavailable, got {error:?}"
|
||||
);
|
||||
|
||||
// Recovery: after the failed-refresh circuit's cool-down, its single
|
||||
// half-open probe succeeds, installs a fresh generation, and the
|
||||
// provider serves requests again.
|
||||
// Recovery: the next retry cycle succeeds, installs a fresh
|
||||
// generation, and the provider serves requests again.
|
||||
state.fail_renew.store(false, Ordering::SeqCst);
|
||||
state.fail_login.store(false, Ordering::SeqCst);
|
||||
tokio::time::sleep(Duration::from_secs(31)).await;
|
||||
tokio::time::sleep(Duration::from_secs(6)).await;
|
||||
let handle = provider.current().expect("provider must recover after a successful refresh");
|
||||
assert!(handle.generation >= 1);
|
||||
|
||||
|
||||
@@ -230,16 +230,8 @@ impl VaultTransitKmsClient {
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
};
|
||||
let source = token_source_for(&config.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
kms_config,
|
||||
&config.auth_method,
|
||||
"vault-transit",
|
||||
&config.address,
|
||||
config.namespace.as_deref(),
|
||||
);
|
||||
let policy = VaultCredentialPolicy::from_kms_config(kms_config, &config.auth_method);
|
||||
let credentials = Arc::new(VaultCredentialProvider::new(settings, source, policy).await?);
|
||||
let retry =
|
||||
RetryPolicy::for_backend(kms_config, "vault-transit", &config.address, config.namespace.as_deref(), "operations");
|
||||
|
||||
Ok(Self {
|
||||
credentials,
|
||||
@@ -250,7 +242,7 @@ impl VaultTransitKmsClient {
|
||||
.max_capacity(METADATA_CACHE_CAPACITY)
|
||||
.time_to_live(METADATA_CACHE_TTL)
|
||||
.build(),
|
||||
retry,
|
||||
retry: RetryPolicy::from_config(kms_config),
|
||||
cancel: CancellationToken::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -72,7 +72,6 @@ impl From<&BackupError> for RestoreBlocker {
|
||||
BackupError::Corrupted { .. } => RestoreBlockerCode::BundleCorrupted,
|
||||
BackupError::Truncated { .. } => RestoreBlockerCode::BundleTruncated,
|
||||
BackupError::UnknownVersion { .. } => RestoreBlockerCode::UnknownFormatVersion,
|
||||
BackupError::UnsupportedFormatVersion { .. } => RestoreBlockerCode::UnknownFormatVersion,
|
||||
BackupError::WrongKek { .. } => RestoreBlockerCode::WrongBackupKek,
|
||||
BackupError::MissingArtifact { .. } => RestoreBlockerCode::MissingArtifact,
|
||||
BackupError::IncompleteBundle { .. } => RestoreBlockerCode::IncompleteBundle,
|
||||
|
||||
@@ -51,11 +51,6 @@ pub enum BackupError {
|
||||
supplied_kek_version: u32,
|
||||
},
|
||||
|
||||
/// A bundled key record declares a format version this build does not
|
||||
/// understand.
|
||||
#[error("bundled key record '{key_id}' declares unsupported format version {version}; this build cannot restore it")]
|
||||
UnsupportedFormatVersion { key_id: String, version: String },
|
||||
|
||||
/// Manifest requires an artifact that is not present in the bundle.
|
||||
#[error("backup bundle is missing a required artifact: {artifact}")]
|
||||
MissingArtifact { artifact: String },
|
||||
@@ -133,15 +128,6 @@ mod tests {
|
||||
"unknown backup manifest format version 9 (this build supports version 1)"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
BackupError::UnsupportedFormatVersion {
|
||||
key_id: "alpha".to_string(),
|
||||
version: "9".to_string(),
|
||||
}
|
||||
.to_string(),
|
||||
"bundled key record 'alpha' declares unsupported format version 9; this build cannot restore it"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
BackupError::missing_artifact("key-material").to_string(),
|
||||
"backup bundle is missing a required artifact: key-material"
|
||||
|
||||
@@ -48,10 +48,7 @@
|
||||
//! published. A crash at any earlier point leaves a bundle without a
|
||||
//! manifest, which decodes as an incomplete bundle and can never be restored.
|
||||
|
||||
use crate::backends::local::{
|
||||
LocalKmsClient, STORED_MASTER_KEY_FORMAT_VERSION, StoredKeyProtection, UNKNOWN_STORED_KEY_PROTECTION,
|
||||
has_unknown_protection_marker, stored_master_key_format_version,
|
||||
};
|
||||
use crate::backends::local::{LocalKmsClient, StoredKeyProtection, unknown_protection_marker};
|
||||
use crate::backup::capability::{AtRestProtection, BackupBackendKind, BackupResponsibility};
|
||||
use crate::backup::error::BackupError;
|
||||
use crate::backup::manifest::{
|
||||
@@ -208,7 +205,8 @@ pub async fn export_local_backup(
|
||||
request: &LocalBackupExportRequest,
|
||||
) -> Result<BackupManifest> {
|
||||
request.validate()?;
|
||||
validate_destination(&request.destination).await?;
|
||||
prepare_destination(&request.destination).await?;
|
||||
|
||||
let snapshot = collect_snapshot(client).await?;
|
||||
if snapshot.records.is_empty() {
|
||||
return Err(KmsError::invalid_operation(
|
||||
@@ -235,7 +233,6 @@ pub async fn export_local_backup(
|
||||
None => None,
|
||||
};
|
||||
|
||||
prepare_destination(&request.destination).await?;
|
||||
let manifest = build_and_write_bundle(kek, request, &snapshot, master_key_verifier).await?;
|
||||
Ok(manifest)
|
||||
}
|
||||
@@ -383,17 +380,11 @@ async fn collect_snapshot(client: &LocalKmsClient) -> Result<CollectedSnapshot>
|
||||
// classified first so a record from a newer build keeps its own
|
||||
// verdict — an operator who reads "material corrupt" starts a
|
||||
// disaster recovery for what is only a version mismatch.
|
||||
let format_version = stored_master_key_format_version(&raw).map_err(|error| {
|
||||
let unknown_marker = unknown_protection_marker(&raw).map_err(|error| {
|
||||
KmsError::material_corrupt(&stem, format!("stored key record is not a readable JSON object: {error}"))
|
||||
})?;
|
||||
if format_version > STORED_MASTER_KEY_FORMAT_VERSION {
|
||||
return Err(KmsError::unsupported_format_version(&stem, format_version.to_string()));
|
||||
}
|
||||
let has_unknown_marker = has_unknown_protection_marker(&raw).map_err(|error| {
|
||||
KmsError::material_corrupt(&stem, format!("stored key record is not a readable JSON object: {error}"))
|
||||
})?;
|
||||
if has_unknown_marker {
|
||||
return Err(KmsError::unsupported_format_version(&stem, UNKNOWN_STORED_KEY_PROTECTION));
|
||||
if let Some(version) = unknown_marker {
|
||||
return Err(KmsError::unsupported_format_version(&stem, version));
|
||||
}
|
||||
let probe: StoredRecordProbe = serde_json::from_slice(&raw)
|
||||
.map_err(|error| KmsError::material_corrupt(&stem, format!("stored key record does not deserialize: {error}")))?;
|
||||
@@ -619,7 +610,7 @@ fn local_kdf_descriptor(snapshot: &CollectedSnapshot, master_key_verifier: Optio
|
||||
}
|
||||
}
|
||||
|
||||
async fn validate_destination(destination: &Path) -> Result<()> {
|
||||
async fn prepare_destination(destination: &Path) -> Result<()> {
|
||||
if fs::try_exists(destination).await? {
|
||||
let mut entries = fs::read_dir(destination)
|
||||
.await
|
||||
@@ -630,11 +621,6 @@ async fn validate_destination(destination: &Path) -> Result<()> {
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn prepare_destination(destination: &Path) -> Result<()> {
|
||||
validate_destination(destination).await?;
|
||||
fs::create_dir_all(destination.join(KEYS_DIR)).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1151,7 +1137,7 @@ mod tests {
|
||||
let record_path = client.key_directory().join("alpha.key");
|
||||
let mut record: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(&record_path).expect("read record")).expect("decode record");
|
||||
record["at_rest_protection"] = serde_json::json!("secret-marker-value-must-not-leak");
|
||||
record["at_rest_protection"] = serde_json::json!("post-quantum-v2");
|
||||
std::fs::write(&record_path, serde_json::to_vec_pretty(&record).expect("encode record")).expect("write record");
|
||||
|
||||
let bundle = TempDir::new().expect("bundle dir");
|
||||
@@ -1160,44 +1146,8 @@ mod tests {
|
||||
.expect_err("an uninterpretable record must abort the export");
|
||||
assert!(
|
||||
matches!(&error, KmsError::UnsupportedFormatVersion { key_id, version }
|
||||
if key_id == "alpha" && version == UNKNOWN_STORED_KEY_PROTECTION),
|
||||
if key_id == "alpha" && version == "post-quantum-v2"),
|
||||
"got {error:?}"
|
||||
);
|
||||
assert!(!error.to_string().contains("secret-marker-value-must-not-leak"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn numeric_record_format_version_from_a_newer_build_aborts_export_as_unsupported_format() {
|
||||
let (client, _key_dir) = encrypted_client().await;
|
||||
client.create_key("alpha", "AES_256", None).await.expect("create key");
|
||||
|
||||
let record_path = client.key_directory().join("alpha.key");
|
||||
let mut record: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(&record_path).expect("read record")).expect("decode record");
|
||||
record["format_version"] = serde_json::json!(99);
|
||||
std::fs::write(&record_path, serde_json::to_vec_pretty(&record).expect("encode record")).expect("write record");
|
||||
|
||||
let bundle = TempDir::new().expect("bundle dir");
|
||||
let destination = bundle.path().join("bundle");
|
||||
let error = export_local_backup(&client, &test_kek(), &export_request(destination.clone()))
|
||||
.await
|
||||
.expect_err("a newer record format must abort the export");
|
||||
assert!(
|
||||
matches!(&error, KmsError::UnsupportedFormatVersion { key_id, version }
|
||||
if key_id == "alpha" && version == "99"),
|
||||
"got {error:?}"
|
||||
);
|
||||
assert!(
|
||||
!destination.exists(),
|
||||
"format validation must finish before the export creates its destination"
|
||||
);
|
||||
|
||||
record["format_version"] = serde_json::json!(STORED_MASTER_KEY_FORMAT_VERSION);
|
||||
std::fs::write(&record_path, serde_json::to_vec_pretty(&record).expect("encode supported record"))
|
||||
.expect("write supported record");
|
||||
export_local_backup(&client, &test_kek(), &export_request(destination.clone()))
|
||||
.await
|
||||
.expect("the same destination must remain usable after validation fails");
|
||||
assert!(destination.join(LOCAL_BUNDLE_MANIFEST_FILE).exists());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,8 +54,7 @@
|
||||
|
||||
use crate::backends::local::{
|
||||
LOCAL_KMS_MASTER_KEY_SALT_FILE, LOCAL_KMS_MASTER_KEY_SALT_LEN, LOCAL_RESTORE_COMMIT_MARKER_FILE, LocalKmsClient,
|
||||
STORED_MASTER_KEY_FORMAT_VERSION, StoredKeyProtection, UNKNOWN_STORED_KEY_PROTECTION, durable_file,
|
||||
has_unknown_protection_marker, is_orphan_commit_temp_name, stored_master_key_format_version, validate_key_id,
|
||||
StoredKeyProtection, durable_file, is_orphan_commit_temp_name, unknown_protection_marker, validate_key_id,
|
||||
};
|
||||
use crate::backup::capability::AtRestProtection;
|
||||
use crate::backup::dry_run::{
|
||||
@@ -609,23 +608,10 @@ fn decode_key_record(
|
||||
// Classify the protection marker before the schema parse: a record from a
|
||||
// newer build is not a damaged bundle, and reporting it as corruption
|
||||
// sends the operator into disaster recovery instead of a version change.
|
||||
let format_version = stored_master_key_format_version(&plaintext)
|
||||
let unknown_marker = unknown_protection_marker(&plaintext)
|
||||
.map_err(|error| BackupError::corrupted(format!("bundled key record '{stem}' is not a readable JSON object: {error}")))?;
|
||||
if format_version > STORED_MASTER_KEY_FORMAT_VERSION {
|
||||
return Err(BackupError::UnsupportedFormatVersion {
|
||||
key_id: stem,
|
||||
version: format_version.to_string(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let has_unknown_marker = has_unknown_protection_marker(&plaintext)
|
||||
.map_err(|error| BackupError::corrupted(format!("bundled key record '{stem}' is not a readable JSON object: {error}")))?;
|
||||
if has_unknown_marker {
|
||||
return Err(BackupError::UnsupportedRecordVersion {
|
||||
key_id: stem,
|
||||
version: UNKNOWN_STORED_KEY_PROTECTION.to_owned(),
|
||||
}
|
||||
.into());
|
||||
if let Some(version) = unknown_marker {
|
||||
return Err(BackupError::UnsupportedRecordVersion { key_id: stem, version }.into());
|
||||
}
|
||||
let probe: RestoredRecordProbe = serde_json::from_slice(&plaintext)
|
||||
.map_err(|error| BackupError::corrupted(format!("bundled key record '{stem}' does not deserialize: {error}")))?;
|
||||
@@ -2088,7 +2074,7 @@ mod tests {
|
||||
fn bundled_record_from_a_newer_build_is_not_reported_as_corruption() {
|
||||
let record = serde_json::json!({
|
||||
"key_id": "alpha",
|
||||
"at_rest_protection": "secret-marker-value-must-not-leak",
|
||||
"at_rest_protection": "post-quantum-v2",
|
||||
"encrypted_key_material": "AAAAAAAAAAAAAAAAAAAAAA==",
|
||||
"nonce": vec![0u8; 12],
|
||||
});
|
||||
@@ -2106,35 +2092,7 @@ mod tests {
|
||||
};
|
||||
assert!(
|
||||
matches!(inner, BackupError::UnsupportedRecordVersion { key_id, version }
|
||||
if key_id == "alpha" && version == UNKNOWN_STORED_KEY_PROTECTION),
|
||||
"got {inner:?}"
|
||||
);
|
||||
assert!(!error.to_string().contains("secret-marker-value-must-not-leak"));
|
||||
assert_eq!(
|
||||
RestoreBlocker::from(inner).code,
|
||||
RestoreBlockerCode::UnknownFormatVersion,
|
||||
"a dry run must report a version blocker, not a corruption blocker"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bundled_record_format_version_from_a_newer_build_is_not_reported_as_corruption() {
|
||||
let record = serde_json::json!({"format_version": 99});
|
||||
let error = match decode_key_record(
|
||||
"artifacts/keys/alpha.key.enc",
|
||||
Zeroizing::new(serde_json::to_vec(&record).expect("encode record")),
|
||||
&[AtRestProtection::EncryptedMasterKey],
|
||||
) {
|
||||
Ok(_) => panic!("a record this build cannot interpret must be rejected"),
|
||||
Err(error) => error,
|
||||
};
|
||||
|
||||
let KmsError::Backup(inner) = &error else {
|
||||
panic!("expected a backup error, got {error:?}");
|
||||
};
|
||||
assert!(
|
||||
matches!(inner, BackupError::UnsupportedFormatVersion { key_id, version }
|
||||
if key_id == "alpha" && version == "99"),
|
||||
if key_id == "alpha" && version == "post-quantum-v2"),
|
||||
"got {inner:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
|
||||
@@ -452,24 +452,12 @@ impl VaultRestoreClient {
|
||||
attempt_timeout: kms_config.effective_timeout(),
|
||||
};
|
||||
let source = token_source_for(&target.auth_method, &settings)?;
|
||||
let policy = VaultCredentialPolicy::from_kms_config(
|
||||
kms_config,
|
||||
&target.auth_method,
|
||||
"vault-restore",
|
||||
&target.address,
|
||||
target.namespace.as_deref(),
|
||||
);
|
||||
let policy = VaultCredentialPolicy::from_kms_config(kms_config, &target.auth_method);
|
||||
let credentials = Arc::new(VaultCredentialProvider::new(settings, source, policy).await?);
|
||||
Ok(Self {
|
||||
credentials,
|
||||
kv_mount: target.kv_mount.clone(),
|
||||
retry: RetryPolicy::for_backend(
|
||||
kms_config,
|
||||
"vault-restore",
|
||||
&target.address,
|
||||
target.namespace.as_deref(),
|
||||
"operations",
|
||||
),
|
||||
retry: RetryPolicy::from_config(kms_config),
|
||||
cancel: CancellationToken::new(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -21,35 +21,12 @@
|
||||
#![allow(dead_code)] // Trait methods may be used by implementations
|
||||
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::persisted_observability::{BoundedUnknownFieldName, UnknownFieldSummary};
|
||||
use async_trait::async_trait;
|
||||
use jiff::Zoned;
|
||||
use rand::Rng;
|
||||
use serde::de::{self, IgnoredAny, MapAccess, Visitor};
|
||||
use serde::de::IgnoredAny;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
impl UnknownFieldSummary {
|
||||
fn record_for_data_key_envelope(&self) {
|
||||
let Some((field, field_name_truncated, field_count)) = self.record("data-key-envelope") else {
|
||||
return;
|
||||
};
|
||||
|
||||
static RECORDS_WITH_UNKNOWN_FIELDS: AtomicU64 = AtomicU64::new(0);
|
||||
let observed_records = RECORDS_WITH_UNKNOWN_FIELDS.fetch_add(1, Ordering::Relaxed).saturating_add(1);
|
||||
if observed_records.is_power_of_two() {
|
||||
tracing::warn!(
|
||||
field = ?field,
|
||||
field_name_truncated,
|
||||
field_count,
|
||||
observed_records,
|
||||
"KMS data-key envelope contains unknown fields"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Data key envelope for encrypting/decrypting data keys
|
||||
///
|
||||
@@ -59,7 +36,7 @@ impl UnknownFieldSummary {
|
||||
/// material. Envelopes written before versioning carry `None`; backends must resolve
|
||||
/// `None` to a deterministic baseline version recorded in key metadata, never
|
||||
/// implicitly to whatever version is current.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DataKeyEnvelope {
|
||||
pub key_id: String,
|
||||
pub master_key_id: String,
|
||||
@@ -77,140 +54,6 @@ pub struct DataKeyEnvelope {
|
||||
pub master_key_version: Option<u32>,
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for DataKeyEnvelope {
|
||||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
enum Field {
|
||||
KeyId,
|
||||
MasterKeyId,
|
||||
KeySpec,
|
||||
EncryptedKey,
|
||||
Nonce,
|
||||
EncryptionContext,
|
||||
CreatedAt,
|
||||
MasterKeyVersion,
|
||||
Unknown(BoundedUnknownFieldName),
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for Field {
|
||||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
struct FieldVisitor;
|
||||
|
||||
impl Visitor<'_> for FieldVisitor {
|
||||
type Value = Field;
|
||||
|
||||
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("a KMS data-key envelope field name")
|
||||
}
|
||||
|
||||
fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E>
|
||||
where
|
||||
E: de::Error,
|
||||
{
|
||||
Ok(match value {
|
||||
"key_id" => Field::KeyId,
|
||||
"master_key_id" => Field::MasterKeyId,
|
||||
"key_spec" => Field::KeySpec,
|
||||
"encrypted_key" => Field::EncryptedKey,
|
||||
"nonce" => Field::Nonce,
|
||||
"encryption_context" => Field::EncryptionContext,
|
||||
"created_at" => Field::CreatedAt,
|
||||
"master_key_version" => Field::MasterKeyVersion,
|
||||
_ => Field::Unknown(BoundedUnknownFieldName::new(value)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_identifier(FieldVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ZonedValue(#[serde(with = "crate::time_serde::zoned")] Zoned);
|
||||
|
||||
struct DataKeyEnvelopeVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for DataKeyEnvelopeVisitor {
|
||||
type Value = DataKeyEnvelope;
|
||||
|
||||
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("a KMS data-key envelope")
|
||||
}
|
||||
|
||||
fn visit_map<A>(self, mut map: A) -> std::result::Result<Self::Value, A::Error>
|
||||
where
|
||||
A: MapAccess<'de>,
|
||||
{
|
||||
macro_rules! read_field {
|
||||
($slot:ident, $name:literal) => {{
|
||||
if $slot.is_some() {
|
||||
return Err(de::Error::duplicate_field($name));
|
||||
}
|
||||
$slot = Some(map.next_value()?);
|
||||
}};
|
||||
}
|
||||
|
||||
let mut key_id = None;
|
||||
let mut master_key_id = None;
|
||||
let mut key_spec = None;
|
||||
let mut encrypted_key = None;
|
||||
let mut nonce = None;
|
||||
let mut encryption_context = None;
|
||||
let mut created_at: Option<ZonedValue> = None;
|
||||
let mut master_key_version = None;
|
||||
let mut unknown_fields = UnknownFieldSummary::default();
|
||||
|
||||
while let Some(field) = map.next_key()? {
|
||||
match field {
|
||||
Field::KeyId => read_field!(key_id, "key_id"),
|
||||
Field::MasterKeyId => read_field!(master_key_id, "master_key_id"),
|
||||
Field::KeySpec => read_field!(key_spec, "key_spec"),
|
||||
Field::EncryptedKey => read_field!(encrypted_key, "encrypted_key"),
|
||||
Field::Nonce => read_field!(nonce, "nonce"),
|
||||
Field::EncryptionContext => read_field!(encryption_context, "encryption_context"),
|
||||
Field::CreatedAt => read_field!(created_at, "created_at"),
|
||||
Field::MasterKeyVersion => read_field!(master_key_version, "master_key_version"),
|
||||
Field::Unknown(field) => {
|
||||
let _: IgnoredAny = map.next_value()?;
|
||||
unknown_fields.observe(field);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let envelope = DataKeyEnvelope {
|
||||
key_id: key_id.ok_or_else(|| de::Error::missing_field("key_id"))?,
|
||||
master_key_id: master_key_id.ok_or_else(|| de::Error::missing_field("master_key_id"))?,
|
||||
key_spec: key_spec.ok_or_else(|| de::Error::missing_field("key_spec"))?,
|
||||
encrypted_key: encrypted_key.ok_or_else(|| de::Error::missing_field("encrypted_key"))?,
|
||||
nonce: nonce.ok_or_else(|| de::Error::missing_field("nonce"))?,
|
||||
encryption_context: encryption_context.ok_or_else(|| de::Error::missing_field("encryption_context"))?,
|
||||
created_at: created_at.ok_or_else(|| de::Error::missing_field("created_at"))?.0,
|
||||
master_key_version: master_key_version.unwrap_or(None),
|
||||
};
|
||||
unknown_fields.record_for_data_key_envelope();
|
||||
Ok(envelope)
|
||||
}
|
||||
}
|
||||
|
||||
const FIELDS: &[&str] = &[
|
||||
"key_id",
|
||||
"master_key_id",
|
||||
"key_spec",
|
||||
"encrypted_key",
|
||||
"nonce",
|
||||
"encryption_context",
|
||||
"created_at",
|
||||
"master_key_version",
|
||||
];
|
||||
deserializer.deserialize_struct("DataKeyEnvelope", FIELDS, DataKeyEnvelopeVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct DataKeyEnvelopeMarker {
|
||||
#[serde(rename = "key_id")]
|
||||
@@ -394,8 +237,6 @@ pub fn generate_key_material(algorithm: &str) -> Result<Vec<u8>> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::test_support::{deserialize_with_ignored_only_unknown, unknown_field_metric};
|
||||
use metrics_util::debugging::DebuggingRecorder;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_aes_dek_crypto_encrypt_decrypt() {
|
||||
@@ -527,76 +368,6 @@ mod tests {
|
||||
assert_eq!(deserialized.master_key_version, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_key_envelope_unknown_fields_remain_readable() {
|
||||
const UNKNOWN_FIELD_VALUE: &str = "field value must not be logged";
|
||||
let envelope = serde_json::json!({
|
||||
"key_id": "test-key-id",
|
||||
"master_key_id": "master-key-id",
|
||||
"key_spec": "AES_256",
|
||||
"encrypted_key": [1, 2, 3, 4],
|
||||
"nonce": [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
|
||||
"encryption_context": {"bucket": "test-bucket"},
|
||||
"created_at": "2024-01-01T00:00:00+00:00[UTC]"
|
||||
});
|
||||
let long_field = format!("{}界", "a".repeat(126));
|
||||
let long_prefix = "a".repeat(126);
|
||||
let injection_field = "b\n\u{1b}[31m";
|
||||
|
||||
let record_with_unknown = |field: &str| {
|
||||
let mut record = envelope.clone();
|
||||
let object = record.as_object_mut().expect("envelope is an object");
|
||||
object.insert(field.to_owned(), serde_json::json!(UNKNOWN_FIELD_VALUE));
|
||||
object.insert("zeta_extension".to_owned(), serde_json::json!("another value must not be logged"));
|
||||
serde_json::to_vec(&record).expect("encode envelope with unknown fields")
|
||||
};
|
||||
let long_record = record_with_unknown(&long_field);
|
||||
let injection_record = record_with_unknown(injection_field);
|
||||
let logs = crate::test_support::CapturedLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_ansi(false)
|
||||
.with_max_level(tracing::Level::WARN)
|
||||
.with_writer(logs.clone())
|
||||
.finish();
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
let parse = |record: &[u8]| {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let envelope = metrics::with_local_recorder(&recorder, || {
|
||||
tracing::dispatcher::with_default(&dispatch, || {
|
||||
serde_json::from_slice(record).expect("unknown fields must remain readable")
|
||||
})
|
||||
});
|
||||
assert_eq!(unknown_field_metric(&recorder, "data-key-envelope"), 2);
|
||||
envelope
|
||||
};
|
||||
let deserialized: DataKeyEnvelope = parse(&long_record);
|
||||
let _: DataKeyEnvelope = parse(&long_record);
|
||||
let _: DataKeyEnvelope = parse(&injection_record);
|
||||
let _: DataKeyEnvelope = parse(&injection_record);
|
||||
assert_eq!(deserialized.key_id, "test-key-id");
|
||||
assert_eq!(deserialized.master_key_version, None);
|
||||
|
||||
let output = logs.output();
|
||||
assert!(output.contains("WARN"));
|
||||
assert_eq!(output.matches("KMS data-key envelope contains unknown fields").count(), 3);
|
||||
assert!(output.contains(&long_prefix));
|
||||
assert!(!output.contains(&long_field));
|
||||
assert!(output.contains("field_name_truncated=true"));
|
||||
assert!(output.contains(r#"\n\u{1b}[31m"#));
|
||||
assert!(!output.contains("zeta_extension"));
|
||||
assert!(output.contains("field_count=2"));
|
||||
for observed_records in [1, 2, 4] {
|
||||
assert!(output.contains(&format!("observed_records={observed_records}")));
|
||||
}
|
||||
assert!(!output.contains("observed_records=3"));
|
||||
assert!(!output.contains(UNKNOWN_FIELD_VALUE));
|
||||
assert!(!output.contains("another value must not be logged"));
|
||||
|
||||
let streamed: DataKeyEnvelope = deserialize_with_ignored_only_unknown(envelope, "stream_only_extension")
|
||||
.expect("unknown values must be consumed through deserialize_ignored_any");
|
||||
assert_eq!(streamed.key_id, "test-key-id");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_key_envelope_none_version_serializes_without_field() {
|
||||
// A `None` version must keep the serialized envelope on the historical
|
||||
|
||||
@@ -75,7 +75,6 @@ mod encryption;
|
||||
mod error;
|
||||
pub mod key_impact;
|
||||
pub mod manager;
|
||||
mod persisted_observability;
|
||||
mod policy;
|
||||
pub mod probe;
|
||||
pub mod service;
|
||||
@@ -83,166 +82,6 @@ pub mod service_manager;
|
||||
mod time_serde;
|
||||
pub mod types;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_support {
|
||||
use crate::persisted_observability::UNKNOWN_FIELDS_METRIC;
|
||||
use metrics_util::MetricKind;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use serde::Deserializer;
|
||||
use serde::de::value::MapDeserializer;
|
||||
use serde::de::{self, DeserializeOwned, IntoDeserializer, Visitor};
|
||||
use std::io::{self, Write};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct CapturedLogs {
|
||||
output: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
pub(crate) struct CapturedWriter(CapturedLogs);
|
||||
|
||||
impl Write for CapturedWriter {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.0.output.lock().expect("log buffer lock poisoned").extend_from_slice(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs {
|
||||
type Writer = CapturedWriter;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
CapturedWriter(self.clone())
|
||||
}
|
||||
}
|
||||
|
||||
impl CapturedLogs {
|
||||
pub(crate) fn output(&self) -> String {
|
||||
String::from_utf8(self.output.lock().expect("log buffer lock poisoned").clone())
|
||||
.expect("captured logs should be UTF-8")
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn unknown_field_metric(recorder: &DebuggingRecorder, record_kind: &str) -> u64 {
|
||||
recorder
|
||||
.snapshotter()
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.filter_map(|(composite, _unit, _description, value)| {
|
||||
let matches = composite.kind() == MetricKind::Counter
|
||||
&& composite.key().name() == UNKNOWN_FIELDS_METRIC
|
||||
&& composite
|
||||
.key()
|
||||
.labels()
|
||||
.any(|label| label.key() == "record_kind" && label.value() == record_kind);
|
||||
match (matches, value) {
|
||||
(true, DebugValue::Counter(count)) => Some(count),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
enum IgnoredOnlyValue {
|
||||
Json(serde_json::Value),
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl<'de> IntoDeserializer<'de, serde_json::Error> for IgnoredOnlyValue {
|
||||
type Deserializer = Self;
|
||||
|
||||
fn into_deserializer(self) -> Self::Deserializer {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserializer<'de> for IgnoredOnlyValue {
|
||||
type Error = serde_json::Error;
|
||||
|
||||
fn deserialize_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>
|
||||
where
|
||||
V: Visitor<'de>,
|
||||
{
|
||||
match self {
|
||||
Self::Json(value) => value.deserialize_any(visitor),
|
||||
Self::Unknown => Err(de::Error::custom("unknown value was materialized")),
|
||||
}
|
||||
}
|
||||
|
||||
fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Self::Error>
|
||||
where
|
||||
V: Visitor<'de>,
|
||||
{
|
||||
match self {
|
||||
Self::Json(value) => value.deserialize_option(visitor),
|
||||
Self::Unknown => Err(de::Error::custom("unknown value was materialized")),
|
||||
}
|
||||
}
|
||||
|
||||
fn deserialize_newtype_struct<V>(self, name: &'static str, visitor: V) -> Result<V::Value, Self::Error>
|
||||
where
|
||||
V: Visitor<'de>,
|
||||
{
|
||||
match self {
|
||||
Self::Json(value) => value.deserialize_newtype_struct(name, visitor),
|
||||
Self::Unknown => Err(de::Error::custom("unknown value was materialized")),
|
||||
}
|
||||
}
|
||||
|
||||
fn deserialize_enum<V>(
|
||||
self,
|
||||
name: &'static str,
|
||||
variants: &'static [&'static str],
|
||||
visitor: V,
|
||||
) -> Result<V::Value, Self::Error>
|
||||
where
|
||||
V: Visitor<'de>,
|
||||
{
|
||||
match self {
|
||||
Self::Json(value) => value.deserialize_enum(name, variants, visitor),
|
||||
Self::Unknown => Err(de::Error::custom("unknown value was materialized")),
|
||||
}
|
||||
}
|
||||
|
||||
fn deserialize_ignored_any<V>(self, visitor: V) -> Result<V::Value, Self::Error>
|
||||
where
|
||||
V: Visitor<'de>,
|
||||
{
|
||||
match self {
|
||||
Self::Json(value) => value.deserialize_ignored_any(visitor),
|
||||
Self::Unknown => visitor.visit_unit(),
|
||||
}
|
||||
}
|
||||
|
||||
serde::forward_to_deserialize_any! {
|
||||
bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string
|
||||
bytes byte_buf unit unit_struct seq tuple tuple_struct map struct identifier
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn deserialize_with_ignored_only_unknown<T>(
|
||||
record: serde_json::Value,
|
||||
unknown_field: &str,
|
||||
) -> Result<T, serde_json::Error>
|
||||
where
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
let object = record
|
||||
.as_object()
|
||||
.ok_or_else(|| de::Error::custom("test record must be an object"))?;
|
||||
let entries = object
|
||||
.iter()
|
||||
.map(|(key, value)| (key.clone(), IgnoredOnlyValue::Json(value.clone())))
|
||||
.chain([(unknown_field.to_owned(), IgnoredOnlyValue::Unknown)]);
|
||||
T::deserialize(MapDeserializer::<_, serde_json::Error>::new(entries))
|
||||
}
|
||||
}
|
||||
|
||||
// Re-export public API
|
||||
pub use api_types::{
|
||||
CacheSummary, ConfigureAwsKmsRequest, ConfigureKmsRequest, ConfigureKmsResponse, ConfigureLocalKmsRequest,
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
const UNKNOWN_FIELD_NAME_MAX_BYTES: usize = 128;
|
||||
pub(crate) const UNKNOWN_FIELDS_METRIC: &str = "rustfs_kms_persisted_unknown_fields_total";
|
||||
|
||||
pub(crate) struct BoundedUnknownFieldName {
|
||||
value: String,
|
||||
truncated: bool,
|
||||
}
|
||||
|
||||
impl BoundedUnknownFieldName {
|
||||
pub(crate) fn new(value: &str) -> Self {
|
||||
if value.len() <= UNKNOWN_FIELD_NAME_MAX_BYTES {
|
||||
return Self {
|
||||
value: value.to_owned(),
|
||||
truncated: false,
|
||||
};
|
||||
}
|
||||
|
||||
let mut end = UNKNOWN_FIELD_NAME_MAX_BYTES;
|
||||
while !value.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
Self {
|
||||
value: value[..end].to_owned(),
|
||||
truncated: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct UnknownFieldSummary {
|
||||
count: u64,
|
||||
first: Option<BoundedUnknownFieldName>,
|
||||
}
|
||||
|
||||
impl UnknownFieldSummary {
|
||||
pub(crate) fn observe(&mut self, field: BoundedUnknownFieldName) {
|
||||
self.count = self.count.saturating_add(1);
|
||||
if self.first.is_none() {
|
||||
self.first = Some(field);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record(&self, record_kind: &'static str) -> Option<(&str, bool, u64)> {
|
||||
let field = self.first.as_ref()?;
|
||||
metrics::counter!(UNKNOWN_FIELDS_METRIC, "record_kind" => record_kind).increment(self.count);
|
||||
Some((&field.value, field.truncated, self.count))
|
||||
}
|
||||
}
|
||||
+12
-1001
File diff suppressed because it is too large
Load Diff
+9
@@ -0,0 +1,9 @@
|
||||
---
|
||||
source: crates/kms/src/api_types.rs
|
||||
expression: "serde_json::to_value(CancelKeyDeletionResponse\n{\n success: true, message: \"key deletion canceled\".to_string(), key_id:\n \"key-a\".to_string(),\n}).expect(\"cancel deletion response should serialize\")"
|
||||
---
|
||||
{
|
||||
"key_id": "key-a",
|
||||
"message": "key deletion canceled",
|
||||
"success": true
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
source: crates/kms/src/api_types.rs
|
||||
expression: "serde_json::to_value(DeleteKeyResponse\n{\n success: true, message: \"key scheduled for deletion\".to_string(), key_id:\n \"key-a\".to_string(), deletion_date:\n Some(\"2026-07-01T00:00:00Z\".to_string()),\n}).expect(\"delete key response should serialize\")"
|
||||
---
|
||||
{
|
||||
"deletion_date": "2026-07-01T00:00:00Z",
|
||||
"key_id": "key-a",
|
||||
"message": "key scheduled for deletion",
|
||||
"success": true
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
---
|
||||
source: crates/kms/src/api_types.rs
|
||||
expression: "serde_json::to_value(DescribeKeyResponse\n{\n success: false, message: \"key not found\".to_string(), key_metadata: None,\n}).expect(\"describe key response should serialize\")"
|
||||
---
|
||||
{
|
||||
"key_metadata": null,
|
||||
"message": "key not found",
|
||||
"success": false
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
---
|
||||
source: crates/kms/src/api_types.rs
|
||||
expression: "serde_json::to_value(ListKeysResponse\n{\n success: true, message: \"keys listed\".to_string(), keys:\n vec![\"key-a\".to_string(), \"key-b\".to_string()], truncated: true,\n next_marker: Some(\"key-b\".to_string()),\n}).expect(\"list keys response should serialize\")"
|
||||
---
|
||||
{
|
||||
"keys": [
|
||||
"key-a",
|
||||
"key-b"
|
||||
],
|
||||
"message": "keys listed",
|
||||
"next_marker": "key-b",
|
||||
"success": true,
|
||||
"truncated": true
|
||||
}
|
||||
+10
-2
@@ -415,7 +415,15 @@ impl DecryptRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/// Request to list keys
|
||||
/// Default page size used when a request omits `limit`.
|
||||
pub const DEFAULT_LIST_KEYS_LIMIT: u32 = 100;
|
||||
/// Maximum page size accepted by KMS list implementations.
|
||||
pub const MAX_LIST_KEYS_LIMIT: u32 = 1000;
|
||||
|
||||
/// Request to list keys.
|
||||
///
|
||||
/// List pages default to [`DEFAULT_LIST_KEYS_LIMIT`] entries and are capped at
|
||||
/// [`MAX_LIST_KEYS_LIMIT`]. A zero limit is a valid request for an empty page.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ListKeysRequest {
|
||||
/// Maximum number of keys to return
|
||||
@@ -431,7 +439,7 @@ pub struct ListKeysRequest {
|
||||
impl Default for ListKeysRequest {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
limit: Some(100),
|
||||
limit: Some(DEFAULT_LIST_KEYS_LIMIT),
|
||||
marker: None,
|
||||
usage_filter: None,
|
||||
status_filter: None,
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Ignored live-Vault checks for the AppRole authentication path.
|
||||
//!
|
||||
//! `scripts/test/vault_approle_kms_live.sh` starts an ephemeral Vault, creates
|
||||
//! a narrowly scoped AppRole, and runs each test with only the generated
|
||||
//! role_id and secret_id. The tests then build the KMS configuration from the
|
||||
//! same environment variables used by RustFS and exercise the real KV2 and
|
||||
//! Transit backend calls with the AppRole-issued token.
|
||||
|
||||
use rustfs_kms::backends::KmsBackend as KmsBackendTrait;
|
||||
use rustfs_kms::backends::vault::VaultKmsBackend;
|
||||
use rustfs_kms::backends::vault_transit::VaultTransitKmsBackend;
|
||||
use rustfs_kms::{
|
||||
BackendConfig, CreateKeyRequest, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT,
|
||||
DecryptRequest, GenerateDataKeyRequest, KeySpec, KeyUsage, KmsBackend, KmsConfig, ListKeysRequest, VaultAuthMethod,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
|
||||
fn assert_approle_config(config: &KmsConfig, expected_backend: KmsBackend) {
|
||||
assert_eq!(config.backend, expected_backend);
|
||||
let auth_method = match &config.backend_config {
|
||||
BackendConfig::VaultKv2(vault) => &vault.auth_method,
|
||||
BackendConfig::VaultTransit(vault) => &vault.auth_method,
|
||||
_ => panic!("expected Vault configuration"),
|
||||
};
|
||||
assert!(
|
||||
matches!(auth_method, VaultAuthMethod::AppRole { .. }),
|
||||
"live check must use Vault AppRole auth"
|
||||
);
|
||||
}
|
||||
|
||||
async fn exercise_backend<B: KmsBackendTrait + ?Sized>(backend: &B, key_prefix: &str) -> rustfs_kms::Result<()> {
|
||||
let key_id = format!("{key_prefix}-{}", uuid::Uuid::new_v4());
|
||||
let created = backend
|
||||
.create_key(CreateKeyRequest {
|
||||
key_name: Some(key_id.clone()),
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(created.key_id, key_id);
|
||||
|
||||
let described = backend
|
||||
.describe_key(rustfs_kms::DescribeKeyRequest { key_id: key_id.clone() })
|
||||
.await?;
|
||||
assert_eq!(described.key_metadata.key_id, key_id);
|
||||
|
||||
let listed = backend
|
||||
.list_keys(ListKeysRequest {
|
||||
limit: Some(100),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
assert!(
|
||||
listed.keys.iter().any(|key| key.key_id == key_id),
|
||||
"created key must be visible in the backend listing"
|
||||
);
|
||||
|
||||
let context = HashMap::from([("live".to_string(), "approle".to_string())]);
|
||||
let generated = backend
|
||||
.generate_data_key(GenerateDataKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
key_spec: KeySpec::Aes256,
|
||||
encryption_context: context.clone(),
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(generated.key_id, key_id);
|
||||
assert_eq!(generated.plaintext_key.len(), 32, "AES-256 must return a 32-byte data key");
|
||||
|
||||
let unwrapped = backend
|
||||
.decrypt(DecryptRequest {
|
||||
ciphertext: generated.ciphertext_blob,
|
||||
encryption_context: context,
|
||||
grant_tokens: Vec::new(),
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(unwrapped.plaintext, generated.plaintext_key);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a real Vault AppRole; run scripts/test/vault_approle_kms_live.sh"]
|
||||
async fn vault_kv2_approle_auth_live() -> rustfs_kms::Result<()> {
|
||||
let config = KmsConfig::from_env()?;
|
||||
assert_approle_config(&config, KmsBackend::VaultKv2);
|
||||
let backend = VaultKmsBackend::new(config).await?;
|
||||
exercise_backend(&backend, "rustfs-approle-kv2").await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a real Vault AppRole; run scripts/test/vault_approle_kms_live.sh"]
|
||||
async fn vault_transit_approle_auth_live() -> rustfs_kms::Result<()> {
|
||||
let config = KmsConfig::from_env()?;
|
||||
assert_approle_config(&config, KmsBackend::VaultTransit);
|
||||
let transit = match &config.backend_config {
|
||||
BackendConfig::VaultTransit(vault) => vault,
|
||||
_ => panic!("expected Vault Transit configuration"),
|
||||
};
|
||||
assert_eq!(transit.metadata_kv_mount, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT);
|
||||
assert_eq!(transit.metadata_key_prefix, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX);
|
||||
|
||||
let backend = VaultTransitKmsBackend::new(config).await?;
|
||||
exercise_backend(&backend, "rustfs-approle-transit").await
|
||||
}
|
||||
@@ -1,390 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Ignored live test for a real three-node Vault Raft leader failure.
|
||||
//!
|
||||
//! `scripts/test/vault_ha_kms_live.sh` owns the official Vault containers and
|
||||
//! kills the active node while this test continuously decrypts through a
|
||||
//! surviving standby. KV2 and Transit requests must remain successful, use a
|
||||
//! bounded number of attempts, and leave the circuit and in-flight gauges at
|
||||
//! zero after a new leader is elected.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use metrics_util::MetricKind;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};
|
||||
use rustfs_kms::backends::KmsBackend as KmsBackendTrait;
|
||||
use rustfs_kms::backends::vault::VaultKmsBackend;
|
||||
use rustfs_kms::backends::vault_transit::VaultTransitKmsBackend;
|
||||
use rustfs_kms::{
|
||||
BackendConfig, CreateKeyRequest, DecryptRequest, GenerateDataKeyRequest, KeySpec, KeyUsage, KmsBackend, KmsConfig,
|
||||
VaultAuthMethod, VaultConfig, VaultTransitConfig,
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const OPERATIONS_TOTAL: &str = "rustfs_kms_backend_operations_total";
|
||||
const ATTEMPT_FAILURES_TOTAL: &str = "rustfs_kms_backend_attempt_failures_total";
|
||||
const OPERATION_ATTEMPTS: &str = "rustfs_kms_backend_operation_attempts";
|
||||
const IN_FLIGHT: &str = "rustfs_kms_backend_in_flight";
|
||||
const CIRCUIT_OPEN: &str = "rustfs_kms_backend_circuit_open";
|
||||
const MAX_ATTEMPTS: u32 = 10;
|
||||
|
||||
type MetricEntry = (
|
||||
metrics_util::CompositeKey,
|
||||
Option<metrics::Unit>,
|
||||
Option<metrics::SharedString>,
|
||||
DebugValue,
|
||||
);
|
||||
|
||||
fn required_env(name: &str) -> String {
|
||||
std::env::var(name).unwrap_or_else(|_| panic!("{name} must be set by scripts/test/vault_ha_kms_live.sh"))
|
||||
}
|
||||
|
||||
fn auth_method() -> VaultAuthMethod {
|
||||
VaultAuthMethod::approle(required_env("RUSTFS_TEST_VAULT_ROLE_ID"), required_env("RUSTFS_TEST_VAULT_SECRET_ID"))
|
||||
}
|
||||
|
||||
fn config(backend: KmsBackend, backend_config: BackendConfig) -> KmsConfig {
|
||||
KmsConfig {
|
||||
backend,
|
||||
backend_config,
|
||||
allow_insecure_dev_defaults: true,
|
||||
timeout: Duration::from_secs(2),
|
||||
retry_attempts: MAX_ATTEMPTS,
|
||||
enable_cache: false,
|
||||
..KmsConfig::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn kv2_config(address: &str) -> KmsConfig {
|
||||
config(
|
||||
KmsBackend::VaultKv2,
|
||||
BackendConfig::VaultKv2(Box::new(VaultConfig {
|
||||
address: address.to_string(),
|
||||
auth_method: auth_method(),
|
||||
namespace: None,
|
||||
mount_path: "transit".to_string(),
|
||||
kv_mount: "secret".to_string(),
|
||||
key_path_prefix: "rustfs/kms/ha-kv2".to_string(),
|
||||
tls: None,
|
||||
})),
|
||||
)
|
||||
}
|
||||
|
||||
fn transit_config(address: &str) -> KmsConfig {
|
||||
config(
|
||||
KmsBackend::VaultTransit,
|
||||
BackendConfig::VaultTransit(Box::new(VaultTransitConfig {
|
||||
address: address.to_string(),
|
||||
auth_method: auth_method(),
|
||||
namespace: None,
|
||||
mount_path: "transit".to_string(),
|
||||
metadata_kv_mount: "secret".to_string(),
|
||||
metadata_key_prefix: "rustfs/kms/ha-transit-metadata".to_string(),
|
||||
tls: None,
|
||||
})),
|
||||
)
|
||||
}
|
||||
|
||||
fn labels_match(key: &metrics::Key, labels: &[(&str, &str)]) -> bool {
|
||||
labels.iter().all(|(label, expected)| {
|
||||
key.labels()
|
||||
.any(|candidate| candidate.key() == *label && candidate.value() == *expected)
|
||||
})
|
||||
}
|
||||
|
||||
fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 {
|
||||
snapshot
|
||||
.iter()
|
||||
.filter_map(|(composite, _, _, value)| {
|
||||
let matches = composite.kind() == MetricKind::Counter
|
||||
&& composite.key().name() == name
|
||||
&& labels_match(composite.key(), labels);
|
||||
match (matches, value) {
|
||||
(true, DebugValue::Counter(count)) => Some(*count),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
fn gauge_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> Option<f64> {
|
||||
snapshot.iter().find_map(|(composite, _, _, value)| {
|
||||
let matches =
|
||||
composite.kind() == MetricKind::Gauge && composite.key().name() == name && labels_match(composite.key(), labels);
|
||||
match (matches, value) {
|
||||
(true, DebugValue::Gauge(value)) => Some(value.into_inner()),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn histogram_values(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> Vec<f64> {
|
||||
snapshot
|
||||
.iter()
|
||||
.filter_map(|(composite, _, _, value)| {
|
||||
let matches = composite.kind() == MetricKind::Histogram
|
||||
&& composite.key().name() == name
|
||||
&& labels_match(composite.key(), labels);
|
||||
match (matches, value) {
|
||||
(true, DebugValue::Histogram(values)) => Some(values),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
.flatten()
|
||||
.map(|value| value.into_inner())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn retryable_failures(snapshot: &[MetricEntry], operation: &str) -> u64 {
|
||||
["retryable_conn", "retryable_status", "attempt_timeout"]
|
||||
.into_iter()
|
||||
.map(|error_class| {
|
||||
counter_value(
|
||||
snapshot,
|
||||
ATTEMPT_FAILURES_TOTAL,
|
||||
&[("operation", operation), ("error_class", error_class)],
|
||||
)
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
async fn wait_for_count(counter: &AtomicU64, minimum: u64, description: &str) {
|
||||
tokio::time::timeout(Duration::from_secs(20), async {
|
||||
while counter.load(Ordering::SeqCst) < minimum {
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| panic!("timed out waiting for {description}"));
|
||||
}
|
||||
|
||||
async fn wait_for_file(path: &Path, description: &str) {
|
||||
tokio::time::timeout(Duration::from_secs(70), async {
|
||||
while !path.exists() {
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| panic!("timed out waiting for {description}"));
|
||||
}
|
||||
|
||||
async fn decrypt_loop<B: KmsBackendTrait + Send + Sync + 'static>(
|
||||
backend: Arc<B>,
|
||||
request: DecryptRequest,
|
||||
expected: Vec<u8>,
|
||||
completed: Arc<AtomicU64>,
|
||||
failed: Arc<AtomicBool>,
|
||||
stop: CancellationToken,
|
||||
) {
|
||||
while !stop.is_cancelled() {
|
||||
match backend.decrypt(request.clone()).await {
|
||||
Ok(response) if response.plaintext == expected => {
|
||||
completed.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
Ok(_) | Err(_) => {
|
||||
failed.store(true, Ordering::SeqCst);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn exercise_failover(snapshotter: &Snapshotter) {
|
||||
let address = required_env("RUSTFS_TEST_VAULT_ADDRESS");
|
||||
let marker = PathBuf::from(required_env("RUSTFS_TEST_VAULT_FAILOVER_MARKER"));
|
||||
let elected = marker.with_extension("elected");
|
||||
let old_leader = required_env("RUSTFS_TEST_VAULT_OLD_LEADER");
|
||||
|
||||
let kv2 = Arc::new(VaultKmsBackend::new(kv2_config(&address)).await.expect("build KV2 backend"));
|
||||
let transit = Arc::new(
|
||||
VaultTransitKmsBackend::new(transit_config(&address))
|
||||
.await
|
||||
.expect("build Transit backend"),
|
||||
);
|
||||
let context = HashMap::from([("live".to_string(), "vault-ha-failover".to_string())]);
|
||||
|
||||
let kv2_key = format!("rustfs-ha-kv2-{}", uuid::Uuid::new_v4());
|
||||
kv2.create_key(CreateKeyRequest {
|
||||
key_name: Some(kv2_key.clone()),
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("create KV2 key");
|
||||
let kv2_data_key = kv2
|
||||
.generate_data_key(GenerateDataKeyRequest {
|
||||
key_id: kv2_key,
|
||||
key_spec: KeySpec::Aes256,
|
||||
encryption_context: context.clone(),
|
||||
})
|
||||
.await
|
||||
.expect("generate KV2 data key");
|
||||
|
||||
let transit_key = format!("rustfs-ha-transit-{}", uuid::Uuid::new_v4());
|
||||
transit
|
||||
.create_key(CreateKeyRequest {
|
||||
key_name: Some(transit_key.clone()),
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("create Transit key");
|
||||
let transit_data_key = transit
|
||||
.generate_data_key(GenerateDataKeyRequest {
|
||||
key_id: transit_key,
|
||||
key_spec: KeySpec::Aes256,
|
||||
encryption_context: context.clone(),
|
||||
})
|
||||
.await
|
||||
.expect("generate Transit data key");
|
||||
|
||||
let kv2_request = DecryptRequest {
|
||||
ciphertext: kv2_data_key.ciphertext_blob,
|
||||
encryption_context: context.clone(),
|
||||
grant_tokens: Vec::new(),
|
||||
};
|
||||
let transit_request = DecryptRequest {
|
||||
ciphertext: transit_data_key.ciphertext_blob,
|
||||
encryption_context: context,
|
||||
grant_tokens: Vec::new(),
|
||||
};
|
||||
|
||||
for _ in 0..2 {
|
||||
let kv2_response = kv2
|
||||
.decrypt(kv2_request.clone())
|
||||
.await
|
||||
.expect("healthy KV2 decrypt before failover");
|
||||
assert!(
|
||||
kv2_response.plaintext == kv2_data_key.plaintext_key,
|
||||
"healthy KV2 decrypt returned unexpected plaintext"
|
||||
);
|
||||
let transit_response = transit
|
||||
.decrypt(transit_request.clone())
|
||||
.await
|
||||
.expect("healthy Transit decrypt before failover");
|
||||
assert!(
|
||||
transit_response.plaintext == transit_data_key.plaintext_key,
|
||||
"healthy Transit decrypt returned unexpected plaintext"
|
||||
);
|
||||
}
|
||||
let baseline = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(
|
||||
retryable_failures(&baseline, "vault_kv2_read_key"),
|
||||
0,
|
||||
"healthy KV2 baseline must not retry"
|
||||
);
|
||||
assert_eq!(
|
||||
retryable_failures(&baseline, "vault_transit_decrypt"),
|
||||
0,
|
||||
"healthy Transit baseline must not retry"
|
||||
);
|
||||
|
||||
let stop = CancellationToken::new();
|
||||
let failed = Arc::new(AtomicBool::new(false));
|
||||
let kv2_completed = Arc::new(AtomicU64::new(0));
|
||||
let transit_completed = Arc::new(AtomicU64::new(0));
|
||||
let kv2_worker = tokio::spawn(decrypt_loop(
|
||||
Arc::clone(&kv2),
|
||||
kv2_request,
|
||||
kv2_data_key.plaintext_key,
|
||||
Arc::clone(&kv2_completed),
|
||||
Arc::clone(&failed),
|
||||
stop.clone(),
|
||||
));
|
||||
let transit_worker = tokio::spawn(decrypt_loop(
|
||||
Arc::clone(&transit),
|
||||
transit_request,
|
||||
transit_data_key.plaintext_key,
|
||||
Arc::clone(&transit_completed),
|
||||
Arc::clone(&failed),
|
||||
stop.clone(),
|
||||
));
|
||||
|
||||
wait_for_count(&kv2_completed, 2, "two healthy KV2 decrypts").await;
|
||||
wait_for_count(&transit_completed, 2, "two healthy Transit decrypts").await;
|
||||
std::fs::write(&marker, b"ready").expect("publish failover readiness marker");
|
||||
|
||||
wait_for_file(&elected, "the replacement Vault leader").await;
|
||||
let new_leader = std::fs::read_to_string(&elected).expect("read replacement Vault leader marker");
|
||||
assert_ne!(new_leader.trim(), old_leader, "the killed active node cannot remain leader");
|
||||
|
||||
let kv2_after_election = kv2_completed.load(Ordering::SeqCst) + 2;
|
||||
let transit_after_election = transit_completed.load(Ordering::SeqCst) + 2;
|
||||
wait_for_count(&kv2_completed, kv2_after_election, "post-failover KV2 decrypts").await;
|
||||
wait_for_count(&transit_completed, transit_after_election, "post-failover Transit decrypts").await;
|
||||
|
||||
stop.cancel();
|
||||
kv2_worker.await.expect("KV2 decrypt worker must join");
|
||||
transit_worker.await.expect("Transit decrypt worker must join");
|
||||
assert!(!failed.load(Ordering::SeqCst), "no decrypt may fail or return different plaintext");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires a real three-node Vault Raft cluster; run scripts/test/vault_ha_kms_live.sh"]
|
||||
fn vault_raft_leader_failure_preserves_kv2_and_transit_decrypts() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("current-thread runtime must build")
|
||||
.block_on(exercise_failover(&snapshotter));
|
||||
});
|
||||
let snapshot = snapshotter.snapshot().into_vec();
|
||||
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "circuit_open")]),
|
||||
0,
|
||||
"a bounded leader election must not open the circuit"
|
||||
);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, OPERATIONS_TOTAL, &[("outcome", "budget_exhausted")]),
|
||||
0,
|
||||
"leader failover must recover within the retry budget"
|
||||
);
|
||||
|
||||
for (backend, operation) in [
|
||||
("vault-kv2", "vault_kv2_read_key"),
|
||||
("vault-transit", "vault_transit_decrypt"),
|
||||
] {
|
||||
assert!(
|
||||
retryable_failures(&snapshot, operation) > 0,
|
||||
"{operation} must observe the killed leader as a retryable attempt failure"
|
||||
);
|
||||
let attempts = histogram_values(&snapshot, OPERATION_ATTEMPTS, &[("operation", operation), ("outcome", "success")]);
|
||||
assert!(!attempts.is_empty(), "{operation} must record successful attempts");
|
||||
assert!(
|
||||
attempts
|
||||
.iter()
|
||||
.all(|attempts| (1.0..=f64::from(MAX_ATTEMPTS)).contains(attempts)),
|
||||
"{operation} attempts must stay within the configured budget: {attempts:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
gauge_value(&snapshot, IN_FLIGHT, &[("backend", backend), ("scope", "operations")]),
|
||||
Some(0.0),
|
||||
"{backend} must release every in-flight permit"
|
||||
);
|
||||
assert_eq!(
|
||||
gauge_value(&snapshot, CIRCUIT_OPEN, &[("backend", backend), ("scope", "operations")]),
|
||||
Some(0.0),
|
||||
"{backend} circuit must remain closed after recovery"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -14,8 +14,7 @@
|
||||
|
||||
use std::{collections::HashMap, time::SystemTime};
|
||||
|
||||
use serde::ser::SerializeStruct;
|
||||
use serde::{Deserialize, Serialize, Serializer};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::metrics::TimedAction;
|
||||
@@ -62,41 +61,17 @@ impl ItemState {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct DiskMetrics {
|
||||
#[serde(rename = "lastMinute", alias = "last_minute")]
|
||||
pub last_minute: HashMap<String, TimedAction>,
|
||||
#[serde(rename = "apiCalls", alias = "api_calls")]
|
||||
pub api_calls: HashMap<String, u64>,
|
||||
#[serde(rename = "totalWaiting", alias = "total_waiting")]
|
||||
pub total_waiting: u32,
|
||||
#[serde(rename = "totalErrsAvailability", alias = "total_errors_availability")]
|
||||
pub total_errors_availability: u64,
|
||||
#[serde(rename = "totalErrsTimeout", alias = "total_errors_timeout")]
|
||||
pub total_errors_timeout: u64,
|
||||
#[serde(rename = "totalWrites", alias = "total_writes")]
|
||||
pub total_writes: u64,
|
||||
#[serde(rename = "totalDeletes", alias = "total_deletes")]
|
||||
pub total_deletes: u64,
|
||||
}
|
||||
|
||||
impl Serialize for DiskMetrics {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
let mut state = serializer.serialize_struct("DiskMetrics", 7)?;
|
||||
state.serialize_field("last_minute", &self.last_minute)?;
|
||||
state.serialize_field("api_calls", &self.api_calls)?;
|
||||
state.serialize_field("total_waiting", &self.total_waiting)?;
|
||||
state.serialize_field("total_errors_availability", &self.total_errors_availability)?;
|
||||
state.serialize_field("total_errors_timeout", &self.total_errors_timeout)?;
|
||||
state.serialize_field("total_writes", &self.total_writes)?;
|
||||
state.serialize_field("total_deletes", &self.total_deletes)?;
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Default, Clone)]
|
||||
pub struct Disk {
|
||||
pub endpoint: String,
|
||||
@@ -457,17 +432,6 @@ mod tests {
|
||||
disk_index: i32,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct LegacyDiskMetricsCompat {
|
||||
last_minute: HashMap<String, TimedAction>,
|
||||
api_calls: HashMap<String, u64>,
|
||||
total_waiting: u32,
|
||||
total_errors_availability: u64,
|
||||
total_errors_timeout: u64,
|
||||
total_writes: u64,
|
||||
total_deletes: u64,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_item_state_to_string() {
|
||||
assert_eq!(ItemState::Offline.to_string(), ITEM_OFFLINE);
|
||||
@@ -531,56 +495,6 @@ mod tests {
|
||||
assert_eq!(metrics.total_deletes, 50);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_disk_metrics_json_preserves_internode_legacy_fields() {
|
||||
let metrics = DiskMetrics {
|
||||
total_waiting: 5,
|
||||
total_errors_availability: 2,
|
||||
total_errors_timeout: 1,
|
||||
total_writes: 1000,
|
||||
total_deletes: 50,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let json = serde_json::to_value(metrics).expect("disk metrics should serialize");
|
||||
|
||||
assert!(json.get("last_minute").is_some());
|
||||
assert!(json.get("api_calls").is_some());
|
||||
assert_eq!(json["total_waiting"], serde_json::json!(5));
|
||||
assert_eq!(json["total_errors_availability"], serde_json::json!(2));
|
||||
assert_eq!(json["total_errors_timeout"], serde_json::json!(1));
|
||||
assert_eq!(json["total_writes"], serde_json::json!(1000));
|
||||
assert_eq!(json["total_deletes"], serde_json::json!(50));
|
||||
assert!(json.get("lastMinute").is_none());
|
||||
assert!(json.get("totalErrsTimeout").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_disk_metrics_msgpack_uses_internode_legacy_fields() {
|
||||
let metrics = DiskMetrics {
|
||||
total_waiting: 5,
|
||||
total_errors_availability: 2,
|
||||
total_errors_timeout: 1,
|
||||
total_writes: 1000,
|
||||
total_deletes: 50,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut encoded = Vec::new();
|
||||
metrics
|
||||
.serialize(&mut Serializer::new(&mut encoded).with_struct_map())
|
||||
.expect("disk metrics should encode as named msgpack");
|
||||
|
||||
let decoded: LegacyDiskMetricsCompat = rmp_serde::from_slice(&encoded).expect("legacy disk metrics should decode");
|
||||
assert!(decoded.last_minute.is_empty());
|
||||
assert!(decoded.api_calls.is_empty());
|
||||
assert_eq!(decoded.total_waiting, 5);
|
||||
assert_eq!(decoded.total_errors_availability, 2);
|
||||
assert_eq!(decoded.total_errors_timeout, 1);
|
||||
assert_eq!(decoded.total_writes, 1000);
|
||||
assert_eq!(decoded.total_deletes, 50);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_disk_default() {
|
||||
let disk = Disk::default();
|
||||
|
||||
@@ -41,13 +41,6 @@ pub struct AuditTargetStats {
|
||||
pub total_messages: u64,
|
||||
}
|
||||
|
||||
/// Audit target statistics with runtime-local node identity.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct AuditTargetRuntimeStats {
|
||||
pub(crate) server: String,
|
||||
pub(crate) target: AuditTargetStats,
|
||||
}
|
||||
|
||||
/// Collects audit metrics from the provided audit target statistics.
|
||||
///
|
||||
/// Uses the metric descriptors from `metrics_type::audit` module.
|
||||
@@ -63,53 +56,19 @@ pub fn collect_audit_metrics(stats: &[AuditTargetStats]) -> Vec<PrometheusMetric
|
||||
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_MD, stat.failed_messages as f64)
|
||||
.with_label(TARGET_ID, target_id_label.clone()),
|
||||
.with_label("target_id", target_id_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&AUDIT_FAILED_STORE_LENGTH_MD, stat.failed_store_length as f64)
|
||||
.with_label(TARGET_ID, target_id_label.clone()),
|
||||
.with_label("target_id", target_id_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_MD, stat.queue_length as f64)
|
||||
.with_label(TARGET_ID, target_id_label.clone()),
|
||||
.with_label("target_id", target_id_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&AUDIT_TOTAL_MESSAGES_MD, stat.total_messages as f64)
|
||||
.with_label(TARGET_ID, target_id_label),
|
||||
);
|
||||
}
|
||||
|
||||
metrics
|
||||
}
|
||||
|
||||
pub(crate) fn collect_audit_runtime_metrics(stats: &[AuditTargetRuntimeStats]) -> Vec<PrometheusMetric> {
|
||||
let legacy_stats = stats.iter().map(|stat| stat.target.clone()).collect::<Vec<_>>();
|
||||
let mut metrics = collect_audit_metrics(&legacy_stats);
|
||||
metrics.reserve(stats.len() * 4);
|
||||
|
||||
for stat in stats.iter().filter(|stat| !stat.server.is_empty()) {
|
||||
let server_label: Cow<'static, str> = Cow::Owned(stat.server.clone());
|
||||
let target_id_label: Cow<'static, str> = Cow::Owned(stat.target.target_id.clone());
|
||||
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&AUDIT_FAILED_MESSAGES_BY_SERVER_MD, stat.target.failed_messages as f64)
|
||||
.with_label(SERVER, server_label.clone())
|
||||
.with_label(TARGET_ID, target_id_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&AUDIT_FAILED_STORE_LENGTH_BY_SERVER_MD, stat.target.failed_store_length as f64)
|
||||
.with_label(SERVER, server_label.clone())
|
||||
.with_label(TARGET_ID, target_id_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD, stat.target.queue_length as f64)
|
||||
.with_label(SERVER, server_label.clone())
|
||||
.with_label(TARGET_ID, target_id_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&AUDIT_TOTAL_MESSAGES_BY_SERVER_MD, stat.target.total_messages as f64)
|
||||
.with_label(SERVER, server_label)
|
||||
.with_label(TARGET_ID, target_id_label),
|
||||
.with_label("target_id", target_id_label),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -142,7 +101,7 @@ mod tests {
|
||||
|
||||
let metrics = collect_audit_metrics(&stats);
|
||||
|
||||
assert_eq!(metrics.len(), 8);
|
||||
assert_eq!(metrics.len(), 8); // 2 targets * 4 metrics each
|
||||
|
||||
let failed = metrics
|
||||
.iter()
|
||||
@@ -155,37 +114,6 @@ mod tests {
|
||||
&& m.labels.iter().any(|(k, v)| *k == "target_id" && v == "target-1")
|
||||
});
|
||||
assert!(failed_store.is_some());
|
||||
|
||||
assert!(
|
||||
metrics
|
||||
.iter()
|
||||
.all(|m| m.name != AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_audit_runtime_metrics_adds_server_dimensions() {
|
||||
let stats = vec![AuditTargetRuntimeStats {
|
||||
server: "node1:9000".to_string(),
|
||||
target: AuditTargetStats {
|
||||
target_id: "target-1".to_string(),
|
||||
failed_messages: 5,
|
||||
failed_store_length: 3,
|
||||
queue_length: 10,
|
||||
total_messages: 1000,
|
||||
},
|
||||
}];
|
||||
|
||||
let metrics = collect_audit_runtime_metrics(&stats);
|
||||
|
||||
assert_eq!(metrics.len(), 8);
|
||||
let server_queue = metrics.iter().find(|m| {
|
||||
m.value == 10.0
|
||||
&& m.name == AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name()
|
||||
&& m.labels.iter().any(|(k, v)| *k == SERVER && v == "node1:9000")
|
||||
&& m.labels.iter().any(|(k, v)| *k == TARGET_ID && v == "target-1")
|
||||
});
|
||||
assert!(server_queue.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -30,18 +30,14 @@ use crate::metrics::schema::bucket_replication::{
|
||||
BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD, BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD, BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD,
|
||||
BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, BUCKET_REPL_RESYNC_FAILED_TOTAL_MD, BUCKET_REPL_RESYNC_STARTED_TOTAL_MD,
|
||||
BUCKET_REPL_SENT_BYTES_MD, BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD,
|
||||
BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD, BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD,
|
||||
BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD, BUCKET_REPL_TARGET_SENT_BYTES_MD, BUCKET_REPL_TARGET_SENT_COUNT_MD,
|
||||
BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD,
|
||||
BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, RESULT_L, TARGET_ARN_L,
|
||||
BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD, BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD,
|
||||
BUCKET_REPL_RESYNC_FAILED_TOTAL_MD, BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD,
|
||||
BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L,
|
||||
TARGET_ARN_L,
|
||||
};
|
||||
use std::borrow::Cow;
|
||||
|
||||
const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 37;
|
||||
const BUCKET_REPLICATION_RUNTIME_FLOW_METRICS_PER_TARGET: usize = 8;
|
||||
const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 25;
|
||||
const BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET: usize = 11;
|
||||
const BUCKET_REPLICATION_BACKLOG_METRICS_PER_TARGET: usize = 4;
|
||||
|
||||
@@ -53,19 +49,6 @@ pub struct BucketReplicationTargetStats {
|
||||
pub latency_ms: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct BucketReplicationTargetFlowStats {
|
||||
pub(crate) target_arn: String,
|
||||
pub sent_bytes: u64,
|
||||
pub sent_count: u64,
|
||||
pub total_failed_bytes: u64,
|
||||
pub total_failed_count: u64,
|
||||
pub last_min_failed_bytes: u64,
|
||||
pub last_min_failed_count: u64,
|
||||
pub last_hour_failed_bytes: u64,
|
||||
pub last_hour_failed_count: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct BucketReplicationBandwidthStats {
|
||||
pub bucket: String,
|
||||
@@ -105,12 +88,6 @@ pub struct BucketReplicationStats {
|
||||
pub targets: Vec<BucketReplicationTargetStats>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct BucketReplicationRuntimeStats {
|
||||
pub(crate) stats: BucketReplicationStats,
|
||||
pub(crate) target_flows: Vec<BucketReplicationTargetFlowStats>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct BucketReplicationBacklogStats {
|
||||
pub(crate) bucket: String,
|
||||
@@ -163,25 +140,6 @@ pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBa
|
||||
metrics
|
||||
}
|
||||
|
||||
fn push_proxy_request_result_metrics(
|
||||
metrics: &mut Vec<PrometheusMetric>,
|
||||
bucket_label: Cow<'static, str>,
|
||||
operation: &'static str,
|
||||
total: u64,
|
||||
failures: u64,
|
||||
) {
|
||||
let failure_count = failures.min(total);
|
||||
let success_count = total.saturating_sub(failure_count);
|
||||
for (result, value) in [("success", success_count), ("failure", failure_count)] {
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD, value as f64)
|
||||
.with_label(BUCKET_L, bucket_label.clone())
|
||||
.with_label(OPERATION_L, operation)
|
||||
.with_label(RESULT_L, result),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> Vec<PrometheusMetric> {
|
||||
if stats.is_empty() {
|
||||
return Vec::new();
|
||||
@@ -305,48 +263,6 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
|
||||
)
|
||||
.with_label(BUCKET_L, bucket_label.clone()),
|
||||
);
|
||||
push_proxy_request_result_metrics(
|
||||
&mut metrics,
|
||||
bucket_label.clone(),
|
||||
"get",
|
||||
stat.proxied_get_requests_total,
|
||||
stat.proxied_get_requests_failures,
|
||||
);
|
||||
push_proxy_request_result_metrics(
|
||||
&mut metrics,
|
||||
bucket_label.clone(),
|
||||
"head",
|
||||
stat.proxied_head_requests_total,
|
||||
stat.proxied_head_requests_failures,
|
||||
);
|
||||
push_proxy_request_result_metrics(
|
||||
&mut metrics,
|
||||
bucket_label.clone(),
|
||||
"put",
|
||||
stat.proxied_put_requests_total,
|
||||
stat.proxied_put_requests_failures,
|
||||
);
|
||||
push_proxy_request_result_metrics(
|
||||
&mut metrics,
|
||||
bucket_label.clone(),
|
||||
"put_tagging",
|
||||
stat.proxied_put_tagging_requests_total,
|
||||
stat.proxied_put_tagging_requests_failures,
|
||||
);
|
||||
push_proxy_request_result_metrics(
|
||||
&mut metrics,
|
||||
bucket_label.clone(),
|
||||
"get_tagging",
|
||||
stat.proxied_get_tagging_requests_total,
|
||||
stat.proxied_get_tagging_requests_failures,
|
||||
);
|
||||
push_proxy_request_result_metrics(
|
||||
&mut metrics,
|
||||
bucket_label.clone(),
|
||||
"delete_tagging",
|
||||
stat.proxied_delete_tagging_requests_total,
|
||||
stat.proxied_delete_tagging_requests_failures,
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, stat.resync_started_count as f64)
|
||||
.with_label(BUCKET_L, bucket_label.clone()),
|
||||
@@ -382,81 +298,6 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
|
||||
metrics
|
||||
}
|
||||
|
||||
pub(crate) fn collect_bucket_replication_runtime_metrics(stats: &[BucketReplicationRuntimeStats]) -> Vec<PrometheusMetric> {
|
||||
if stats.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let legacy_stats = stats.iter().map(|stat| stat.stats.clone()).collect::<Vec<_>>();
|
||||
let mut metrics = collect_bucket_replication_metrics(&legacy_stats);
|
||||
let flow_count = stats
|
||||
.iter()
|
||||
.map(|stat| stat.target_flows.len() * BUCKET_REPLICATION_RUNTIME_FLOW_METRICS_PER_TARGET)
|
||||
.sum();
|
||||
metrics.reserve(flow_count);
|
||||
|
||||
for stat in stats {
|
||||
let bucket_label: Cow<'static, str> = Cow::Owned(stat.stats.bucket.clone());
|
||||
for target in &stat.target_flows {
|
||||
let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_SENT_BYTES_MD, target.sent_bytes as f64)
|
||||
.with_label(BUCKET_L, bucket_label.clone())
|
||||
.with_label(TARGET_ARN_L, target_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_SENT_COUNT_MD, target.sent_count as f64)
|
||||
.with_label(BUCKET_L, bucket_label.clone())
|
||||
.with_label(TARGET_ARN_L, target_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD, target.total_failed_bytes as f64)
|
||||
.with_label(BUCKET_L, bucket_label.clone())
|
||||
.with_label(TARGET_ARN_L, target_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD, target.total_failed_count as f64)
|
||||
.with_label(BUCKET_L, bucket_label.clone())
|
||||
.with_label(TARGET_ARN_L, target_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(
|
||||
&BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD,
|
||||
target.last_min_failed_bytes as f64,
|
||||
)
|
||||
.with_label(BUCKET_L, bucket_label.clone())
|
||||
.with_label(TARGET_ARN_L, target_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(
|
||||
&BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD,
|
||||
target.last_min_failed_count as f64,
|
||||
)
|
||||
.with_label(BUCKET_L, bucket_label.clone())
|
||||
.with_label(TARGET_ARN_L, target_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(
|
||||
&BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD,
|
||||
target.last_hour_failed_bytes as f64,
|
||||
)
|
||||
.with_label(BUCKET_L, bucket_label.clone())
|
||||
.with_label(TARGET_ARN_L, target_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(
|
||||
&BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD,
|
||||
target.last_hour_failed_count as f64,
|
||||
)
|
||||
.with_label(BUCKET_L, bucket_label.clone())
|
||||
.with_label(TARGET_ARN_L, target_label.clone()),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
metrics
|
||||
}
|
||||
|
||||
pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicationBacklogStats]) -> Vec<PrometheusMetric> {
|
||||
if stats.is_empty() {
|
||||
return Vec::new();
|
||||
@@ -571,56 +412,43 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_collect_bucket_replication_metrics() {
|
||||
let stats = vec![BucketReplicationRuntimeStats {
|
||||
stats: BucketReplicationStats {
|
||||
bucket: "b1".to_string(),
|
||||
total_failed_bytes: 64,
|
||||
total_failed_count: 2,
|
||||
last_min_failed_bytes: 32,
|
||||
last_min_failed_count: 1,
|
||||
last_hour_failed_bytes: 64,
|
||||
last_hour_failed_count: 2,
|
||||
sent_bytes: 1024,
|
||||
sent_count: 8,
|
||||
proxied_get_requests_total: 5,
|
||||
proxied_get_requests_failures: 1,
|
||||
proxied_head_requests_total: 4,
|
||||
proxied_head_requests_failures: 0,
|
||||
proxied_put_requests_total: 6,
|
||||
proxied_put_requests_failures: 2,
|
||||
proxied_put_tagging_requests_total: 3,
|
||||
proxied_put_tagging_requests_failures: 1,
|
||||
proxied_get_tagging_requests_total: 2,
|
||||
proxied_get_tagging_requests_failures: 0,
|
||||
proxied_delete_tagging_requests_total: 1,
|
||||
proxied_delete_tagging_requests_failures: 1,
|
||||
resync_started_count: 2,
|
||||
resync_completed_count: 1,
|
||||
resync_failed_count: 1,
|
||||
resync_canceled_count: 0,
|
||||
resync_duration_ms: 1500,
|
||||
targets: vec![BucketReplicationTargetStats {
|
||||
target_arn: "arn:rustfs:replication:us-east-1:1:target".to_string(),
|
||||
bandwidth_limit_bytes_per_sec: 2048,
|
||||
current_bandwidth_bytes_per_sec: 1024.0,
|
||||
latency_ms: 15.0,
|
||||
}],
|
||||
},
|
||||
target_flows: vec![BucketReplicationTargetFlowStats {
|
||||
let stats = vec![BucketReplicationStats {
|
||||
bucket: "b1".to_string(),
|
||||
total_failed_bytes: 64,
|
||||
total_failed_count: 2,
|
||||
last_min_failed_bytes: 32,
|
||||
last_min_failed_count: 1,
|
||||
last_hour_failed_bytes: 64,
|
||||
last_hour_failed_count: 2,
|
||||
sent_bytes: 1024,
|
||||
sent_count: 8,
|
||||
proxied_get_requests_total: 5,
|
||||
proxied_get_requests_failures: 1,
|
||||
proxied_head_requests_total: 4,
|
||||
proxied_head_requests_failures: 0,
|
||||
proxied_put_requests_total: 6,
|
||||
proxied_put_requests_failures: 2,
|
||||
proxied_put_tagging_requests_total: 3,
|
||||
proxied_put_tagging_requests_failures: 1,
|
||||
proxied_get_tagging_requests_total: 2,
|
||||
proxied_get_tagging_requests_failures: 0,
|
||||
proxied_delete_tagging_requests_total: 1,
|
||||
proxied_delete_tagging_requests_failures: 1,
|
||||
resync_started_count: 2,
|
||||
resync_completed_count: 1,
|
||||
resync_failed_count: 1,
|
||||
resync_canceled_count: 0,
|
||||
resync_duration_ms: 1500,
|
||||
targets: vec![BucketReplicationTargetStats {
|
||||
target_arn: "arn:rustfs:replication:us-east-1:1:target".to_string(),
|
||||
sent_bytes: 512,
|
||||
sent_count: 4,
|
||||
total_failed_bytes: 96,
|
||||
total_failed_count: 3,
|
||||
last_min_failed_bytes: 32,
|
||||
last_min_failed_count: 1,
|
||||
last_hour_failed_bytes: 64,
|
||||
last_hour_failed_count: 2,
|
||||
bandwidth_limit_bytes_per_sec: 2048,
|
||||
current_bandwidth_bytes_per_sec: 1024.0,
|
||||
latency_ms: 15.0,
|
||||
}],
|
||||
}];
|
||||
|
||||
let metrics = collect_bucket_replication_runtime_metrics(&stats);
|
||||
assert_eq!(metrics.len(), 46);
|
||||
let metrics = collect_bucket_replication_metrics(&stats);
|
||||
assert_eq!(metrics.len(), 26);
|
||||
|
||||
let sent_name = BUCKET_REPL_SENT_COUNT_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
@@ -643,28 +471,6 @@ mod tests {
|
||||
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
|
||||
}));
|
||||
|
||||
let proxy_requests_name = BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == proxy_requests_name
|
||||
&& metric.value == 4.0
|
||||
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
|
||||
&& metric.labels.iter().any(|(key, value)| *key == OPERATION_L && value == "put")
|
||||
&& metric
|
||||
.labels
|
||||
.iter()
|
||||
.any(|(key, value)| *key == RESULT_L && value == "success")
|
||||
}));
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == proxy_requests_name
|
||||
&& metric.value == 2.0
|
||||
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
|
||||
&& metric.labels.iter().any(|(key, value)| *key == OPERATION_L && value == "put")
|
||||
&& metric
|
||||
.labels
|
||||
.iter()
|
||||
.any(|(key, value)| *key == RESULT_L && value == "failure")
|
||||
}));
|
||||
|
||||
let latency_name = BUCKET_REPL_LATENCY_MS_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == latency_name
|
||||
@@ -675,27 +481,6 @@ mod tests {
|
||||
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target")
|
||||
}));
|
||||
|
||||
let target_sent_name = BUCKET_REPL_TARGET_SENT_COUNT_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == target_sent_name
|
||||
&& metric.value == 4.0
|
||||
&& metric
|
||||
.labels
|
||||
.iter()
|
||||
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target")
|
||||
}));
|
||||
|
||||
let target_last_min_failed_name = BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == target_last_min_failed_name
|
||||
&& metric.value == 32.0
|
||||
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
|
||||
&& metric
|
||||
.labels
|
||||
.iter()
|
||||
.any(|(key, value)| *key == TARGET_ARN_L && value == "arn:rustfs:replication:us-east-1:1:target")
|
||||
}));
|
||||
|
||||
let delete_tagging_total_name = BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == delete_tagging_total_name
|
||||
|
||||
@@ -48,26 +48,6 @@ pub struct IlmStats {
|
||||
pub versions_scanned: u64,
|
||||
}
|
||||
|
||||
/// ILM task metrics by action and state.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct IlmActionTaskStats {
|
||||
pub(crate) action: String,
|
||||
pub(crate) state: String,
|
||||
pub(crate) value: u64,
|
||||
}
|
||||
|
||||
/// ILM statistics with runtime-local node identity and bounded action/state details.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct IlmRuntimeStats {
|
||||
pub(crate) server: String,
|
||||
pub(crate) stats: IlmStats,
|
||||
pub(crate) action_tasks: Vec<IlmActionTaskStats>,
|
||||
}
|
||||
|
||||
fn is_live_action_task_state(state: &str) -> bool {
|
||||
matches!(state, "pending" | "active" | "compensation_running")
|
||||
}
|
||||
|
||||
/// Collects ILM metrics from the given stats.
|
||||
///
|
||||
/// Uses the metric descriptors from `metrics_type::ilm` module.
|
||||
@@ -98,25 +78,6 @@ pub fn collect_ilm_metrics(stats: &IlmStats) -> Vec<PrometheusMetric> {
|
||||
]
|
||||
}
|
||||
|
||||
pub(crate) fn collect_ilm_runtime_metrics(stats: &IlmRuntimeStats) -> Vec<PrometheusMetric> {
|
||||
let mut metrics = collect_ilm_metrics(&stats.stats);
|
||||
|
||||
metrics.extend(
|
||||
stats
|
||||
.action_tasks
|
||||
.iter()
|
||||
.filter(|task| is_live_action_task_state(&task.state))
|
||||
.map(|task| {
|
||||
PrometheusMetric::from_descriptor(&ILM_ACTION_TASKS_MD, task.value as f64)
|
||||
.with_label_owned(SERVER_LABEL, stats.server.clone())
|
||||
.with_label_owned(ACTION_LABEL, task.action.clone())
|
||||
.with_label_owned(STATE_LABEL, task.state.clone())
|
||||
}),
|
||||
);
|
||||
|
||||
metrics
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -134,62 +95,16 @@ mod tests {
|
||||
transition_compensation_running_tasks: 1,
|
||||
versions_scanned: 1000000,
|
||||
};
|
||||
let runtime_stats = IlmRuntimeStats {
|
||||
server: "node1:9000".to_string(),
|
||||
stats,
|
||||
action_tasks: vec![
|
||||
IlmActionTaskStats {
|
||||
action: "expiry".to_string(),
|
||||
state: "pending".to_string(),
|
||||
value: 100,
|
||||
},
|
||||
IlmActionTaskStats {
|
||||
action: "transition".to_string(),
|
||||
state: "queue_send_timeout".to_string(),
|
||||
value: 3,
|
||||
},
|
||||
IlmActionTaskStats {
|
||||
action: "transition".to_string(),
|
||||
state: "active".to_string(),
|
||||
value: 5,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
let metrics = collect_ilm_runtime_metrics(&runtime_stats);
|
||||
let metrics = collect_ilm_metrics(&stats);
|
||||
|
||||
assert_eq!(metrics.len(), 11);
|
||||
assert_eq!(metrics.len(), 9);
|
||||
|
||||
let pending = metrics.iter().find(|m| m.value == 100.0);
|
||||
assert!(pending.is_some());
|
||||
|
||||
let scanned = metrics.iter().find(|m| m.value == 1000000.0);
|
||||
assert!(scanned.is_some());
|
||||
|
||||
let transition_timeout = metrics.iter().find(|m| {
|
||||
m.name == ILM_ACTION_TASKS_MD.get_full_metric_name()
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000")
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == ACTION_LABEL && value.as_ref() == "transition")
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "queue_send_timeout")
|
||||
});
|
||||
assert!(transition_timeout.is_none());
|
||||
|
||||
let transition_active = metrics.iter().find(|m| {
|
||||
m.name == ILM_ACTION_TASKS_MD.get_full_metric_name()
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == ACTION_LABEL && value.as_ref() == "transition")
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "active")
|
||||
});
|
||||
assert_eq!(transition_active.map(|m| m.value), Some(5.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -40,12 +40,10 @@ pub mod system_network;
|
||||
pub mod system_network_host;
|
||||
pub mod system_process;
|
||||
|
||||
pub(crate) use audit::{AuditTargetRuntimeStats, collect_audit_runtime_metrics};
|
||||
pub use audit::{AuditTargetStats, collect_audit_metrics};
|
||||
pub use bucket::{BucketStats, collect_bucket_metrics};
|
||||
pub(crate) use bucket_replication::{
|
||||
BucketReplicationBacklogStats, BucketReplicationRuntimeStats, BucketReplicationTargetBacklogStats,
|
||||
BucketReplicationTargetFlowStats, collect_bucket_replication_backlog_metrics, collect_bucket_replication_runtime_metrics,
|
||||
BucketReplicationBacklogStats, BucketReplicationTargetBacklogStats, collect_bucket_replication_backlog_metrics,
|
||||
};
|
||||
pub use bucket_replication::{
|
||||
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
|
||||
@@ -59,24 +57,18 @@ pub use cluster_iam::{IamStats, collect_iam_metrics};
|
||||
pub use cluster_usage::{BucketUsageStats, ClusterUsageStats, collect_bucket_usage_metrics, collect_cluster_usage_metrics};
|
||||
pub use compression::{CompressionClusterStats, collect_compression_cluster_metrics};
|
||||
pub use dial9::{Dial9Stats, collect_current_dial9_metrics, collect_dial9_metrics, is_dial9_enabled};
|
||||
pub(crate) use ilm::{IlmActionTaskStats, IlmRuntimeStats, collect_ilm_runtime_metrics};
|
||||
pub use ilm::{IlmStats, collect_ilm_metrics};
|
||||
pub use node::{DiskStats, collect_node_metrics};
|
||||
pub use notification::{NotificationStats, collect_notification_metrics};
|
||||
pub(crate) use notification_target::{NotificationTargetRuntimeStats, collect_notification_target_runtime_metrics};
|
||||
pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics};
|
||||
pub(crate) use replication::{ReplicationRuntimeStats, collect_replication_runtime_metrics};
|
||||
pub use replication::{ReplicationStats, collect_replication_metrics};
|
||||
pub(crate) use request::{ApiRequestMetricSupport, ApiRequestStats, collect_request_metrics};
|
||||
pub use resource::{ResourceStats, collect_resource_metrics};
|
||||
pub(crate) use scanner::{ScannerRuntimeStats, collect_scanner_runtime_metrics};
|
||||
pub use scanner::{ScannerStats, collect_scanner_metrics};
|
||||
pub use system_cpu::{CpuStats, ProcessCpuStats, collect_cpu_metrics, collect_process_cpu_metrics};
|
||||
pub use system_drive::{
|
||||
DriveCountStats, DriveDetailedStats, ProcessDiskStats, collect_drive_count_metrics, collect_drive_detailed_metrics,
|
||||
collect_process_disk_metrics,
|
||||
};
|
||||
pub(crate) use system_drive::{DriveRuntimeDetailedStats, collect_drive_runtime_detailed_metrics};
|
||||
#[cfg(feature = "gpu")]
|
||||
pub use system_gpu::{GpuCollector, GpuError, GpuStats, collect_gpu_metrics};
|
||||
pub use system_memory::{MemoryStats, ProcessMemoryStats, collect_memory_metrics, collect_process_memory_metrics};
|
||||
|
||||
@@ -16,10 +16,8 @@
|
||||
|
||||
use crate::metrics::report::PrometheusMetric;
|
||||
use crate::metrics::schema::notification_target::{
|
||||
NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD, NOTIFICATION_TARGET_FAILED_MESSAGES_MD,
|
||||
NOTIFICATION_TARGET_FAILED_STORE_LENGTH_BY_SERVER_MD, NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD,
|
||||
NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD,
|
||||
NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD, NOTIFICATION_TARGET_TOTAL_MESSAGES_MD, SERVER, TARGET_ID, TARGET_TYPE,
|
||||
NOTIFICATION_TARGET_FAILED_MESSAGES_MD, NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD, NOTIFICATION_TARGET_QUEUE_LENGTH_MD,
|
||||
NOTIFICATION_TARGET_TOTAL_MESSAGES_MD, TARGET_ID, TARGET_TYPE,
|
||||
};
|
||||
use std::borrow::Cow;
|
||||
|
||||
@@ -33,18 +31,12 @@ pub struct NotificationTargetStats {
|
||||
pub total_messages: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct NotificationTargetRuntimeStats {
|
||||
pub(crate) server: String,
|
||||
pub(crate) target: NotificationTargetStats,
|
||||
}
|
||||
|
||||
pub fn collect_notification_target_metrics(stats: &[NotificationTargetStats]) -> Vec<PrometheusMetric> {
|
||||
if stats.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut metrics = Vec::with_capacity(stats.len() * 8);
|
||||
let mut metrics = Vec::with_capacity(stats.len() * 4);
|
||||
for stat in stats {
|
||||
let target_id: Cow<'static, str> = Cow::Owned(stat.target_id.clone());
|
||||
let target_type: Cow<'static, str> = Cow::Owned(stat.target_type.clone());
|
||||
@@ -74,57 +66,6 @@ pub fn collect_notification_target_metrics(stats: &[NotificationTargetStats]) ->
|
||||
metrics
|
||||
}
|
||||
|
||||
pub(crate) fn collect_notification_target_runtime_metrics(stats: &[NotificationTargetRuntimeStats]) -> Vec<PrometheusMetric> {
|
||||
if stats.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let legacy_stats = stats.iter().map(|stat| stat.target.clone()).collect::<Vec<_>>();
|
||||
let mut metrics = collect_notification_target_metrics(&legacy_stats);
|
||||
metrics.reserve(stats.len() * 4);
|
||||
for stat in stats {
|
||||
let server: Cow<'static, str> = Cow::Owned(stat.server.clone());
|
||||
let target_id: Cow<'static, str> = Cow::Owned(stat.target.target_id.clone());
|
||||
let target_type: Cow<'static, str> = Cow::Owned(stat.target.target_type.clone());
|
||||
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(
|
||||
&NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD,
|
||||
stat.target.failed_messages as f64,
|
||||
)
|
||||
.with_label(SERVER, server.clone())
|
||||
.with_label(TARGET_ID, target_id.clone())
|
||||
.with_label(TARGET_TYPE, target_type.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(
|
||||
&NOTIFICATION_TARGET_FAILED_STORE_LENGTH_BY_SERVER_MD,
|
||||
stat.target.failed_store_length as f64,
|
||||
)
|
||||
.with_label(SERVER, server.clone())
|
||||
.with_label(TARGET_ID, target_id.clone())
|
||||
.with_label(TARGET_TYPE, target_type.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD, stat.target.queue_length as f64)
|
||||
.with_label(SERVER, server.clone())
|
||||
.with_label(TARGET_ID, target_id.clone())
|
||||
.with_label(TARGET_TYPE, target_type.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(
|
||||
&NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD,
|
||||
stat.target.total_messages as f64,
|
||||
)
|
||||
.with_label(SERVER, server)
|
||||
.with_label(TARGET_ID, target_id)
|
||||
.with_label(TARGET_TYPE, target_type),
|
||||
);
|
||||
}
|
||||
|
||||
metrics
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -132,7 +73,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_collect_notification_target_metrics() {
|
||||
let stats = [NotificationTargetStats {
|
||||
let stats = vec![NotificationTargetStats {
|
||||
failed_messages: 2,
|
||||
failed_store_length: 3,
|
||||
queue_length: 4,
|
||||
@@ -141,12 +82,9 @@ mod tests {
|
||||
total_messages: 42,
|
||||
}];
|
||||
|
||||
let metrics = collect_notification_target_runtime_metrics(&[NotificationTargetRuntimeStats {
|
||||
server: "node1:9000".to_string(),
|
||||
target: stats[0].clone(),
|
||||
}]);
|
||||
let metrics = collect_notification_target_metrics(&stats);
|
||||
|
||||
assert_eq!(metrics.len(), 8);
|
||||
assert_eq!(metrics.len(), 4);
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.value == 3.0
|
||||
&& metric.name == NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD.get_full_metric_name()
|
||||
@@ -166,22 +104,6 @@ mod tests {
|
||||
.iter()
|
||||
.any(|(key, value)| *key == TARGET_TYPE && value == "webhook")
|
||||
}));
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.value == 4.0
|
||||
&& metric.name == NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD.get_full_metric_name()
|
||||
&& metric
|
||||
.labels
|
||||
.iter()
|
||||
.any(|(key, value)| *key == SERVER && value == "node1:9000")
|
||||
&& metric
|
||||
.labels
|
||||
.iter()
|
||||
.any(|(key, value)| *key == TARGET_ID && value == "primary:webhook")
|
||||
&& metric
|
||||
.labels
|
||||
.iter()
|
||||
.any(|(key, value)| *key == TARGET_TYPE && value == "webhook")
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -191,18 +113,4 @@ mod tests {
|
||||
assert_eq!(NOTIFICATION_TARGET_QUEUE_LENGTH_MD.metric_type, MetricType::Gauge);
|
||||
assert_eq!(NOTIFICATION_TARGET_TOTAL_MESSAGES_MD.metric_type, MetricType::Gauge);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notification_target_stats_struct_literal_keeps_legacy_fields() {
|
||||
let stats = vec![NotificationTargetStats {
|
||||
failed_messages: 2,
|
||||
failed_store_length: 3,
|
||||
queue_length: 4,
|
||||
target_id: "primary:webhook".to_string(),
|
||||
target_type: "webhook".to_string(),
|
||||
total_messages: 42,
|
||||
}];
|
||||
|
||||
assert_eq!(collect_notification_target_metrics(&stats).len(), 4);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,12 +53,6 @@ pub struct ReplicationStats {
|
||||
pub recent_backlog_count: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct ReplicationRuntimeStats {
|
||||
pub(crate) server: String,
|
||||
pub(crate) stats: ReplicationStats,
|
||||
}
|
||||
|
||||
/// Collects replication metrics from the given stats.
|
||||
///
|
||||
/// Returns a vector of Prometheus metrics for replication statistics.
|
||||
@@ -80,41 +74,6 @@ pub fn collect_replication_metrics(stats: &ReplicationStats) -> Vec<PrometheusMe
|
||||
]
|
||||
}
|
||||
|
||||
pub(crate) fn collect_replication_runtime_metrics(runtime: &ReplicationRuntimeStats) -> Vec<PrometheusMetric> {
|
||||
let stats = &runtime.stats;
|
||||
let mut metrics = collect_replication_metrics(stats);
|
||||
metrics.extend([
|
||||
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_ACTIVE_WORKERS_BY_SERVER_MD, stats.average_active_workers)
|
||||
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
|
||||
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_BYTES_BY_SERVER_MD, stats.average_queued_bytes as f64)
|
||||
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
|
||||
PrometheusMetric::from_descriptor(&REPLICATION_AVERAGE_QUEUED_COUNT_BY_SERVER_MD, stats.average_queued_count as f64)
|
||||
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
|
||||
PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_ACTIVE_WORKERS_BY_SERVER_MD, stats.active_workers as f64)
|
||||
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
|
||||
PrometheusMetric::from_descriptor(&REPLICATION_CURRENT_DATA_TRANSFER_RATE_BY_SERVER_MD, stats.current_data_transfer_rate)
|
||||
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&REPLICATION_LAST_MINUTE_QUEUED_BYTES_BY_SERVER_MD,
|
||||
stats.last_minute_queued_bytes as f64,
|
||||
)
|
||||
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
|
||||
PrometheusMetric::from_descriptor(
|
||||
&REPLICATION_LAST_MINUTE_QUEUED_COUNT_BY_SERVER_MD,
|
||||
stats.last_minute_queued_count as f64,
|
||||
)
|
||||
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
|
||||
PrometheusMetric::from_descriptor(&REPLICATION_MAX_ACTIVE_WORKERS_BY_SERVER_MD, stats.max_active_workers as f64)
|
||||
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
|
||||
PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_BYTES_BY_SERVER_MD, stats.max_queued_bytes as f64)
|
||||
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
|
||||
PrometheusMetric::from_descriptor(&REPLICATION_MAX_QUEUED_COUNT_BY_SERVER_MD, stats.max_queued_count as f64)
|
||||
.with_label_owned(SERVER_LABEL, runtime.server.clone()),
|
||||
]);
|
||||
|
||||
metrics
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -138,13 +97,10 @@ mod tests {
|
||||
recent_backlog_count: 1500,
|
||||
};
|
||||
|
||||
let metrics = collect_replication_runtime_metrics(&ReplicationRuntimeStats {
|
||||
server: "node-a:9000".to_string(),
|
||||
stats,
|
||||
});
|
||||
let metrics = collect_replication_metrics(&stats);
|
||||
report_metrics(&metrics);
|
||||
|
||||
assert_eq!(metrics.len(), 23);
|
||||
assert_eq!(metrics.len(), 13);
|
||||
|
||||
// Verify active workers
|
||||
let active_name = REPLICATION_CURRENT_ACTIVE_WORKERS_MD.get_full_metric_name();
|
||||
@@ -155,31 +111,6 @@ mod tests {
|
||||
let avg_active_name = REPLICATION_AVERAGE_ACTIVE_WORKERS_MD.get_full_metric_name();
|
||||
let avg_active = metrics.iter().find(|m| m.name == avg_active_name);
|
||||
assert_eq!(avg_active.map(|m| m.value), Some(8.5));
|
||||
|
||||
let active_by_server_name = REPLICATION_CURRENT_ACTIVE_WORKERS_BY_SERVER_MD.get_full_metric_name();
|
||||
let active_by_server = metrics.iter().find(|m| m.name == active_by_server_name);
|
||||
assert_eq!(active_by_server.map(|m| m.value), Some(10.0));
|
||||
assert_eq!(
|
||||
active_by_server
|
||||
.and_then(|m| m.labels.iter().find(|(name, _)| *name == SERVER_LABEL))
|
||||
.map(|(_, value)| value.as_ref()),
|
||||
Some("node-a:9000")
|
||||
);
|
||||
assert!(
|
||||
metrics
|
||||
.iter()
|
||||
.all(|m| m.name != REPLICATION_AVERAGE_DATA_TRANSFER_RATE_BY_SERVER_MD.get_full_metric_name())
|
||||
);
|
||||
assert!(
|
||||
metrics
|
||||
.iter()
|
||||
.all(|m| m.name != REPLICATION_MAX_DATA_TRANSFER_RATE_BY_SERVER_MD.get_full_metric_name())
|
||||
);
|
||||
assert!(
|
||||
metrics
|
||||
.iter()
|
||||
.all(|m| m.name != REPLICATION_RECENT_BACKLOG_COUNT_BY_SERVER_MD.get_full_metric_name())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -193,25 +124,4 @@ mod tests {
|
||||
assert!(metric.labels.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_stats_struct_literal_keeps_legacy_fields() {
|
||||
let stats = ReplicationStats {
|
||||
average_active_workers: 1.0,
|
||||
average_queued_bytes: 2,
|
||||
average_queued_count: 3,
|
||||
average_data_transfer_rate: 4.0,
|
||||
active_workers: 5,
|
||||
current_data_transfer_rate: 6.0,
|
||||
last_minute_queued_bytes: 7,
|
||||
last_minute_queued_count: 8,
|
||||
max_active_workers: 9,
|
||||
max_queued_bytes: 10,
|
||||
max_queued_count: 11,
|
||||
max_data_transfer_rate: 12.0,
|
||||
recent_backlog_count: 13,
|
||||
};
|
||||
|
||||
assert_eq!(collect_replication_metrics(&stats).len(), 13);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,32 +23,9 @@ use crate::metrics::report::PrometheusMetric;
|
||||
use crate::metrics::schema::request::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
|
||||
pub(crate) struct ApiRequestMetricSupport {
|
||||
pub(crate) lifecycle: bool,
|
||||
pub(crate) traffic: bool,
|
||||
pub(crate) ttfb: bool,
|
||||
}
|
||||
|
||||
impl ApiRequestMetricSupport {
|
||||
pub(crate) const ALL: Self = Self {
|
||||
lifecycle: true,
|
||||
traffic: true,
|
||||
ttfb: true,
|
||||
};
|
||||
|
||||
pub(crate) const TOTALS_ONLY: Self = Self {
|
||||
lifecycle: false,
|
||||
traffic: false,
|
||||
ttfb: false,
|
||||
};
|
||||
}
|
||||
|
||||
/// API request statistics for a specific API endpoint.
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ApiRequestStats {
|
||||
/// Server identifier
|
||||
pub server: String,
|
||||
/// API name (e.g., "GetObject", "PutObject")
|
||||
pub name: String,
|
||||
/// Request type (e.g., "s3", "admin")
|
||||
@@ -71,27 +48,6 @@ pub struct ApiRequestStats {
|
||||
pub sent_bytes: u64,
|
||||
/// Bytes received
|
||||
pub recv_bytes: u64,
|
||||
pub(crate) supported_metrics: ApiRequestMetricSupport,
|
||||
}
|
||||
|
||||
impl Default for ApiRequestStats {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
server: String::new(),
|
||||
name: String::new(),
|
||||
req_type: String::new(),
|
||||
in_flight: 0,
|
||||
total: 0,
|
||||
errors_total: 0,
|
||||
errors_5xx: 0,
|
||||
errors_4xx: 0,
|
||||
canceled: 0,
|
||||
ttfb_distribution: Vec::new(),
|
||||
sent_bytes: 0,
|
||||
recv_bytes: 0,
|
||||
supported_metrics: ApiRequestMetricSupport::ALL,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Collects API request metrics from the given stats.
|
||||
@@ -100,119 +56,62 @@ impl Default for ApiRequestStats {
|
||||
pub fn collect_request_metrics(stats: &[ApiRequestStats]) -> Vec<PrometheusMetric> {
|
||||
let mut metrics = Vec::new();
|
||||
let mut traffic_by_type: HashMap<&str, (u64, u64)> = HashMap::with_capacity(stats.len());
|
||||
let mut traffic_by_server_type: HashMap<(&str, &str), (u64, u64)> = HashMap::with_capacity(stats.len());
|
||||
|
||||
for stat in stats {
|
||||
if stat.supported_metrics.traffic {
|
||||
let entry = traffic_by_type.entry(stat.req_type.as_str()).or_default();
|
||||
entry.0 = entry.0.saturating_add(stat.sent_bytes);
|
||||
entry.1 = entry.1.saturating_add(stat.recv_bytes);
|
||||
if !stat.server.is_empty() {
|
||||
let entry = traffic_by_server_type
|
||||
.entry((stat.server.as_str(), stat.req_type.as_str()))
|
||||
.or_default();
|
||||
entry.0 = entry.0.saturating_add(stat.sent_bytes);
|
||||
entry.1 = entry.1.saturating_add(stat.recv_bytes);
|
||||
}
|
||||
}
|
||||
let entry = traffic_by_type.entry(stat.req_type.as_str()).or_default();
|
||||
entry.0 = entry.0.saturating_add(stat.sent_bytes);
|
||||
entry.1 = entry.1.saturating_add(stat.recv_bytes);
|
||||
|
||||
// In-flight requests
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_MD, stat.in_flight as f64)
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
|
||||
// Total requests
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_TOTAL_MD, stat.total as f64)
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
|
||||
if stat.supported_metrics.lifecycle {
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_MD, stat.in_flight as f64)
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_MD, stat.errors_total as f64)
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_MD, stat.errors_5xx as f64)
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_MD, stat.errors_4xx as f64)
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_MD, stat.canceled as f64)
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
}
|
||||
// Total errors
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_MD, stat.errors_total as f64)
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
|
||||
if stat.supported_metrics.ttfb {
|
||||
for (le, value) in &stat.ttfb_distribution {
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD, *value)
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone())
|
||||
.with_label_owned(LE_LABEL, le.clone()),
|
||||
);
|
||||
}
|
||||
}
|
||||
// 5xx errors
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_MD, stat.errors_5xx as f64)
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
|
||||
if !stat.server.is_empty() {
|
||||
// 4xx errors
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_MD, stat.errors_4xx as f64)
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
|
||||
// Canceled requests
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_MD, stat.canceled as f64)
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
|
||||
// TTFB distribution (histogram buckets)
|
||||
for (le, value) in &stat.ttfb_distribution {
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_TOTAL_BY_SERVER_MD, stat.total as f64)
|
||||
.with_label_owned(SERVER_LABEL, stat.server.clone())
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD, *value)
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone())
|
||||
.with_label_owned(LE_LABEL, le.clone()),
|
||||
);
|
||||
|
||||
if stat.supported_metrics.lifecycle {
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_IN_FLIGHT_TOTAL_BY_SERVER_MD, stat.in_flight as f64)
|
||||
.with_label_owned(SERVER_LABEL, stat.server.clone())
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_ERRORS_TOTAL_BY_SERVER_MD, stat.errors_total as f64)
|
||||
.with_label_owned(SERVER_LABEL, stat.server.clone())
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_5XX_ERRORS_TOTAL_BY_SERVER_MD, stat.errors_5xx as f64)
|
||||
.with_label_owned(SERVER_LABEL, stat.server.clone())
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_4XX_ERRORS_TOTAL_BY_SERVER_MD, stat.errors_4xx as f64)
|
||||
.with_label_owned(SERVER_LABEL, stat.server.clone())
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_CANCELED_TOTAL_BY_SERVER_MD, stat.canceled as f64)
|
||||
.with_label_owned(SERVER_LABEL, stat.server.clone())
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone()),
|
||||
);
|
||||
}
|
||||
|
||||
if stat.supported_metrics.ttfb {
|
||||
for (le, value) in &stat.ttfb_distribution {
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_BY_SERVER_MD, *value)
|
||||
.with_label_owned(SERVER_LABEL, stat.server.clone())
|
||||
.with_label_owned(NAME_LABEL, stat.name.clone())
|
||||
.with_label_owned(TYPE_LABEL, stat.req_type.clone())
|
||||
.with_label_owned(LE_LABEL, le.clone()),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,19 +126,6 @@ pub fn collect_request_metrics(stats: &[ApiRequestStats]) -> Vec<PrometheusMetri
|
||||
);
|
||||
}
|
||||
|
||||
for ((server, req_type), (sent_bytes, recv_bytes)) in traffic_by_server_type {
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_TRAFFIC_SENT_BYTES_BY_SERVER_MD, sent_bytes as f64)
|
||||
.with_label_owned(SERVER_LABEL, server.to_string())
|
||||
.with_label_owned(TYPE_LABEL, req_type.to_string()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&API_TRAFFIC_RECV_BYTES_BY_SERVER_MD, recv_bytes as f64)
|
||||
.with_label_owned(SERVER_LABEL, server.to_string())
|
||||
.with_label_owned(TYPE_LABEL, req_type.to_string()),
|
||||
);
|
||||
}
|
||||
|
||||
metrics
|
||||
}
|
||||
|
||||
@@ -251,7 +137,6 @@ mod tests {
|
||||
#[test]
|
||||
fn test_collect_request_metrics() {
|
||||
let stats = vec![ApiRequestStats {
|
||||
server: "node1:9000".to_string(),
|
||||
name: "GetObject".to_string(),
|
||||
req_type: "s3".to_string(),
|
||||
in_flight: 10,
|
||||
@@ -268,13 +153,13 @@ mod tests {
|
||||
],
|
||||
sent_bytes: 1024 * 1024 * 500, // 500 MB
|
||||
recv_bytes: 1024 * 1024 * 100, // 100 MB
|
||||
supported_metrics: ApiRequestMetricSupport::ALL,
|
||||
}];
|
||||
|
||||
let metrics = collect_request_metrics(&stats);
|
||||
report_metrics(&metrics);
|
||||
|
||||
assert_eq!(metrics.len(), 24);
|
||||
// 6 base metrics + 4 TTFB buckets + 2 traffic metrics = 12
|
||||
assert_eq!(metrics.len(), 12);
|
||||
|
||||
let total_name = API_REQUESTS_TOTAL_MD.get_full_metric_name();
|
||||
let total = metrics.iter().find(|m| m.name == total_name);
|
||||
@@ -285,27 +170,6 @@ mod tests {
|
||||
let in_flight = metrics.iter().find(|m| m.name == in_flight_name);
|
||||
assert!(in_flight.is_some());
|
||||
assert_eq!(in_flight.map(|m| m.value), Some(10.0));
|
||||
|
||||
let by_server_total_name = API_REQUESTS_TOTAL_BY_SERVER_MD.get_full_metric_name();
|
||||
let by_server_total = metrics.iter().find(|m| {
|
||||
m.name == by_server_total_name
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(key, value)| *key == SERVER_LABEL && value == "node1:9000")
|
||||
&& m.labels.iter().any(|(key, value)| *key == NAME_LABEL && value == "GetObject")
|
||||
&& m.labels.iter().any(|(key, value)| *key == TYPE_LABEL && value == "s3")
|
||||
});
|
||||
assert_eq!(by_server_total.map(|m| m.value), Some(10000.0));
|
||||
|
||||
let by_server_sent_name = API_TRAFFIC_SENT_BYTES_BY_SERVER_MD.get_full_metric_name();
|
||||
let by_server_sent = metrics.iter().find(|m| {
|
||||
m.name == by_server_sent_name
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(key, value)| *key == SERVER_LABEL && value == "node1:9000")
|
||||
&& m.labels.iter().any(|(key, value)| *key == TYPE_LABEL && value == "s3")
|
||||
});
|
||||
assert_eq!(by_server_sent.map(|m| m.value), Some((1024 * 1024 * 500) as f64));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -315,63 +179,10 @@ mod tests {
|
||||
assert!(metrics.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_request_metrics_totals_only_skips_unsupported_dimensions() {
|
||||
let stats = vec![ApiRequestStats {
|
||||
server: "node1:9000".to_string(),
|
||||
name: "GetObject".to_string(),
|
||||
req_type: "s3".to_string(),
|
||||
in_flight: 10,
|
||||
total: 100,
|
||||
errors_total: 5,
|
||||
errors_5xx: 2,
|
||||
errors_4xx: 3,
|
||||
canceled: 1,
|
||||
ttfb_distribution: vec![("+Inf".to_string(), 100.0)],
|
||||
sent_bytes: 2048,
|
||||
recv_bytes: 1024,
|
||||
supported_metrics: ApiRequestMetricSupport::TOTALS_ONLY,
|
||||
}];
|
||||
|
||||
let metrics = collect_request_metrics(&stats);
|
||||
|
||||
assert!(
|
||||
metrics
|
||||
.iter()
|
||||
.any(|metric| metric.name == API_REQUESTS_TOTAL_MD.get_full_metric_name())
|
||||
);
|
||||
assert!(
|
||||
metrics
|
||||
.iter()
|
||||
.any(|metric| metric.name == API_REQUESTS_TOTAL_BY_SERVER_MD.get_full_metric_name())
|
||||
);
|
||||
assert!(
|
||||
!metrics
|
||||
.iter()
|
||||
.any(|metric| metric.name == API_REQUESTS_IN_FLIGHT_TOTAL_MD.get_full_metric_name())
|
||||
);
|
||||
assert!(
|
||||
!metrics
|
||||
.iter()
|
||||
.any(|metric| metric.name == API_REQUESTS_ERRORS_TOTAL_MD.get_full_metric_name())
|
||||
);
|
||||
assert!(
|
||||
!metrics
|
||||
.iter()
|
||||
.any(|metric| metric.name == API_TRAFFIC_SENT_BYTES_MD.get_full_metric_name())
|
||||
);
|
||||
assert!(
|
||||
!metrics
|
||||
.iter()
|
||||
.any(|metric| metric.name == API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD.get_full_metric_name())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_request_metrics_aggregates_traffic_per_type() {
|
||||
let stats = vec![
|
||||
ApiRequestStats {
|
||||
server: String::new(),
|
||||
name: "GetObject".to_string(),
|
||||
req_type: "s3".to_string(),
|
||||
in_flight: 1,
|
||||
@@ -383,10 +194,8 @@ mod tests {
|
||||
ttfb_distribution: vec![],
|
||||
sent_bytes: 100,
|
||||
recv_bytes: 10,
|
||||
supported_metrics: ApiRequestMetricSupport::ALL,
|
||||
},
|
||||
ApiRequestStats {
|
||||
server: String::new(),
|
||||
name: "HeadObject".to_string(),
|
||||
req_type: "s3".to_string(),
|
||||
in_flight: 2,
|
||||
@@ -398,7 +207,6 @@ mod tests {
|
||||
ttfb_distribution: vec![],
|
||||
sent_bytes: 200,
|
||||
recv_bytes: 20,
|
||||
supported_metrics: ApiRequestMetricSupport::ALL,
|
||||
},
|
||||
];
|
||||
|
||||
|
||||
@@ -20,7 +20,34 @@
|
||||
//! directory scans, and object scans.
|
||||
|
||||
use crate::metrics::report::PrometheusMetric;
|
||||
use crate::metrics::schema::scanner::*;
|
||||
use crate::metrics::schema::scanner::{
|
||||
SCANNER_ACTIVE_PATHS_MD, SCANNER_BITROT_CYCLE_ENABLED_MD, SCANNER_BITROT_CYCLE_SECONDS_MD, SCANNER_BUCKET_SCANS_FAILED_MD,
|
||||
SCANNER_BUCKET_SCANS_FINISHED_MD, SCANNER_BUCKET_SCANS_STARTED_MD, SCANNER_COMPLETED_CYCLES_MD,
|
||||
SCANNER_CURRENT_CYCLE_AGE_SECONDS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_FAILURES_MD,
|
||||
SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_MD, SCANNER_CURRENT_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD,
|
||||
SCANNER_CURRENT_CYCLE_DIRECTORIES_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_DIRECTORIES_SCANNED_MD,
|
||||
SCANNER_CURRENT_CYCLE_HEAL_OBJECTS_MD, SCANNER_CURRENT_CYCLE_ILM_ACTIONS_MD, SCANNER_CURRENT_CYCLE_MD,
|
||||
SCANNER_CURRENT_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_CURRENT_CYCLE_OBJECTS_SCANNED_MD,
|
||||
SCANNER_CURRENT_CYCLE_REPLICATION_CHECKS_MD, SCANNER_CURRENT_CYCLE_THROTTLE_SLEEP_DURATION_SECONDS_MD,
|
||||
SCANNER_CURRENT_CYCLE_THROTTLE_SLEEP_EVENTS_MD, SCANNER_CURRENT_CYCLE_USAGE_SAVES_MD,
|
||||
SCANNER_CURRENT_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_CURRENT_CYCLE_YIELD_EVENTS_MD,
|
||||
SCANNER_CURRENT_DISK_BUCKET_SCANS_ACTIVE_MD, SCANNER_CURRENT_DISK_BUCKET_SCANS_QUEUED_MD,
|
||||
SCANNER_CURRENT_DISK_SCAN_CONCURRENCY_LIMIT_MD, SCANNER_CURRENT_SCAN_MODE_MD, SCANNER_CURRENT_SET_SCAN_CONCURRENCY_LIMIT_MD,
|
||||
SCANNER_CURRENT_SET_SCANS_ACTIVE_MD, SCANNER_CURRENT_SET_SCANS_QUEUED_MD, SCANNER_CYCLE_INTERVAL_SECONDS_MD,
|
||||
SCANNER_CYCLE_MAX_DIRECTORIES_MD, SCANNER_CYCLE_MAX_DURATION_SECONDS_MD, SCANNER_CYCLE_MAX_OBJECTS_MD,
|
||||
SCANNER_DIRECTORIES_SCANNED_MD, SCANNER_FAILED_CYCLES_MD, SCANNER_LAST_ACTIVITY_SECONDS_MD,
|
||||
SCANNER_LAST_CYCLE_BUCKET_DRIVE_FAILURES_MD, SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_MD,
|
||||
SCANNER_LAST_CYCLE_BUCKET_DRIVE_SCANS_PER_SECOND_MD, SCANNER_LAST_CYCLE_DIRECTORIES_PER_SECOND_MD,
|
||||
SCANNER_LAST_CYCLE_DIRECTORIES_SCANNED_MD, SCANNER_LAST_CYCLE_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_HEAL_OBJECTS_MD,
|
||||
SCANNER_LAST_CYCLE_ILM_ACTIONS_MD, SCANNER_LAST_CYCLE_OBJECTS_PER_SECOND_MD, SCANNER_LAST_CYCLE_OBJECTS_SCANNED_MD,
|
||||
SCANNER_LAST_CYCLE_PARTIAL_REASON_MD, SCANNER_LAST_CYCLE_REPLICATION_CHECKS_MD, SCANNER_LAST_CYCLE_RESULT_MD,
|
||||
SCANNER_LAST_CYCLE_THROTTLE_SLEEP_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_THROTTLE_SLEEP_EVENTS_MD,
|
||||
SCANNER_LAST_CYCLE_USAGE_SAVES_MD, SCANNER_LAST_CYCLE_YIELD_DURATION_SECONDS_MD, SCANNER_LAST_CYCLE_YIELD_EVENTS_MD,
|
||||
SCANNER_OBJECTS_SCANNED_MD, SCANNER_OLDEST_ACTIVE_PATH_AGE_SECONDS_MD, SCANNER_PARTIAL_CYCLES_BY_REASON_MD,
|
||||
SCANNER_PARTIAL_CYCLES_MD, SCANNER_SUPERSEDED_CYCLES_MD, SCANNER_THROTTLE_IDLE_MODE_ENABLED_MD,
|
||||
SCANNER_THROTTLE_MAX_SLEEP_SECONDS_MD, SCANNER_THROTTLE_SLEEP_FACTOR_MD, SCANNER_VERSIONS_SCANNED_MD,
|
||||
SCANNER_YIELD_EVERY_N_OBJECTS_MD,
|
||||
};
|
||||
|
||||
/// Scanner statistics.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
@@ -165,123 +192,12 @@ pub struct ScannerStats {
|
||||
pub partial_cycles_directories: u64,
|
||||
}
|
||||
|
||||
/// Scanner source-work metrics for a source.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ScannerSourceWorkStats {
|
||||
pub source: String,
|
||||
pub checked: u64,
|
||||
pub queued: u64,
|
||||
pub executed: u64,
|
||||
pub failed: u64,
|
||||
pub skipped: u64,
|
||||
pub missed: u64,
|
||||
}
|
||||
|
||||
/// Scanner bucket-drive result metrics for a structured bucket/drive pair.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ScannerBucketDriveResultStats {
|
||||
pub bucket: String,
|
||||
pub drive: String,
|
||||
pub result: String,
|
||||
pub count: u64,
|
||||
}
|
||||
|
||||
/// Scanner statistics with runtime-local node identity and bounded source/result details.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct ScannerRuntimeStats {
|
||||
pub(crate) server: String,
|
||||
pub(crate) stats: ScannerStats,
|
||||
pub(crate) source_work: Vec<ScannerSourceWorkStats>,
|
||||
pub(crate) current_cycle_source_work: Vec<ScannerSourceWorkStats>,
|
||||
pub(crate) last_cycle_source_work: Vec<ScannerSourceWorkStats>,
|
||||
pub(crate) bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
|
||||
pub(crate) current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
|
||||
pub(crate) last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultStats>,
|
||||
}
|
||||
|
||||
/// Collects scanner metrics from the given stats.
|
||||
///
|
||||
/// Uses the metric descriptors from `metrics_type::scanner` module.
|
||||
/// Returns a vector of Prometheus metrics for scanner statistics.
|
||||
pub fn collect_scanner_metrics(stats: &ScannerStats) -> Vec<PrometheusMetric> {
|
||||
collect_scanner_metrics_with_runtime(stats, None)
|
||||
}
|
||||
|
||||
pub(crate) fn collect_scanner_runtime_metrics(stats: &ScannerRuntimeStats) -> Vec<PrometheusMetric> {
|
||||
collect_scanner_metrics_with_runtime(&stats.stats, Some(stats))
|
||||
}
|
||||
|
||||
fn collect_scanner_metrics_with_runtime(stats: &ScannerStats, runtime: Option<&ScannerRuntimeStats>) -> Vec<PrometheusMetric> {
|
||||
fn push_source_work_metric(
|
||||
metrics: &mut Vec<PrometheusMetric>,
|
||||
descriptor: &'static crate::metrics::schema::MetricDescriptor,
|
||||
server: &str,
|
||||
source: &str,
|
||||
state: &str,
|
||||
value: u64,
|
||||
cycle_scope: Option<&str>,
|
||||
) {
|
||||
let mut metric =
|
||||
PrometheusMetric::from_descriptor(descriptor, value as f64).with_label_owned(SERVER_LABEL, server.to_string());
|
||||
if let Some(cycle_scope) = cycle_scope {
|
||||
metric = metric.with_label_owned(CYCLE_SCOPE_LABEL, cycle_scope.to_string());
|
||||
}
|
||||
metric = metric
|
||||
.with_label_owned(SOURCE_LABEL, source.to_string())
|
||||
.with_label_owned(STATE_LABEL, state.to_string());
|
||||
metrics.push(metric);
|
||||
}
|
||||
|
||||
fn push_source_work_metrics(
|
||||
metrics: &mut Vec<PrometheusMetric>,
|
||||
descriptor: &'static crate::metrics::schema::MetricDescriptor,
|
||||
server: &str,
|
||||
source_work: &[ScannerSourceWorkStats],
|
||||
cycle_scope: Option<&str>,
|
||||
) {
|
||||
for work in source_work {
|
||||
push_source_work_metric(metrics, descriptor, server, &work.source, "checked", work.checked, cycle_scope);
|
||||
push_source_work_metric(metrics, descriptor, server, &work.source, "queued", work.queued, cycle_scope);
|
||||
push_source_work_metric(metrics, descriptor, server, &work.source, "executed", work.executed, cycle_scope);
|
||||
push_source_work_metric(metrics, descriptor, server, &work.source, "failed", work.failed, cycle_scope);
|
||||
push_source_work_metric(metrics, descriptor, server, &work.source, "skipped", work.skipped, cycle_scope);
|
||||
push_source_work_metric(metrics, descriptor, server, &work.source, "missed", work.missed, cycle_scope);
|
||||
}
|
||||
}
|
||||
|
||||
fn push_bucket_drive_result_metric(
|
||||
metrics: &mut Vec<PrometheusMetric>,
|
||||
descriptor: &'static crate::metrics::schema::MetricDescriptor,
|
||||
server: &str,
|
||||
result: &ScannerBucketDriveResultStats,
|
||||
cycle_scope: Option<&str>,
|
||||
) {
|
||||
let mut metric =
|
||||
PrometheusMetric::from_descriptor(descriptor, result.count as f64).with_label_owned(SERVER_LABEL, server.to_string());
|
||||
if let Some(cycle_scope) = cycle_scope {
|
||||
metric = metric.with_label_owned(CYCLE_SCOPE_LABEL, cycle_scope.to_string());
|
||||
}
|
||||
metrics.push(
|
||||
metric
|
||||
.with_label_owned(BUCKET_LABEL, result.bucket.clone())
|
||||
.with_label_owned(DRIVE_LABEL, result.drive.clone())
|
||||
.with_label_owned(RESULT_LABEL, result.result.clone()),
|
||||
);
|
||||
}
|
||||
|
||||
fn push_bucket_drive_result_metrics(
|
||||
metrics: &mut Vec<PrometheusMetric>,
|
||||
descriptor: &'static crate::metrics::schema::MetricDescriptor,
|
||||
server: &str,
|
||||
results: &[ScannerBucketDriveResultStats],
|
||||
cycle_scope: Option<&str>,
|
||||
) {
|
||||
for result in results {
|
||||
push_bucket_drive_result_metric(metrics, descriptor, server, result, cycle_scope);
|
||||
}
|
||||
}
|
||||
|
||||
let mut metrics = vec![
|
||||
vec![
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FINISHED_MD, stats.bucket_scans_finished as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_STARTED_MD, stats.bucket_scans_started as f64),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_BUCKET_SCANS_FAILED_MD, stats.bucket_scans_failed as f64),
|
||||
@@ -415,48 +331,7 @@ fn collect_scanner_metrics_with_runtime(stats: &ScannerStats, runtime: Option<&S
|
||||
.with_label("reason", "objects"),
|
||||
PrometheusMetric::from_descriptor(&SCANNER_PARTIAL_CYCLES_BY_REASON_MD, stats.partial_cycles_directories as f64)
|
||||
.with_label("reason", "directories"),
|
||||
];
|
||||
|
||||
if let Some(runtime) = runtime {
|
||||
push_source_work_metrics(&mut metrics, &SCANNER_SOURCE_WORK_TOTAL_MD, &runtime.server, &runtime.source_work, None);
|
||||
push_source_work_metrics(
|
||||
&mut metrics,
|
||||
&SCANNER_CYCLE_SOURCE_WORK_MD,
|
||||
&runtime.server,
|
||||
&runtime.current_cycle_source_work,
|
||||
Some("current"),
|
||||
);
|
||||
push_source_work_metrics(
|
||||
&mut metrics,
|
||||
&SCANNER_CYCLE_SOURCE_WORK_MD,
|
||||
&runtime.server,
|
||||
&runtime.last_cycle_source_work,
|
||||
Some("last"),
|
||||
);
|
||||
push_bucket_drive_result_metrics(
|
||||
&mut metrics,
|
||||
&SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD,
|
||||
&runtime.server,
|
||||
&runtime.bucket_drive_results,
|
||||
None,
|
||||
);
|
||||
push_bucket_drive_result_metrics(
|
||||
&mut metrics,
|
||||
&SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD,
|
||||
&runtime.server,
|
||||
&runtime.current_cycle_bucket_drive_results,
|
||||
Some("current"),
|
||||
);
|
||||
push_bucket_drive_result_metrics(
|
||||
&mut metrics,
|
||||
&SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD,
|
||||
&runtime.server,
|
||||
&runtime.last_cycle_bucket_drive_results,
|
||||
Some("last"),
|
||||
);
|
||||
}
|
||||
|
||||
metrics
|
||||
]
|
||||
}
|
||||
|
||||
fn bool_metric_value(enabled: bool) -> f64 {
|
||||
@@ -470,130 +345,82 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_collect_scanner_metrics() {
|
||||
let stats = ScannerRuntimeStats {
|
||||
server: "node1:9000".to_string(),
|
||||
source_work: vec![ScannerSourceWorkStats {
|
||||
source: "lifecycle".to_string(),
|
||||
checked: 11,
|
||||
queued: 2,
|
||||
executed: 3,
|
||||
failed: 4,
|
||||
skipped: 5,
|
||||
missed: 6,
|
||||
}],
|
||||
current_cycle_source_work: vec![ScannerSourceWorkStats {
|
||||
source: "usage".to_string(),
|
||||
checked: 21,
|
||||
queued: 7,
|
||||
executed: 8,
|
||||
failed: 9,
|
||||
skipped: 10,
|
||||
missed: 11,
|
||||
}],
|
||||
last_cycle_source_work: vec![ScannerSourceWorkStats {
|
||||
source: "heal".to_string(),
|
||||
checked: 31,
|
||||
queued: 12,
|
||||
executed: 13,
|
||||
failed: 14,
|
||||
skipped: 15,
|
||||
missed: 16,
|
||||
}],
|
||||
bucket_drive_results: vec![ScannerBucketDriveResultStats {
|
||||
bucket: "photos".to_string(),
|
||||
drive: "/data1".to_string(),
|
||||
result: "success".to_string(),
|
||||
count: 3,
|
||||
}],
|
||||
current_cycle_bucket_drive_results: vec![ScannerBucketDriveResultStats {
|
||||
bucket: "photos".to_string(),
|
||||
drive: "/data1".to_string(),
|
||||
result: "partial".to_string(),
|
||||
count: 1,
|
||||
}],
|
||||
last_cycle_bucket_drive_results: vec![ScannerBucketDriveResultStats {
|
||||
bucket: "videos".to_string(),
|
||||
drive: "/data2".to_string(),
|
||||
result: "error".to_string(),
|
||||
count: 2,
|
||||
}],
|
||||
stats: ScannerStats {
|
||||
bucket_scans_finished: 100,
|
||||
bucket_scans_started: 100,
|
||||
bucket_scans_failed: 2,
|
||||
directories_scanned: 50000,
|
||||
objects_scanned: 1000000,
|
||||
versions_scanned: 1500000,
|
||||
last_activity_seconds: 30,
|
||||
active_paths: 4,
|
||||
oldest_active_path_age_seconds: 17,
|
||||
current_set_scan_concurrency_limit: 3,
|
||||
current_set_scans_queued: 5,
|
||||
current_set_scans_active: 2,
|
||||
current_disk_scan_concurrency_limit: 6,
|
||||
current_disk_bucket_scans_queued: 18,
|
||||
current_disk_bucket_scans_active: 4,
|
||||
throttle_idle_mode_enabled: true,
|
||||
throttle_sleep_factor: 10.0,
|
||||
throttle_max_sleep_seconds: 15.0,
|
||||
yield_every_n_objects: 128,
|
||||
cycle_interval_seconds: 3600.0,
|
||||
cycle_max_duration_seconds: 1800.0,
|
||||
cycle_max_objects: 1_000_000,
|
||||
cycle_max_directories: 100_000,
|
||||
bitrot_cycle_enabled: true,
|
||||
bitrot_cycle_seconds: 86400.0,
|
||||
current_cycle: 12,
|
||||
completed_cycles: 11,
|
||||
current_cycle_age_seconds: 90,
|
||||
current_cycle_objects_scanned: 250,
|
||||
current_cycle_directories_scanned: 20,
|
||||
current_cycle_bucket_drive_scans: 2,
|
||||
current_cycle_bucket_drive_failures: 1,
|
||||
current_cycle_objects_per_second: 12.5,
|
||||
current_cycle_directories_per_second: 1.0,
|
||||
current_cycle_bucket_drive_scans_per_second: 0.1,
|
||||
current_cycle_yield_events: 8,
|
||||
current_cycle_yield_duration_seconds: 1.25,
|
||||
current_cycle_throttle_sleep_events: 4,
|
||||
current_cycle_throttle_sleep_duration_seconds: 2.5,
|
||||
current_cycle_ilm_actions: 6,
|
||||
current_cycle_heal_objects: 2,
|
||||
current_cycle_replication_checks: 5,
|
||||
current_cycle_usage_saves: 3,
|
||||
current_scan_mode: 2,
|
||||
last_cycle_result: 1,
|
||||
last_cycle_partial_reason: 3,
|
||||
last_cycle_duration_seconds: 42.5,
|
||||
last_cycle_objects_scanned: 900,
|
||||
last_cycle_directories_scanned: 80,
|
||||
last_cycle_bucket_drive_scans: 6,
|
||||
last_cycle_bucket_drive_failures: 2,
|
||||
last_cycle_objects_per_second: 18.0,
|
||||
last_cycle_directories_per_second: 1.6,
|
||||
last_cycle_bucket_drive_scans_per_second: 0.12,
|
||||
last_cycle_yield_events: 30,
|
||||
last_cycle_yield_duration_seconds: 9.5,
|
||||
last_cycle_throttle_sleep_events: 12,
|
||||
last_cycle_throttle_sleep_duration_seconds: 6.75,
|
||||
last_cycle_ilm_actions: 44,
|
||||
last_cycle_heal_objects: 7,
|
||||
last_cycle_replication_checks: 12,
|
||||
last_cycle_usage_saves: 9,
|
||||
failed_cycles: 3,
|
||||
superseded_cycles: 5,
|
||||
partial_cycles: 10,
|
||||
partial_cycles_unknown: 1,
|
||||
partial_cycles_runtime: 2,
|
||||
partial_cycles_objects: 3,
|
||||
partial_cycles_directories: 4,
|
||||
},
|
||||
let stats = ScannerStats {
|
||||
bucket_scans_finished: 100,
|
||||
bucket_scans_started: 100,
|
||||
bucket_scans_failed: 2,
|
||||
directories_scanned: 50000,
|
||||
objects_scanned: 1000000,
|
||||
versions_scanned: 1500000,
|
||||
last_activity_seconds: 30,
|
||||
active_paths: 4,
|
||||
oldest_active_path_age_seconds: 17,
|
||||
current_set_scan_concurrency_limit: 3,
|
||||
current_set_scans_queued: 5,
|
||||
current_set_scans_active: 2,
|
||||
current_disk_scan_concurrency_limit: 6,
|
||||
current_disk_bucket_scans_queued: 18,
|
||||
current_disk_bucket_scans_active: 4,
|
||||
throttle_idle_mode_enabled: true,
|
||||
throttle_sleep_factor: 10.0,
|
||||
throttle_max_sleep_seconds: 15.0,
|
||||
yield_every_n_objects: 128,
|
||||
cycle_interval_seconds: 3600.0,
|
||||
cycle_max_duration_seconds: 1800.0,
|
||||
cycle_max_objects: 1_000_000,
|
||||
cycle_max_directories: 100_000,
|
||||
bitrot_cycle_enabled: true,
|
||||
bitrot_cycle_seconds: 86400.0,
|
||||
current_cycle: 12,
|
||||
completed_cycles: 11,
|
||||
current_cycle_age_seconds: 90,
|
||||
current_cycle_objects_scanned: 250,
|
||||
current_cycle_directories_scanned: 20,
|
||||
current_cycle_bucket_drive_scans: 2,
|
||||
current_cycle_bucket_drive_failures: 1,
|
||||
current_cycle_objects_per_second: 12.5,
|
||||
current_cycle_directories_per_second: 1.0,
|
||||
current_cycle_bucket_drive_scans_per_second: 0.1,
|
||||
current_cycle_yield_events: 8,
|
||||
current_cycle_yield_duration_seconds: 1.25,
|
||||
current_cycle_throttle_sleep_events: 4,
|
||||
current_cycle_throttle_sleep_duration_seconds: 2.5,
|
||||
current_cycle_ilm_actions: 6,
|
||||
current_cycle_heal_objects: 2,
|
||||
current_cycle_replication_checks: 5,
|
||||
current_cycle_usage_saves: 3,
|
||||
current_scan_mode: 2,
|
||||
last_cycle_result: 1,
|
||||
last_cycle_partial_reason: 3,
|
||||
last_cycle_duration_seconds: 42.5,
|
||||
last_cycle_objects_scanned: 900,
|
||||
last_cycle_directories_scanned: 80,
|
||||
last_cycle_bucket_drive_scans: 6,
|
||||
last_cycle_bucket_drive_failures: 2,
|
||||
last_cycle_objects_per_second: 18.0,
|
||||
last_cycle_directories_per_second: 1.6,
|
||||
last_cycle_bucket_drive_scans_per_second: 0.12,
|
||||
last_cycle_yield_events: 30,
|
||||
last_cycle_yield_duration_seconds: 9.5,
|
||||
last_cycle_throttle_sleep_events: 12,
|
||||
last_cycle_throttle_sleep_duration_seconds: 6.75,
|
||||
last_cycle_ilm_actions: 44,
|
||||
last_cycle_heal_objects: 7,
|
||||
last_cycle_replication_checks: 12,
|
||||
last_cycle_usage_saves: 9,
|
||||
failed_cycles: 3,
|
||||
superseded_cycles: 5,
|
||||
partial_cycles: 10,
|
||||
partial_cycles_unknown: 1,
|
||||
partial_cycles_runtime: 2,
|
||||
partial_cycles_objects: 3,
|
||||
partial_cycles_directories: 4,
|
||||
};
|
||||
|
||||
let metrics = collect_scanner_runtime_metrics(&stats);
|
||||
let metrics = collect_scanner_metrics(&stats);
|
||||
report_metrics(&metrics);
|
||||
|
||||
assert_eq!(metrics.len(), 90);
|
||||
assert_eq!(metrics.len(), 69);
|
||||
|
||||
let objects = metrics.iter().find(|m| m.value == 1000000.0);
|
||||
assert!(objects.is_some());
|
||||
@@ -605,18 +432,6 @@ mod tests {
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_ACTIVE_PATHS_MD.get_full_metric_name());
|
||||
assert_eq!(active_paths.map(|m| m.value), Some(4.0));
|
||||
assert_eq!(active_paths.map(|m| m.labels.len()), Some(0));
|
||||
|
||||
let bucket_drive_result = metrics
|
||||
.iter()
|
||||
.find(|m| m.name == SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD.get_full_metric_name());
|
||||
assert_eq!(bucket_drive_result.map(|m| m.value), Some(3.0));
|
||||
assert_eq!(
|
||||
bucket_drive_result
|
||||
.and_then(|m| m.labels.iter().find(|(name, _)| *name == BUCKET_LABEL))
|
||||
.map(|(_, value)| value.as_ref()),
|
||||
Some("photos")
|
||||
);
|
||||
|
||||
let oldest_active_path_age = metrics
|
||||
.iter()
|
||||
@@ -931,37 +746,6 @@ mod tests {
|
||||
.any(|(name, value)| *name == "reason" && value.as_ref() == "directories")
|
||||
});
|
||||
assert_eq!(partial_cycles_directories.map(|m| m.value), Some(4.0));
|
||||
|
||||
let lifecycle_failed = metrics.iter().find(|m| {
|
||||
m.name == SCANNER_SOURCE_WORK_TOTAL_MD.get_full_metric_name()
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000")
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == SOURCE_LABEL && value.as_ref() == "lifecycle")
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "failed")
|
||||
});
|
||||
assert_eq!(lifecycle_failed.map(|m| m.value), Some(4.0));
|
||||
|
||||
let current_usage_executed = metrics.iter().find(|m| {
|
||||
m.name == SCANNER_CYCLE_SOURCE_WORK_MD.get_full_metric_name()
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000")
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == CYCLE_SCOPE_LABEL && value.as_ref() == "current")
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == SOURCE_LABEL && value.as_ref() == "usage")
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "executed")
|
||||
});
|
||||
assert_eq!(current_usage_executed.map(|m| m.value), Some(8.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -78,24 +78,6 @@ pub struct DriveDetailedStats {
|
||||
pub perc_util: Option<f64>,
|
||||
}
|
||||
|
||||
/// Detailed drive statistics with runtime topology and per-operation dimensions.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct DriveRuntimeDetailedStats {
|
||||
pub(crate) stats: DriveDetailedStats,
|
||||
pub(crate) pool_index: Option<String>,
|
||||
pub(crate) set_index: Option<String>,
|
||||
pub(crate) drive_index: Option<String>,
|
||||
pub(crate) disk_id: Option<String>,
|
||||
pub(crate) runtime_state: Option<String>,
|
||||
pub(crate) healing: bool,
|
||||
pub(crate) scanning: bool,
|
||||
pub(crate) offline_duration_seconds: Option<u64>,
|
||||
/// Drive API calls by operation
|
||||
pub(crate) api_calls: Vec<(String, u64)>,
|
||||
/// Last-minute API latency by operation, in microseconds
|
||||
pub(crate) api_latency_by_api_micros: Vec<(String, u64)>,
|
||||
}
|
||||
|
||||
/// Aggregate drive count statistics.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct DriveCountStats {
|
||||
@@ -111,58 +93,6 @@ pub struct DriveCountStats {
|
||||
///
|
||||
/// Returns a vector of Prometheus metrics for each drive.
|
||||
pub fn collect_drive_detailed_metrics(stats: &[DriveDetailedStats]) -> Vec<PrometheusMetric> {
|
||||
let runtime_stats = stats
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|stats| DriveRuntimeDetailedStats {
|
||||
stats,
|
||||
..Default::default()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
collect_drive_runtime_detailed_metrics(&runtime_stats)
|
||||
}
|
||||
|
||||
pub(crate) fn collect_drive_runtime_detailed_metrics(stats: &[DriveRuntimeDetailedStats]) -> Vec<PrometheusMetric> {
|
||||
const DRIVE_RUNTIME_STATES: [&str; 5] = ["online", "offline", "returning", "suspect", "unknown"];
|
||||
|
||||
fn topology_labels(stat: &DriveRuntimeDetailedStats) -> Option<[Cow<'static, str>; 5]> {
|
||||
Some([
|
||||
Cow::Owned(stat.stats.server.clone()),
|
||||
Cow::Owned(stat.stats.drive.clone()),
|
||||
Cow::Owned(stat.pool_index.as_ref()?.clone()),
|
||||
Cow::Owned(stat.set_index.as_ref()?.clone()),
|
||||
Cow::Owned(stat.drive_index.as_ref()?.clone()),
|
||||
])
|
||||
}
|
||||
|
||||
fn has_topology_labels(stat: &DriveRuntimeDetailedStats) -> bool {
|
||||
stat.pool_index.is_some() && stat.set_index.is_some() && stat.drive_index.is_some()
|
||||
}
|
||||
|
||||
fn normalized_runtime_state(runtime_state: &str) -> &str {
|
||||
DRIVE_RUNTIME_STATES
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|state| state.eq_ignore_ascii_case(runtime_state))
|
||||
.unwrap_or("unknown")
|
||||
}
|
||||
|
||||
fn push_topology_metric(
|
||||
metrics: &mut Vec<PrometheusMetric>,
|
||||
descriptor: &'static crate::metrics::schema::MetricDescriptor,
|
||||
value: f64,
|
||||
labels: &[Cow<'static, str>; 5],
|
||||
) {
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(descriptor, value)
|
||||
.with_label(SERVER_LABEL, labels[0].clone())
|
||||
.with_label(DRIVE_LABEL, labels[1].clone())
|
||||
.with_label(POOL_INDEX_LABEL, labels[2].clone())
|
||||
.with_label(SET_INDEX_LABEL, labels[3].clone())
|
||||
.with_label(DRIVE_INDEX_LABEL, labels[4].clone()),
|
||||
);
|
||||
}
|
||||
|
||||
fn push_drive_metric(
|
||||
metrics: &mut Vec<PrometheusMetric>,
|
||||
descriptor: &'static crate::metrics::schema::MetricDescriptor,
|
||||
@@ -177,49 +107,19 @@ pub(crate) fn collect_drive_runtime_detailed_metrics(stats: &[DriveRuntimeDetail
|
||||
);
|
||||
}
|
||||
|
||||
let metric_capacity = stats
|
||||
.iter()
|
||||
.map(|stat| {
|
||||
let api_metrics = if has_topology_labels(stat) {
|
||||
stat.api_calls.len() + stat.api_latency_by_api_micros.len()
|
||||
} else {
|
||||
0
|
||||
};
|
||||
31 + api_metrics
|
||||
})
|
||||
.sum();
|
||||
let mut metrics = Vec::with_capacity(metric_capacity);
|
||||
let mut metrics = Vec::with_capacity(stats.len() * 23);
|
||||
|
||||
for stat in stats {
|
||||
let server_label = stat.stats.server.as_str();
|
||||
let drive_label = stat.stats.drive.as_str();
|
||||
let topology_labels = topology_labels(stat);
|
||||
let server_label = stat.server.as_str();
|
||||
let drive_label = stat.drive.as_str();
|
||||
|
||||
push_drive_metric(
|
||||
&mut metrics,
|
||||
&DRIVE_TOTAL_BYTES_MD,
|
||||
stat.stats.total_bytes as f64,
|
||||
server_label,
|
||||
drive_label,
|
||||
);
|
||||
push_drive_metric(
|
||||
&mut metrics,
|
||||
&DRIVE_USED_BYTES_MD,
|
||||
stat.stats.used_bytes as f64,
|
||||
server_label,
|
||||
drive_label,
|
||||
);
|
||||
push_drive_metric(
|
||||
&mut metrics,
|
||||
&DRIVE_FREE_BYTES_MD,
|
||||
stat.stats.free_bytes as f64,
|
||||
server_label,
|
||||
drive_label,
|
||||
);
|
||||
push_drive_metric(&mut metrics, &DRIVE_TOTAL_BYTES_MD, stat.total_bytes as f64, server_label, drive_label);
|
||||
push_drive_metric(&mut metrics, &DRIVE_USED_BYTES_MD, stat.used_bytes as f64, server_label, drive_label);
|
||||
push_drive_metric(&mut metrics, &DRIVE_FREE_BYTES_MD, stat.free_bytes as f64, server_label, drive_label);
|
||||
push_drive_metric(
|
||||
&mut metrics,
|
||||
&DRIVE_CAPACITY_OBSERVATION_AGE_SECONDS_MD,
|
||||
stat.stats.capacity_observation_age_seconds as f64,
|
||||
stat.capacity_observation_age_seconds as f64,
|
||||
server_label,
|
||||
drive_label,
|
||||
);
|
||||
@@ -227,123 +127,59 @@ pub(crate) fn collect_drive_runtime_detailed_metrics(stats: &[DriveRuntimeDetail
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(
|
||||
&DRIVE_CAPACITY_OBSERVATION_STATE_MD,
|
||||
if state == stat.stats.capacity_observation_state {
|
||||
1.0
|
||||
} else {
|
||||
0.0
|
||||
},
|
||||
if state == stat.capacity_observation_state { 1.0 } else { 0.0 },
|
||||
)
|
||||
.with_label_owned(SERVER_LABEL, server_label.to_string())
|
||||
.with_label_owned(DRIVE_LABEL, drive_label.to_string())
|
||||
.with_label_owned("state", state.to_string()),
|
||||
);
|
||||
}
|
||||
if let Some(value) = stat.stats.used_inodes {
|
||||
if let Some(value) = stat.used_inodes {
|
||||
push_drive_metric(&mut metrics, &DRIVE_USED_INODES_MD, value as f64, server_label, drive_label);
|
||||
}
|
||||
if let Some(value) = stat.stats.free_inodes {
|
||||
if let Some(value) = stat.free_inodes {
|
||||
push_drive_metric(&mut metrics, &DRIVE_FREE_INODES_MD, value as f64, server_label, drive_label);
|
||||
}
|
||||
if let Some(value) = stat.stats.total_inodes {
|
||||
if let Some(value) = stat.total_inodes {
|
||||
push_drive_metric(&mut metrics, &DRIVE_TOTAL_INODES_MD, value as f64, server_label, drive_label);
|
||||
}
|
||||
if let Some(value) = stat.stats.timeout_errors_total {
|
||||
if let Some(value) = stat.timeout_errors_total {
|
||||
push_drive_metric(&mut metrics, &DRIVE_TIMEOUT_ERRORS_MD, value as f64, server_label, drive_label);
|
||||
}
|
||||
if let Some(value) = stat.stats.io_errors_total {
|
||||
if let Some(value) = stat.io_errors_total {
|
||||
push_drive_metric(&mut metrics, &DRIVE_IO_ERRORS_MD, value as f64, server_label, drive_label);
|
||||
}
|
||||
if let Some(value) = stat.stats.availability_errors_total {
|
||||
if let Some(value) = stat.availability_errors_total {
|
||||
push_drive_metric(&mut metrics, &DRIVE_AVAILABILITY_ERRORS_MD, value as f64, server_label, drive_label);
|
||||
}
|
||||
if let Some(value) = stat.stats.waiting_io {
|
||||
if let Some(value) = stat.waiting_io {
|
||||
push_drive_metric(&mut metrics, &DRIVE_WAITING_IO_MD, value as f64, server_label, drive_label);
|
||||
}
|
||||
if let Some(value) = stat.stats.api_latency_micros {
|
||||
if let Some(value) = stat.api_latency_micros {
|
||||
push_drive_metric(&mut metrics, &DRIVE_API_LATENCY_MD, value as f64, server_label, drive_label);
|
||||
}
|
||||
push_drive_metric(&mut metrics, &DRIVE_HEALTH_MD, stat.stats.health as f64, server_label, drive_label);
|
||||
if let Some(value) = stat.stats.reads_per_sec {
|
||||
push_drive_metric(&mut metrics, &DRIVE_HEALTH_MD, stat.health as f64, server_label, drive_label);
|
||||
if let Some(value) = stat.reads_per_sec {
|
||||
push_drive_metric(&mut metrics, &DRIVE_READS_PER_SEC_MD, value, server_label, drive_label);
|
||||
}
|
||||
if let Some(value) = stat.stats.reads_kb_per_sec {
|
||||
if let Some(value) = stat.reads_kb_per_sec {
|
||||
push_drive_metric(&mut metrics, &DRIVE_READS_KB_PER_SEC_MD, value, server_label, drive_label);
|
||||
}
|
||||
if let Some(value) = stat.stats.reads_await {
|
||||
if let Some(value) = stat.reads_await {
|
||||
push_drive_metric(&mut metrics, &DRIVE_READS_AWAIT_MD, value, server_label, drive_label);
|
||||
}
|
||||
if let Some(value) = stat.stats.writes_per_sec {
|
||||
if let Some(value) = stat.writes_per_sec {
|
||||
push_drive_metric(&mut metrics, &DRIVE_WRITES_PER_SEC_MD, value, server_label, drive_label);
|
||||
}
|
||||
if let Some(value) = stat.stats.writes_kb_per_sec {
|
||||
if let Some(value) = stat.writes_kb_per_sec {
|
||||
push_drive_metric(&mut metrics, &DRIVE_WRITES_KB_PER_SEC_MD, value, server_label, drive_label);
|
||||
}
|
||||
if let Some(value) = stat.stats.writes_await {
|
||||
if let Some(value) = stat.writes_await {
|
||||
push_drive_metric(&mut metrics, &DRIVE_WRITES_AWAIT_MD, value, server_label, drive_label);
|
||||
}
|
||||
if let Some(value) = stat.stats.perc_util {
|
||||
if let Some(value) = stat.perc_util {
|
||||
push_drive_metric(&mut metrics, &DRIVE_PERC_UTIL_MD, value, server_label, drive_label);
|
||||
}
|
||||
if let Some(labels) = &topology_labels {
|
||||
if let Some(disk_id) = stat.disk_id.as_ref().filter(|disk_id| !disk_id.is_empty()) {
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&DRIVE_INFO_MD, 1.0)
|
||||
.with_label(SERVER_LABEL, labels[0].clone())
|
||||
.with_label(DRIVE_LABEL, labels[1].clone())
|
||||
.with_label(POOL_INDEX_LABEL, labels[2].clone())
|
||||
.with_label(SET_INDEX_LABEL, labels[3].clone())
|
||||
.with_label(DRIVE_INDEX_LABEL, labels[4].clone())
|
||||
.with_label_owned(DISK_ID_LABEL, disk_id.clone()),
|
||||
);
|
||||
}
|
||||
if let Some(runtime_state) = stat.runtime_state.as_ref().filter(|state| !state.is_empty()) {
|
||||
let runtime_state = normalized_runtime_state(runtime_state);
|
||||
for state in DRIVE_RUNTIME_STATES {
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(
|
||||
&DRIVE_RUNTIME_STATE_MD,
|
||||
if state == runtime_state { 1.0 } else { 0.0 },
|
||||
)
|
||||
.with_label(SERVER_LABEL, labels[0].clone())
|
||||
.with_label(DRIVE_LABEL, labels[1].clone())
|
||||
.with_label(POOL_INDEX_LABEL, labels[2].clone())
|
||||
.with_label(SET_INDEX_LABEL, labels[3].clone())
|
||||
.with_label(DRIVE_INDEX_LABEL, labels[4].clone())
|
||||
.with_label(STATE_LABEL, state),
|
||||
);
|
||||
}
|
||||
}
|
||||
push_topology_metric(&mut metrics, &DRIVE_HEALING_MD, if stat.healing { 1.0 } else { 0.0 }, labels);
|
||||
push_topology_metric(&mut metrics, &DRIVE_SCANNING_MD, if stat.scanning { 1.0 } else { 0.0 }, labels);
|
||||
push_topology_metric(
|
||||
&mut metrics,
|
||||
&DRIVE_OFFLINE_DURATION_SECONDS_MD,
|
||||
stat.offline_duration_seconds.unwrap_or(0) as f64,
|
||||
labels,
|
||||
);
|
||||
for (api, value) in &stat.api_calls {
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&DRIVE_API_CALLS_MD, *value as f64)
|
||||
.with_label(SERVER_LABEL, labels[0].clone())
|
||||
.with_label(DRIVE_LABEL, labels[1].clone())
|
||||
.with_label(POOL_INDEX_LABEL, labels[2].clone())
|
||||
.with_label(SET_INDEX_LABEL, labels[3].clone())
|
||||
.with_label(DRIVE_INDEX_LABEL, labels[4].clone())
|
||||
.with_label_owned(API_LABEL, api.clone()),
|
||||
);
|
||||
}
|
||||
for (api, value) in &stat.api_latency_by_api_micros {
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&DRIVE_API_LATENCY_BY_API_MD, *value as f64)
|
||||
.with_label(SERVER_LABEL, labels[0].clone())
|
||||
.with_label(DRIVE_LABEL, labels[1].clone())
|
||||
.with_label(POOL_INDEX_LABEL, labels[2].clone())
|
||||
.with_label(SET_INDEX_LABEL, labels[3].clone())
|
||||
.with_label(DRIVE_INDEX_LABEL, labels[4].clone())
|
||||
.with_label_owned(API_LABEL, api.clone()),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
metrics
|
||||
@@ -423,48 +259,36 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_collect_drive_detailed_metrics() {
|
||||
let stats = vec![DriveRuntimeDetailedStats {
|
||||
pool_index: Some("0".to_string()),
|
||||
set_index: Some("1".to_string()),
|
||||
drive_index: Some("2".to_string()),
|
||||
disk_id: Some("disk-uuid-1".to_string()),
|
||||
runtime_state: Some("online".to_string()),
|
||||
healing: true,
|
||||
scanning: false,
|
||||
offline_duration_seconds: Some(0),
|
||||
api_calls: vec![("read".to_string(), 7)],
|
||||
api_latency_by_api_micros: vec![("read".to_string(), 2500)],
|
||||
stats: DriveDetailedStats {
|
||||
server: "node1:9000".to_string(),
|
||||
drive: "/data/disk1".to_string(),
|
||||
total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
|
||||
used_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
|
||||
free_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
|
||||
capacity_observation_state: "live",
|
||||
capacity_observation_age_seconds: 0,
|
||||
used_inodes: Some(100000),
|
||||
free_inodes: Some(900000),
|
||||
total_inodes: Some(1000000),
|
||||
timeout_errors_total: Some(5),
|
||||
io_errors_total: Some(10),
|
||||
availability_errors_total: Some(2),
|
||||
waiting_io: Some(3),
|
||||
api_latency_micros: Some(1500),
|
||||
health: 1,
|
||||
reads_per_sec: Some(100.0),
|
||||
reads_kb_per_sec: Some(1024.0),
|
||||
reads_await: Some(5.5),
|
||||
writes_per_sec: Some(50.0),
|
||||
writes_kb_per_sec: Some(512.0),
|
||||
writes_await: Some(10.2),
|
||||
perc_util: Some(75.5),
|
||||
},
|
||||
let stats = vec![DriveDetailedStats {
|
||||
server: "node1:9000".to_string(),
|
||||
drive: "/data/disk1".to_string(),
|
||||
total_bytes: 1024 * 1024 * 1024 * 100, // 100 GB
|
||||
used_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
|
||||
free_bytes: 1024 * 1024 * 1024 * 50, // 50 GB
|
||||
capacity_observation_state: "live",
|
||||
capacity_observation_age_seconds: 0,
|
||||
used_inodes: Some(100000),
|
||||
free_inodes: Some(900000),
|
||||
total_inodes: Some(1000000),
|
||||
timeout_errors_total: Some(5),
|
||||
io_errors_total: Some(10),
|
||||
availability_errors_total: Some(2),
|
||||
waiting_io: Some(3),
|
||||
api_latency_micros: Some(1500),
|
||||
health: 1,
|
||||
reads_per_sec: Some(100.0),
|
||||
reads_kb_per_sec: Some(1024.0),
|
||||
reads_await: Some(5.5),
|
||||
writes_per_sec: Some(50.0),
|
||||
writes_kb_per_sec: Some(512.0),
|
||||
writes_await: Some(10.2),
|
||||
perc_util: Some(75.5),
|
||||
}];
|
||||
|
||||
let metrics = collect_drive_runtime_detailed_metrics(&stats);
|
||||
let metrics = collect_drive_detailed_metrics(&stats);
|
||||
report_metrics(&metrics);
|
||||
|
||||
assert_eq!(metrics.len(), 34);
|
||||
assert_eq!(metrics.len(), 23);
|
||||
|
||||
// Verify total bytes metric
|
||||
let total_bytes_name = DRIVE_TOTAL_BYTES_MD.get_full_metric_name();
|
||||
@@ -479,32 +303,6 @@ mod tests {
|
||||
);
|
||||
assert_metric_label_keys(&metrics, &DRIVE_API_LATENCY_MD, 1500.0, &[SERVER_LABEL, DRIVE_LABEL]);
|
||||
assert_metric_label_keys(&metrics, &DRIVE_CAPACITY_OBSERVATION_STATE_MD, 1.0, &[SERVER_LABEL, DRIVE_LABEL, "state"]);
|
||||
assert_metric_label_keys(
|
||||
&metrics,
|
||||
&DRIVE_INFO_MD,
|
||||
1.0,
|
||||
&[
|
||||
SERVER_LABEL,
|
||||
DRIVE_LABEL,
|
||||
POOL_INDEX_LABEL,
|
||||
SET_INDEX_LABEL,
|
||||
DRIVE_INDEX_LABEL,
|
||||
DISK_ID_LABEL,
|
||||
],
|
||||
);
|
||||
assert_metric_label_keys(
|
||||
&metrics,
|
||||
&DRIVE_API_CALLS_MD,
|
||||
7.0,
|
||||
&[
|
||||
SERVER_LABEL,
|
||||
DRIVE_LABEL,
|
||||
POOL_INDEX_LABEL,
|
||||
SET_INDEX_LABEL,
|
||||
DRIVE_INDEX_LABEL,
|
||||
API_LABEL,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -555,75 +353,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_runtime_state_metrics_keep_suspect_state_active() {
|
||||
let stats = vec![DriveRuntimeDetailedStats {
|
||||
pool_index: Some("0".to_string()),
|
||||
set_index: Some("1".to_string()),
|
||||
drive_index: Some("2".to_string()),
|
||||
runtime_state: Some("suspect".to_string()),
|
||||
stats: DriveDetailedStats {
|
||||
server: "node1:9000".to_string(),
|
||||
drive: "/data/disk1".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}];
|
||||
|
||||
let metrics = collect_drive_runtime_detailed_metrics(&stats);
|
||||
let state_metrics = metrics
|
||||
.iter()
|
||||
.filter(|metric| metric.name == DRIVE_RUNTIME_STATE_MD.get_full_metric_name())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(state_metrics.len(), 5);
|
||||
assert!(state_metrics.iter().any(|metric| {
|
||||
metric.value == 1.0
|
||||
&& metric
|
||||
.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "suspect")
|
||||
}));
|
||||
assert!(state_metrics.iter().filter(|metric| metric.value == 1.0).all(|metric| {
|
||||
metric
|
||||
.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == STATE_LABEL && value.as_ref() == "suspect")
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_offline_duration_zeroes_recovered_topology_drive() {
|
||||
let stats = vec![DriveRuntimeDetailedStats {
|
||||
pool_index: Some("0".to_string()),
|
||||
set_index: Some("1".to_string()),
|
||||
drive_index: Some("2".to_string()),
|
||||
runtime_state: Some("online".to_string()),
|
||||
offline_duration_seconds: None,
|
||||
stats: DriveDetailedStats {
|
||||
server: "node1:9000".to_string(),
|
||||
drive: "/data/disk1".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}];
|
||||
|
||||
let metrics = collect_drive_runtime_detailed_metrics(&stats);
|
||||
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == DRIVE_OFFLINE_DURATION_SECONDS_MD.get_full_metric_name()
|
||||
&& metric.value == 0.0
|
||||
&& metric
|
||||
.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == SERVER_LABEL && value == "node1:9000")
|
||||
&& metric
|
||||
.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == DRIVE_INDEX_LABEL && value == "2")
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_drive_count_metrics() {
|
||||
let stats = DriveCountStats {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -17,31 +17,18 @@
|
||||
use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub const TARGET_ID: &str = "target_id";
|
||||
pub const SERVER: &str = "server";
|
||||
const TARGET_ID: &str = "target_id";
|
||||
pub const RESULT: &str = "result"; // success / failure
|
||||
pub const STATUS: &str = "status"; // success / failure
|
||||
|
||||
pub const SUCCESS: &str = "success";
|
||||
pub const FAILURE: &str = "failure";
|
||||
|
||||
const TARGET_LABELS: [&str; 1] = [TARGET_ID];
|
||||
const TARGET_SERVER_LABELS: [&str; 2] = [SERVER, TARGET_ID];
|
||||
|
||||
pub static AUDIT_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::AuditFailedMessages,
|
||||
"Total number of messages that failed to send since start",
|
||||
&TARGET_LABELS,
|
||||
subsystems::AUDIT,
|
||||
)
|
||||
});
|
||||
|
||||
pub static AUDIT_FAILED_MESSAGES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("failed_messages_by_server".to_string()),
|
||||
"Total number of messages that failed to send since start by server and target",
|
||||
&TARGET_SERVER_LABELS,
|
||||
&[TARGET_ID],
|
||||
subsystems::AUDIT,
|
||||
)
|
||||
});
|
||||
@@ -50,16 +37,7 @@ pub static AUDIT_FAILED_STORE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::
|
||||
new_gauge_md(
|
||||
MetricName::AuditFailedStoreLength,
|
||||
"Number of audit messages held in the failed-events store for target",
|
||||
&TARGET_LABELS,
|
||||
subsystems::AUDIT,
|
||||
)
|
||||
});
|
||||
|
||||
pub static AUDIT_FAILED_STORE_LENGTH_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("failed_store_length_by_server".to_string()),
|
||||
"Number of audit messages held in the failed-events store by server and target",
|
||||
&TARGET_SERVER_LABELS,
|
||||
&[TARGET_ID],
|
||||
subsystems::AUDIT,
|
||||
)
|
||||
});
|
||||
@@ -68,16 +46,7 @@ pub static AUDIT_TARGET_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::
|
||||
new_gauge_md(
|
||||
MetricName::AuditTargetQueueLength,
|
||||
"Number of unsent messages in queue for target",
|
||||
&TARGET_LABELS,
|
||||
subsystems::AUDIT,
|
||||
)
|
||||
});
|
||||
|
||||
pub static AUDIT_TARGET_QUEUE_LENGTH_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("target_queue_length_by_server".to_string()),
|
||||
"Number of unsent audit messages in queue by server and target",
|
||||
&TARGET_SERVER_LABELS,
|
||||
&[TARGET_ID],
|
||||
subsystems::AUDIT,
|
||||
)
|
||||
});
|
||||
@@ -86,16 +55,7 @@ pub static AUDIT_TOTAL_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|
|
||||
new_gauge_md(
|
||||
MetricName::AuditTotalMessages,
|
||||
"Total number of messages sent since start",
|
||||
&TARGET_LABELS,
|
||||
subsystems::AUDIT,
|
||||
)
|
||||
});
|
||||
|
||||
pub static AUDIT_TOTAL_MESSAGES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("total_messages_by_server".to_string()),
|
||||
"Total number of messages sent since start by server and target",
|
||||
&TARGET_SERVER_LABELS,
|
||||
&[TARGET_ID],
|
||||
subsystems::AUDIT,
|
||||
)
|
||||
});
|
||||
|
||||
@@ -21,8 +21,6 @@ use std::sync::LazyLock;
|
||||
pub const BUCKET_L: &str = "bucket";
|
||||
/// Replication operation
|
||||
pub const OPERATION_L: &str = "operation";
|
||||
/// Replication proxy result
|
||||
pub const RESULT_L: &str = "result";
|
||||
/// Replication target ARN
|
||||
pub const TARGET_ARN_L: &str = "target_arn";
|
||||
/// Replication range
|
||||
@@ -50,16 +48,6 @@ const MRF_DROPPED_COUNT: &str = "mrf_dropped_count";
|
||||
const MRF_MISSED_COUNT: &str = "mrf_missed_count";
|
||||
const MRF_FLUSH_FAILURES: &str = "mrf_flush_failures";
|
||||
const MRF_LAST_FLUSH_DURATION_MILLIS: &str = "mrf_last_flush_duration_millis";
|
||||
const TARGET_SENT_BYTES: &str = "target_sent_bytes";
|
||||
const TARGET_SENT_COUNT: &str = "target_sent_count";
|
||||
const TARGET_TOTAL_FAILED_BYTES: &str = "target_total_failed_bytes";
|
||||
const TARGET_TOTAL_FAILED_COUNT: &str = "target_total_failed_count";
|
||||
const TARGET_LAST_MIN_FAILED_BYTES: &str = "target_last_min_failed_bytes";
|
||||
const TARGET_LAST_MIN_FAILED_COUNT: &str = "target_last_min_failed_count";
|
||||
const TARGET_LAST_HOUR_FAILED_BYTES: &str = "target_last_hour_failed_bytes";
|
||||
const TARGET_LAST_HOUR_FAILED_COUNT: &str = "target_last_hour_failed_count";
|
||||
|
||||
const BUCKET_TARGET_LABELS: [&str; 2] = [BUCKET_L, TARGET_ARN_L];
|
||||
|
||||
pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
@@ -70,15 +58,6 @@ pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = Laz
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_TARGET_LAST_HOUR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::from(TARGET_LAST_HOUR_FAILED_BYTES),
|
||||
"Total number of bytes failed at least once to replicate in the last hour on a bucket and target ARN",
|
||||
&BUCKET_TARGET_LABELS,
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_LAST_HR_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::LastHourFailedCount,
|
||||
@@ -88,15 +67,6 @@ pub static BUCKET_REPL_LAST_HR_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = Laz
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_TARGET_LAST_HOUR_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::from(TARGET_LAST_HOUR_FAILED_COUNT),
|
||||
"Total number of objects which failed replication in the last hour on a bucket and target ARN",
|
||||
&BUCKET_TARGET_LABELS,
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::LastMinFailedBytes,
|
||||
@@ -106,15 +76,6 @@ pub static BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = La
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_TARGET_LAST_MIN_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::from(TARGET_LAST_MIN_FAILED_BYTES),
|
||||
"Total number of bytes failed at least once to replicate in the last full minute on a bucket and target ARN",
|
||||
&BUCKET_TARGET_LABELS,
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::LastMinFailedCount,
|
||||
@@ -124,15 +85,6 @@ pub static BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = La
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_TARGET_LAST_MIN_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::from(TARGET_LAST_MIN_FAILED_COUNT),
|
||||
"Total number of objects which failed replication in the last full minute on a bucket and target ARN",
|
||||
&BUCKET_TARGET_LABELS,
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_LATENCY_MS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::LatencyMilliSec,
|
||||
@@ -385,15 +337,6 @@ pub static BUCKET_REPL_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_TARGET_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::from(TARGET_SENT_BYTES),
|
||||
"Total number of bytes replicated to a bucket replication target ARN",
|
||||
&BUCKET_TARGET_LABELS,
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::SentCount,
|
||||
@@ -403,15 +346,6 @@ pub static BUCKET_REPL_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_TARGET_SENT_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::from(TARGET_SENT_COUNT),
|
||||
"Total number of objects replicated to a bucket replication target ARN",
|
||||
&BUCKET_TARGET_LABELS,
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_RESYNC_STARTED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::from(RESYNC_STARTED_TOTAL),
|
||||
@@ -466,15 +400,6 @@ pub static BUCKET_REPL_TOTAL_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyL
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_TARGET_TOTAL_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::from(TARGET_TOTAL_FAILED_BYTES),
|
||||
"Total number of bytes failed at least once to replicate since server start by bucket and target ARN",
|
||||
&BUCKET_TARGET_LABELS,
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::TotalFailedCount,
|
||||
@@ -484,15 +409,6 @@ pub static BUCKET_REPL_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyL
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_TARGET_TOTAL_FAILED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::from(TARGET_TOTAL_FAILED_COUNT),
|
||||
"Total number of objects which failed replication since server start by bucket and target ARN",
|
||||
&BUCKET_TARGET_LABELS,
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_BANDWIDTH_LIMIT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::BandwidthLimitBytesPerSecond,
|
||||
@@ -519,12 +435,3 @@ pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD: LazyLock<Met
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_PROXY_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::Custom("proxy_requests_total".to_string()),
|
||||
"Total number of bucket replication proxy requests by operation and result",
|
||||
&[BUCKET_L, OPERATION_L, RESULT_L],
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
@@ -17,19 +17,6 @@
|
||||
use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub const SERVER_LABEL: &str = "server";
|
||||
pub const ACTION_LABEL: &str = "action";
|
||||
pub const STATE_LABEL: &str = "state";
|
||||
|
||||
pub static ILM_ACTION_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("action_tasks".to_string()),
|
||||
"ILM task counts by server, action, and state",
|
||||
&[SERVER_LABEL, ACTION_LABEL, STATE_LABEL],
|
||||
subsystems::ILM,
|
||||
)
|
||||
});
|
||||
|
||||
pub static ILM_EXPIRY_PENDING_TASKS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::IlmExpiryPendingTasks,
|
||||
|
||||
@@ -19,10 +19,8 @@ use std::sync::LazyLock;
|
||||
|
||||
pub const TARGET_ID: &str = "target_id";
|
||||
pub const TARGET_TYPE: &str = "target_type";
|
||||
pub const SERVER: &str = "server";
|
||||
|
||||
const NOTIFICATION_TARGET_LABELS: [&str; 2] = [TARGET_ID, TARGET_TYPE];
|
||||
const NOTIFICATION_TARGET_SERVER_LABELS: [&str; 3] = [SERVER, TARGET_ID, TARGET_TYPE];
|
||||
|
||||
pub static NOTIFICATION_TARGET_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
@@ -33,15 +31,6 @@ pub static NOTIFICATION_TARGET_FAILED_MESSAGES_MD: LazyLock<MetricDescriptor> =
|
||||
)
|
||||
});
|
||||
|
||||
pub static NOTIFICATION_TARGET_FAILED_MESSAGES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("target_failed_messages_by_server".to_string()),
|
||||
"Total number of notification messages that permanently failed to send by server and target",
|
||||
&NOTIFICATION_TARGET_SERVER_LABELS,
|
||||
subsystems::NOTIFICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::NotificationTargetFailedStoreLength,
|
||||
@@ -51,15 +40,6 @@ pub static NOTIFICATION_TARGET_FAILED_STORE_LENGTH_MD: LazyLock<MetricDescriptor
|
||||
)
|
||||
});
|
||||
|
||||
pub static NOTIFICATION_TARGET_FAILED_STORE_LENGTH_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("target_failed_store_length_by_server".to_string()),
|
||||
"Number of notification messages held in the failed-events store by server and target",
|
||||
&NOTIFICATION_TARGET_SERVER_LABELS,
|
||||
subsystems::NOTIFICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static NOTIFICATION_TARGET_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::NotificationTargetQueueLength,
|
||||
@@ -69,15 +49,6 @@ pub static NOTIFICATION_TARGET_QUEUE_LENGTH_MD: LazyLock<MetricDescriptor> = Laz
|
||||
)
|
||||
});
|
||||
|
||||
pub static NOTIFICATION_TARGET_QUEUE_LENGTH_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("target_queue_length_by_server".to_string()),
|
||||
"Number of queued notification messages pending delivery by server and target",
|
||||
&NOTIFICATION_TARGET_SERVER_LABELS,
|
||||
subsystems::NOTIFICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static NOTIFICATION_TARGET_TOTAL_MESSAGES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::NotificationTargetTotalMessages,
|
||||
@@ -86,12 +57,3 @@ pub static NOTIFICATION_TARGET_TOTAL_MESSAGES_MD: LazyLock<MetricDescriptor> = L
|
||||
subsystems::NOTIFICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static NOTIFICATION_TARGET_TOTAL_MESSAGES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("target_total_messages_by_server".to_string()),
|
||||
"Total number of notification messages successfully delivered by server and target",
|
||||
&NOTIFICATION_TARGET_SERVER_LABELS,
|
||||
subsystems::NOTIFICATION,
|
||||
)
|
||||
});
|
||||
|
||||
@@ -17,8 +17,6 @@
|
||||
use crate::{MetricDescriptor, MetricName, new_gauge_md, subsystems};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub const SERVER_LABEL: &str = "server";
|
||||
|
||||
pub static REPLICATION_AVERAGE_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationAverageActiveWorkers,
|
||||
@@ -28,15 +26,6 @@ pub static REPLICATION_AVERAGE_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = L
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_AVERAGE_ACTIVE_WORKERS_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("average_active_workers_by_server".to_string()),
|
||||
"Average number of active replication workers by server",
|
||||
&[SERVER_LABEL],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_AVERAGE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationAverageQueuedBytes,
|
||||
@@ -46,15 +35,6 @@ pub static REPLICATION_AVERAGE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = Laz
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_AVERAGE_QUEUED_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("average_queued_bytes_by_server".to_string()),
|
||||
"Average number of bytes queued for replication since server start by server",
|
||||
&[SERVER_LABEL],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_AVERAGE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationAverageQueuedCount,
|
||||
@@ -64,15 +44,6 @@ pub static REPLICATION_AVERAGE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = Laz
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_AVERAGE_QUEUED_COUNT_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("average_queued_count_by_server".to_string()),
|
||||
"Average number of objects queued for replication since server start by server",
|
||||
&[SERVER_LABEL],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_AVERAGE_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationAverageDataTransferRate,
|
||||
@@ -82,15 +53,6 @@ pub static REPLICATION_AVERAGE_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor>
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_AVERAGE_DATA_TRANSFER_RATE_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("average_data_transfer_rate_by_server".to_string()),
|
||||
"Average replication data transfer rate in bytes/sec by server",
|
||||
&[SERVER_LABEL],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_CURRENT_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationCurrentActiveWorkers,
|
||||
@@ -100,15 +62,6 @@ pub static REPLICATION_CURRENT_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = L
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_CURRENT_ACTIVE_WORKERS_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("current_active_workers_by_server".to_string()),
|
||||
"Total number of active replication workers by server",
|
||||
&[SERVER_LABEL],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationCurrentDataTransferRate,
|
||||
@@ -118,15 +71,6 @@ pub static REPLICATION_CURRENT_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor>
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_CURRENT_DATA_TRANSFER_RATE_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("current_data_transfer_rate_by_server".to_string()),
|
||||
"Current replication data transfer rate in bytes/sec by server",
|
||||
&[SERVER_LABEL],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationLastMinuteQueuedBytes,
|
||||
@@ -136,15 +80,6 @@ pub static REPLICATION_LAST_MINUTE_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> =
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_LAST_MINUTE_QUEUED_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("last_minute_queued_bytes_by_server".to_string()),
|
||||
"Number of bytes queued for replication in the last full minute by server",
|
||||
&[SERVER_LABEL],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationLastMinuteQueuedCount,
|
||||
@@ -154,15 +89,6 @@ pub static REPLICATION_LAST_MINUTE_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> =
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_LAST_MINUTE_QUEUED_COUNT_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("last_minute_queued_count_by_server".to_string()),
|
||||
"Number of objects queued for replication in the last full minute by server",
|
||||
&[SERVER_LABEL],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_MAX_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationMaxActiveWorkers,
|
||||
@@ -172,15 +98,6 @@ pub static REPLICATION_MAX_ACTIVE_WORKERS_MD: LazyLock<MetricDescriptor> = LazyL
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_MAX_ACTIVE_WORKERS_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("max_active_workers_by_server".to_string()),
|
||||
"Maximum number of active replication workers seen since server start by server",
|
||||
&[SERVER_LABEL],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_MAX_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationMaxQueuedBytes,
|
||||
@@ -190,15 +107,6 @@ pub static REPLICATION_MAX_QUEUED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLoc
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_MAX_QUEUED_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("max_queued_bytes_by_server".to_string()),
|
||||
"Maximum number of bytes queued for replication since server start by server",
|
||||
&[SERVER_LABEL],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_MAX_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationMaxQueuedCount,
|
||||
@@ -208,15 +116,6 @@ pub static REPLICATION_MAX_QUEUED_COUNT_MD: LazyLock<MetricDescriptor> = LazyLoc
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_MAX_QUEUED_COUNT_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("max_queued_count_by_server".to_string()),
|
||||
"Maximum number of objects queued for replication since server start by server",
|
||||
&[SERVER_LABEL],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationMaxDataTransferRate,
|
||||
@@ -226,15 +125,6 @@ pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = L
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_MAX_DATA_TRANSFER_RATE_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("max_data_transfer_rate_by_server".to_string()),
|
||||
"Maximum replication data transfer rate in bytes/sec seen since server start by server",
|
||||
&[SERVER_LABEL],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationRecentBacklogCount,
|
||||
@@ -243,12 +133,3 @@ pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = Laz
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static REPLICATION_RECENT_BACKLOG_COUNT_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("recent_backlog_count_by_server".to_string()),
|
||||
"Objects currently in replication backlog by server",
|
||||
&[SERVER_LABEL],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
@@ -22,15 +22,6 @@ pub const NAME_LABEL: &str = "name";
|
||||
pub const TYPE_LABEL: &str = "type";
|
||||
/// le label (for histogram buckets)
|
||||
pub const LE_LABEL: &str = "le";
|
||||
/// server label
|
||||
pub const SERVER_LABEL: &str = "server";
|
||||
|
||||
const API_NAME_TYPE_LABELS: [&str; 2] = [NAME_LABEL, TYPE_LABEL];
|
||||
const API_SERVER_NAME_TYPE_LABELS: [&str; 3] = [SERVER_LABEL, NAME_LABEL, TYPE_LABEL];
|
||||
const API_NAME_TYPE_LE_LABELS: [&str; 3] = [NAME_LABEL, TYPE_LABEL, LE_LABEL];
|
||||
const API_SERVER_NAME_TYPE_LE_LABELS: [&str; 4] = [SERVER_LABEL, NAME_LABEL, TYPE_LABEL, LE_LABEL];
|
||||
const API_TYPE_LABELS: [&str; 1] = [TYPE_LABEL];
|
||||
const API_SERVER_TYPE_LABELS: [&str; 2] = [SERVER_LABEL, TYPE_LABEL];
|
||||
|
||||
pub static API_REJECTED_AUTH_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
@@ -90,16 +81,7 @@ pub static API_REQUESTS_IN_FLIGHT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLoc
|
||||
new_gauge_md(
|
||||
MetricName::ApiRequestsInFlightTotal,
|
||||
"Total number of requests currently in flight",
|
||||
&API_NAME_TYPE_LABELS,
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
|
||||
pub static API_REQUESTS_IN_FLIGHT_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("requests_in_flight_total_by_server".to_string()),
|
||||
"Total number of requests currently in flight by server",
|
||||
&API_SERVER_NAME_TYPE_LABELS,
|
||||
&["name", "type"],
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
@@ -108,16 +90,7 @@ pub static API_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(||
|
||||
new_counter_md(
|
||||
MetricName::ApiRequestsTotal,
|
||||
"Total number of requests",
|
||||
&API_NAME_TYPE_LABELS,
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
|
||||
pub static API_REQUESTS_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::Custom("requests_total_by_server".to_string()),
|
||||
"Total number of requests by server",
|
||||
&API_SERVER_NAME_TYPE_LABELS,
|
||||
&["name", "type"],
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
@@ -126,16 +99,7 @@ pub static API_REQUESTS_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::
|
||||
new_counter_md(
|
||||
MetricName::ApiRequestsErrorsTotal,
|
||||
"Total number of requests with (4xx and 5xx) errors",
|
||||
&API_NAME_TYPE_LABELS,
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
|
||||
pub static API_REQUESTS_ERRORS_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::Custom("requests_errors_total_by_server".to_string()),
|
||||
"Total number of requests with (4xx and 5xx) errors by server",
|
||||
&API_SERVER_NAME_TYPE_LABELS,
|
||||
&["name", "type"],
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
@@ -144,16 +108,7 @@ pub static API_REQUESTS_5XX_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLo
|
||||
new_counter_md(
|
||||
MetricName::ApiRequests5xxErrorsTotal,
|
||||
"Total number of requests with 5xx errors",
|
||||
&API_NAME_TYPE_LABELS,
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
|
||||
pub static API_REQUESTS_5XX_ERRORS_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::Custom("requests_5xx_errors_total_by_server".to_string()),
|
||||
"Total number of requests with 5xx errors by server",
|
||||
&API_SERVER_NAME_TYPE_LABELS,
|
||||
&["name", "type"],
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
@@ -162,16 +117,7 @@ pub static API_REQUESTS_4XX_ERRORS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLo
|
||||
new_counter_md(
|
||||
MetricName::ApiRequests4xxErrorsTotal,
|
||||
"Total number of requests with 4xx errors",
|
||||
&API_NAME_TYPE_LABELS,
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
|
||||
pub static API_REQUESTS_4XX_ERRORS_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::Custom("requests_4xx_errors_total_by_server".to_string()),
|
||||
"Total number of requests with 4xx errors by server",
|
||||
&API_SERVER_NAME_TYPE_LABELS,
|
||||
&["name", "type"],
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
@@ -180,16 +126,7 @@ pub static API_REQUESTS_CANCELED_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock
|
||||
new_counter_md(
|
||||
MetricName::ApiRequestsCanceledTotal,
|
||||
"Total number of requests canceled by the client",
|
||||
&API_NAME_TYPE_LABELS,
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
|
||||
pub static API_REQUESTS_CANCELED_TOTAL_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::Custom("requests_canceled_total_by_server".to_string()),
|
||||
"Total number of requests canceled by the client by server",
|
||||
&API_SERVER_NAME_TYPE_LABELS,
|
||||
&["name", "type"],
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
@@ -198,16 +135,7 @@ pub static API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_MD: LazyLock<MetricDescriptor>
|
||||
new_counter_md(
|
||||
MetricName::ApiRequestsTTFBSecondsDistribution,
|
||||
"Distribution of time to first byte across API calls",
|
||||
&API_NAME_TYPE_LE_LABELS,
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
|
||||
pub static API_REQUESTS_TTFB_SECONDS_DISTRIBUTION_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::Custom("requests_ttfb_seconds_distribution_by_server".to_string()),
|
||||
"Distribution of time to first byte across API calls by server",
|
||||
&API_SERVER_NAME_TYPE_LE_LABELS,
|
||||
&["name", "type", "le"],
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
@@ -216,16 +144,7 @@ pub static API_TRAFFIC_SENT_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new
|
||||
new_counter_md(
|
||||
MetricName::ApiTrafficSentBytes,
|
||||
"Total number of bytes sent",
|
||||
&API_TYPE_LABELS,
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
|
||||
pub static API_TRAFFIC_SENT_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::Custom("traffic_sent_bytes_by_server".to_string()),
|
||||
"Total number of bytes sent by server",
|
||||
&API_SERVER_TYPE_LABELS,
|
||||
&["type"],
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
@@ -234,16 +153,7 @@ pub static API_TRAFFIC_RECV_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new
|
||||
new_counter_md(
|
||||
MetricName::ApiTrafficRecvBytes,
|
||||
"Total number of bytes received",
|
||||
&API_TYPE_LABELS,
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
|
||||
pub static API_TRAFFIC_RECV_BYTES_BY_SERVER_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::Custom("traffic_recv_bytes_by_server".to_string()),
|
||||
"Total number of bytes received by server",
|
||||
&API_SERVER_TYPE_LABELS,
|
||||
&["type"],
|
||||
MetricSubsystem::ApiRequests,
|
||||
)
|
||||
});
|
||||
|
||||
@@ -17,50 +17,6 @@
|
||||
use crate::{MetricDescriptor, MetricName, new_counter_md, new_gauge_md, subsystems};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub const SERVER_LABEL: &str = "server";
|
||||
pub const SOURCE_LABEL: &str = "source";
|
||||
pub const STATE_LABEL: &str = "state";
|
||||
pub const CYCLE_SCOPE_LABEL: &str = "cycle_scope";
|
||||
pub const BUCKET_LABEL: &str = "bucket";
|
||||
pub const DRIVE_LABEL: &str = "drive";
|
||||
pub const RESULT_LABEL: &str = "result";
|
||||
|
||||
pub static SCANNER_SOURCE_WORK_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::Custom("source_work_total".to_string()),
|
||||
"Total scanner work by source and state since server start",
|
||||
&[SERVER_LABEL, SOURCE_LABEL, STATE_LABEL],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CYCLE_SOURCE_WORK_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("cycle_source_work".to_string()),
|
||||
"Scanner work by cycle scope, source, and state",
|
||||
&[SERVER_LABEL, CYCLE_SCOPE_LABEL, SOURCE_LABEL, STATE_LABEL],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_BUCKET_DRIVE_RESULT_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::Custom("bucket_drive_result_total".to_string()),
|
||||
"Total scanner bucket-drive scan results by server, bucket, drive, and result",
|
||||
&[SERVER_LABEL, BUCKET_LABEL, DRIVE_LABEL, RESULT_LABEL],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_CYCLE_BUCKET_DRIVE_RESULT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::Custom("cycle_bucket_drive_result".to_string()),
|
||||
"Scanner bucket-drive scan results by cycle scope, server, bucket, drive, and result",
|
||||
&[SERVER_LABEL, CYCLE_SCOPE_LABEL, BUCKET_LABEL, DRIVE_LABEL, RESULT_LABEL],
|
||||
subsystems::SCANNER,
|
||||
)
|
||||
});
|
||||
|
||||
pub static SCANNER_BUCKET_SCANS_FINISHED_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::ScannerBucketScansFinished,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user