mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-09 21:56:03 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 houseme
|
ab7d3f9a7d | ||
|
houseme
|
c123ae2123 |
@@ -1,2 +1,2 @@
|
||||
sha256-linux=563bff8f1171d6dbe166ff8440310dbe98430e466aa3ecd8dc39e3c872b320f7
|
||||
sha256-darwin=563bff8f1171d6dbe166ff8440310dbe98430e466aa3ecd8dc39e3c872b320f7
|
||||
sha256-linux=4696a43b167ac608b3b8677027c9fe9fdac3396d37c8cca11dce531c720ac6d2
|
||||
sha256-darwin=9785867929047dfd8c6f768e0d2b1e0a8fdba85216f4a4139093b1619d03ff07
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=874c881d7b45f12378a5817c7f42c95c4981960a2ec9ce12dcf4af239ae1f9d5
|
||||
sha256-linux=9351e25b45bf7dfce18b951a5e3740225f457cacc53b8bf9f500f6947763ec0e
|
||||
sha256-darwin=f0c78fdb93471575d9a64c5c46eae6c806bdd0bc10a6e33d7fb574aabd8db5a3
|
||||
sha256-linux=03ed7016cab672de9320e31375a0358eceacb4408b0e79cf063614fa7c878b87
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=83a7dcaffd5a789517ae9f02a224f66a9713937885cff96fca2ad7e216f197ae
|
||||
sha256-linux=626c10f8c964507ff987b6c86069e9019dc6d2ae7fb02db9be5df5aa8cc5145b
|
||||
sha256-darwin=364f2329a7b72eb9f1608dbe1a3af37af4095354014f3cbe23ca448492d89961
|
||||
sha256-linux=60983f1ebe7068cf660d473c5f76c76a650410ccc99d71934ddca7fd67607987
|
||||
|
||||
@@ -89,7 +89,6 @@ offline-enrollment-e2e-check: core-deps ## Build and exercise the dedicated offl
|
||||
test-wiring-check: ## Check tests stay registered and selected by their intended runners
|
||||
@echo "🧪 Checking test wiring..."
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --check-workflow
|
||||
|
||||
.PHONY: log-analyzer-rules-check
|
||||
log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source
|
||||
|
||||
@@ -32,16 +32,6 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_hotpath_warp_ab_gate.sh
|
||||
./scripts/test_hotpath_warp_abba.sh
|
||||
./scripts/test_scanner_validation_harness.sh
|
||||
./scripts/test_scanner_heal_checkpoint_crash_evidence.sh
|
||||
./scripts/test_scanner_heal_authority_evidence.sh
|
||||
./scripts/test_scanner_heal_scoped_ack_evidence.sh
|
||||
./scripts/test_scanner_heal_legacy_rollback_evidence.sh
|
||||
./scripts/test_scanner_heal_g14_multiset_evidence.sh
|
||||
./scripts/test_scanner_heal_scheduler_pressure_evidence.sh
|
||||
./scripts/test_scanner_heal_status_outcome_evidence.sh
|
||||
./scripts/test_scanner_heal_maintenance_evidence.sh
|
||||
./scripts/test_scanner_heal_w13_mrf_evidence.sh
|
||||
./scripts/test_scanner_heal_w16_recovery_evidence.sh
|
||||
./scripts/test_exact_1mib_handoff_abba.sh
|
||||
./scripts/test_pinned_paired_abba_bench.sh
|
||||
./scripts/test_manual_transition_runbooks.sh
|
||||
@@ -49,7 +39,6 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_python_bin.sh
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
|
||||
|
||||
@@ -197,12 +197,6 @@ test-group = 'e2e-cluster-nightly'
|
||||
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
|
||||
test-group = 'e2e-vault'
|
||||
|
||||
# This four-disk, 65-member rollback probe already drives up to 32 concurrent
|
||||
# durable deletions. Reserve this nextest run's capacity for its progress oracle.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(=store::init::tests::dispatch_manifest_rollback_bounded_concurrency_reaches_tail_behind_slow_member)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ci profile — the strict CI gate (ci.yml `cargo nextest run --profile ci`)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -78,43 +78,6 @@
|
||||
"erasure": {"data_blocks": 8, "parity_blocks": 4},
|
||||
"erasure_set_drive_count": 12,
|
||||
"scope": "Target process killed during partial background rebuild on a single 3x4 EC8+4 set; real unclean-shutdown marker, exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool."
|
||||
},
|
||||
"background-target-restart-ec8-4-multi-set": {
|
||||
"gate": "G14",
|
||||
"task": "W21",
|
||||
"lane": "e2e-nightly",
|
||||
"suite": "e2e_test",
|
||||
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart",
|
||||
"oracle": "background-target-restart-ec8-4-multi-set.json",
|
||||
"evidence": "process-restart",
|
||||
"unclean_shutdown_marker": false,
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 3, "drives_per_node": 8},
|
||||
"erasure": {"data_blocks": 8, "parity_blocks": 4},
|
||||
"erasure_set_drive_count": 12,
|
||||
"sets": 2,
|
||||
"pools": 1,
|
||||
"scope": "Target process restart during partial background rebuild on a 3x8 EC8+4 layout with two erasure sets in one pool; exact unversioned S3 bodies and replacement-drive shards; not power loss or multi-pool."
|
||||
},
|
||||
"background-target-crash-ec8-4-multi-pool": {
|
||||
"gate": "G14",
|
||||
"task": "W21",
|
||||
"lane": "e2e-nightly",
|
||||
"suite": "e2e_test",
|
||||
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash",
|
||||
"oracle": "background-target-crash-ec8-4-multi-pool.json",
|
||||
"evidence": "process-crash-restart",
|
||||
"unclean_shutdown_marker": true,
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 3, "drives_per_node": 12},
|
||||
"erasure": {"data_blocks": 8, "parity_blocks": 4},
|
||||
"erasure_set_drive_count": 12,
|
||||
"sets": 3,
|
||||
"pools": 3,
|
||||
"outage_target_manifest_required": false,
|
||||
"scope": "Target process crash during partial background rebuild on three single-node EC8+4 pools; exact baseline S3 bodies and replacement-drive shards, with outage object verified through S3 but not forced onto the replaced target drive."
|
||||
}
|
||||
},
|
||||
"release_lanes": {
|
||||
@@ -171,11 +134,7 @@
|
||||
"lane": "authority-coverage",
|
||||
"status": "pending",
|
||||
"description": "Complete root and quota authority coverage",
|
||||
"requires": ["root authority evidence", "quota authority evidence"],
|
||||
"evidence_fields": [
|
||||
"root_authority_evidence",
|
||||
"quota_authority_evidence"
|
||||
]
|
||||
"requires": ["root authority evidence", "quota authority evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G02",
|
||||
@@ -183,11 +142,7 @@
|
||||
"lane": "checkpoint-and-crash",
|
||||
"status": "pending",
|
||||
"description": "Bounded checkpoint progress and independent version inventory",
|
||||
"requires": ["bounded checkpoint oracle", "independent version inventory"],
|
||||
"evidence_fields": [
|
||||
"bounded_checkpoint_oracle",
|
||||
"independent_version_inventory"
|
||||
]
|
||||
"requires": ["bounded checkpoint oracle", "independent version inventory"]
|
||||
},
|
||||
{
|
||||
"gate": "G03",
|
||||
@@ -209,11 +164,7 @@
|
||||
"lane": "checkpoint-and-crash",
|
||||
"status": "pending",
|
||||
"description": "Crash at every cache, root, floor and intent boundary",
|
||||
"requires": ["cache boundary crash evidence", "root/floor/intent crash evidence"],
|
||||
"evidence_fields": [
|
||||
"cache_boundary_crash_evidence",
|
||||
"root_floor_intent_crash_evidence"
|
||||
]
|
||||
"requires": ["cache boundary crash evidence", "root/floor/intent crash evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G05",
|
||||
@@ -221,11 +172,7 @@
|
||||
"lane": "status-and-outcome",
|
||||
"status": "pending",
|
||||
"description": "Per-object outcomes and bounded terminal retention",
|
||||
"requires": ["per-object outcome oracle", "terminal retention bounds"],
|
||||
"evidence_fields": [
|
||||
"per_object_outcome_oracle",
|
||||
"terminal_retention_bounds"
|
||||
]
|
||||
"requires": ["per-object outcome oracle", "terminal retention bounds"]
|
||||
},
|
||||
{
|
||||
"gate": "G06",
|
||||
@@ -233,12 +180,7 @@
|
||||
"lane": "status-and-outcome",
|
||||
"status": "pending",
|
||||
"description": "Concurrent status, legacy clients and truncation",
|
||||
"requires": ["concurrent status evidence", "legacy client compatibility", "truncation behavior"],
|
||||
"evidence_fields": [
|
||||
"concurrent_status_evidence",
|
||||
"legacy_client_compatibility",
|
||||
"truncation_behavior"
|
||||
]
|
||||
"requires": ["concurrent status evidence", "legacy client compatibility", "truncation behavior"]
|
||||
},
|
||||
{
|
||||
"gate": "G07",
|
||||
@@ -284,11 +226,7 @@
|
||||
"lane": "scheduler-pressure",
|
||||
"status": "pending",
|
||||
"description": "Bounded scheduling and pressure recovery",
|
||||
"requires": ["scheduler bound evidence", "pressure recovery evidence"],
|
||||
"evidence_fields": [
|
||||
"scheduler_bound_evidence",
|
||||
"pressure_recovery_evidence"
|
||||
]
|
||||
"requires": ["scheduler bound evidence", "pressure recovery evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G11",
|
||||
@@ -313,11 +251,7 @@
|
||||
"lane": "authority-coverage",
|
||||
"status": "pending",
|
||||
"description": "Both quota paths during reset and settlement",
|
||||
"requires": ["reset quota-path evidence", "settlement quota-path evidence"],
|
||||
"evidence_fields": [
|
||||
"reset_quota_path_evidence",
|
||||
"settlement_quota_path_evidence"
|
||||
]
|
||||
"requires": ["reset quota-path evidence", "settlement quota-path evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G13",
|
||||
@@ -325,12 +259,7 @@
|
||||
"lane": "maintenance-producers",
|
||||
"status": "pending",
|
||||
"description": "Quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
|
||||
"requires": ["quorum-minus-one matrix", "unknown-disk/remount matrix", "Object Lock dry-run grace evidence"],
|
||||
"evidence_fields": [
|
||||
"quorum_minus_one_matrix",
|
||||
"unknown_disk_remount_matrix",
|
||||
"object_lock_dry_run_grace_evidence"
|
||||
]
|
||||
"requires": ["quorum-minus-one matrix", "unknown-disk/remount matrix", "Object Lock dry-run grace evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "G14",
|
||||
@@ -359,12 +288,7 @@
|
||||
"lane": "scheduler-pressure",
|
||||
"status": "pending",
|
||||
"description": "Measured cold-walk share and foreground latency/throughput",
|
||||
"requires": ["cold-walk share measurement", "foreground latency/throughput measurement"],
|
||||
"evidence_fields": [
|
||||
"cold_walk_share_measurement",
|
||||
"foreground_latency_throughput_measurement",
|
||||
"profile_evidence"
|
||||
]
|
||||
"requires": ["cold-walk share measurement", "foreground latency/throughput measurement"]
|
||||
},
|
||||
{
|
||||
"gate": "P2",
|
||||
@@ -384,12 +308,7 @@
|
||||
"lane": "scheduler-pressure",
|
||||
"status": "pending",
|
||||
"description": "Measured two-hour pressure/heal capacity and recovery window",
|
||||
"requires": ["two-hour pressure measurement", "heal capacity measurement", "recovery-window measurement"],
|
||||
"evidence_fields": [
|
||||
"two_hour_pressure_measurement",
|
||||
"heal_capacity_measurement",
|
||||
"recovery_window_measurement"
|
||||
]
|
||||
"requires": ["two-hour pressure measurement", "heal capacity measurement", "recovery-window measurement"]
|
||||
},
|
||||
{
|
||||
"gate": "P4",
|
||||
@@ -399,9 +318,6 @@
|
||||
"description": "Measured MRF scale and replay cost with retained responsibility",
|
||||
"requires": ["MRF scale measurement", "MRF replay-cost measurement", "retained responsibility evidence", "cleanup/GC soak evidence"],
|
||||
"evidence_fields": [
|
||||
"mrf_scale_measurement",
|
||||
"mrf_replay_cost_measurement",
|
||||
"retained_responsibility_evidence",
|
||||
"mrf_cleanup_gc_soak_evidence"
|
||||
]
|
||||
},
|
||||
@@ -411,12 +327,7 @@
|
||||
"lane": "checkpoint-and-crash",
|
||||
"status": "pending",
|
||||
"description": "Fixed-budget real process restart through enumeration and classification",
|
||||
"requires": ["fixed-budget restart evidence", "enumeration evidence", "classification evidence"],
|
||||
"evidence_fields": [
|
||||
"fixed_budget_restart_evidence",
|
||||
"enumeration_evidence",
|
||||
"classification_evidence"
|
||||
]
|
||||
"requires": ["fixed-budget restart evidence", "enumeration evidence", "classification evidence"]
|
||||
},
|
||||
{
|
||||
"gate": "R-D",
|
||||
@@ -424,13 +335,7 @@
|
||||
"lane": "status-and-outcome",
|
||||
"status": "pending",
|
||||
"description": "Manager-to-event-to-ledger exact disposition, including grace",
|
||||
"requires": ["manager disposition evidence", "event disposition evidence", "ledger disposition evidence", "grace handling"],
|
||||
"evidence_fields": [
|
||||
"manager_disposition_evidence",
|
||||
"event_disposition_evidence",
|
||||
"ledger_disposition_evidence",
|
||||
"grace_handling"
|
||||
]
|
||||
"requires": ["manager disposition evidence", "event disposition evidence", "ledger disposition evidence", "grace handling"]
|
||||
},
|
||||
{
|
||||
"gate": "R-L",
|
||||
@@ -438,12 +343,7 @@
|
||||
"lane": "mixed-version-rollback",
|
||||
"status": "pending",
|
||||
"description": "Legacy source conflicts, migration gaps and crash-safe source retirement",
|
||||
"requires": ["legacy source-conflict evidence", "migration-gap evidence", "crash-safe source retirement evidence"],
|
||||
"evidence_fields": [
|
||||
"legacy_source_conflict_evidence",
|
||||
"migration_gap_evidence",
|
||||
"crash_safe_source_retirement_evidence"
|
||||
]
|
||||
"requires": ["legacy source-conflict evidence", "migration-gap evidence", "crash-safe source retirement evidence"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -43,7 +43,6 @@ services:
|
||||
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
|
||||
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
|
||||
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
|
||||
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
|
||||
# `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
|
||||
# should show richer nested spans during request-path verification.
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
|
||||
@@ -90,7 +89,6 @@ services:
|
||||
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
|
||||
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
|
||||
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
|
||||
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
|
||||
# `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
|
||||
# should show richer nested spans during request-path verification.
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
|
||||
@@ -137,7 +135,6 @@ services:
|
||||
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
|
||||
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
|
||||
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
|
||||
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
|
||||
# `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
|
||||
# should show richer nested spans during request-path verification.
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
|
||||
@@ -184,7 +181,6 @@ services:
|
||||
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
|
||||
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
|
||||
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
|
||||
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
|
||||
# `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
|
||||
# should show richer nested spans during request-path verification.
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
|
||||
|
||||
@@ -26,7 +26,6 @@ services:
|
||||
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
|
||||
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
|
||||
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
|
||||
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
|
||||
- RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
|
||||
- RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
|
||||
@@ -73,7 +72,6 @@ services:
|
||||
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
|
||||
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
|
||||
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
|
||||
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
|
||||
- RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
|
||||
- RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
|
||||
@@ -120,7 +118,6 @@ services:
|
||||
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
|
||||
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
|
||||
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
|
||||
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
|
||||
- RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
|
||||
- RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
|
||||
@@ -167,7 +164,6 @@ services:
|
||||
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
|
||||
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
|
||||
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
|
||||
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
|
||||
- RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
|
||||
- RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
|
||||
|
||||
@@ -201,7 +201,6 @@ services:
|
||||
- RUSTFS_ADDRESS=:9000
|
||||
- RUSTFS_CONSOLE_ENABLE=true
|
||||
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
|
||||
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=debug
|
||||
platform: linux/amd64
|
||||
ports:
|
||||
@@ -221,7 +220,6 @@ services:
|
||||
- RUSTFS_ADDRESS=:9000
|
||||
- RUSTFS_CONSOLE_ENABLE=true
|
||||
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
|
||||
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=debug
|
||||
platform: linux/amd64
|
||||
ports:
|
||||
@@ -241,7 +239,6 @@ services:
|
||||
- RUSTFS_ADDRESS=:9000
|
||||
- RUSTFS_CONSOLE_ENABLE=true
|
||||
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
|
||||
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=debug
|
||||
platform: linux/amd64
|
||||
ports:
|
||||
@@ -261,7 +258,6 @@ services:
|
||||
- RUSTFS_ADDRESS=:9000
|
||||
- RUSTFS_CONSOLE_ENABLE=true
|
||||
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
|
||||
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=debug
|
||||
platform: linux/amd64
|
||||
ports:
|
||||
|
||||
@@ -47,10 +47,6 @@ Three pre-built Grafana dashboards are included for monitoring RustFS GET perfor
|
||||
| **GET Resource Impact** | `grafana-get-resource-impact.json` | Monitors resource usage: concurrent requests, IO queue utilization, disk permit wait, RSS trend |
|
||||
| **Object Data Cache** | `grafana-object-data-cache.json` | Monitors the GET body cache (`rustfs_object_data_cache_*`): hit ratio, lookup/plan/fill outcomes, fill duration quantiles, hit vs fill throughput, entries/weighted bytes, inflight fills, memory-pressure skips, invalidations, and size-class breakdowns |
|
||||
|
||||
### Storage Metrics
|
||||
|
||||
Storage panels require `prometheus-rules/rustfs-storage.yml` and the cluster resource attribute. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for ownership, observer selection, freshness, and rolling upgrades.
|
||||
|
||||
### Prometheus Alert Rules
|
||||
|
||||
The file `prometheus-rules/rustfs-get-optimization-alerts.yaml` contains pre-configured alerting rules:
|
||||
|
||||
@@ -32,7 +32,6 @@ services:
|
||||
- RUSTFS_SECRET_KEY=rustfsadmin
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=info
|
||||
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
|
||||
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
|
||||
- RUSTFS_OBS_PROFILING_ENDPOINT=http://pyroscope:4040
|
||||
volumes:
|
||||
- rustfs-data:/data/rustfs
|
||||
|
||||
@@ -223,15 +223,14 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_buckets_total\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
|
||||
"expr": "sum(rustfs_cluster_buckets_total{job=~\"$job\"})",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Total Buckets",
|
||||
"type": "stat",
|
||||
"description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
@@ -290,15 +289,14 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_objects_total\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
|
||||
"expr": "sum(rustfs_cluster_objects_total{job=~\"$job\"})",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Total Objects",
|
||||
"type": "stat",
|
||||
"description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
@@ -429,7 +427,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_used_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
|
||||
"expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"})",
|
||||
"legendFormat": "Used",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
@@ -440,7 +438,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_raw_total_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
|
||||
"expr": "sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
|
||||
"hide": false,
|
||||
"legendFormat": "Total",
|
||||
"range": true,
|
||||
@@ -452,7 +450,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_used_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"} / ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_raw_total_bytes\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
|
||||
"expr": "sum(rustfs_cluster_capacity_used_bytes{job=~\"$job\"}) / sum(rustfs_cluster_capacity_raw_total_bytes{job=~\"$job\"})",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"legendFormat": "Percent",
|
||||
@@ -461,8 +459,7 @@
|
||||
}
|
||||
],
|
||||
"title": "Capacity",
|
||||
"type": "stat",
|
||||
"description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
@@ -528,7 +525,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_stale_drives\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
|
||||
"expr": "sum(rustfs_cluster_capacity_stale_drives{job=~\"$job\"})",
|
||||
"legendFormat": "Stale Drives",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
@@ -539,15 +536,14 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_capacity_missing_drives\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
|
||||
"expr": "sum(rustfs_cluster_capacity_missing_drives{job=~\"$job\"})",
|
||||
"legendFormat": "Missing Drives",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Capacity Observation",
|
||||
"type": "stat",
|
||||
"description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
@@ -1993,8 +1989,8 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
|
||||
"legendFormat": "{{server}} | {{drive}} (bytes)",
|
||||
"expr": "sum by (drive) (rustfs_system_drive_used_bytes{job=~\"$job\", drive=~\"$drive\"})",
|
||||
"legendFormat": "{{drive}} (bytes)",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
@@ -2004,17 +2000,16 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"} / ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_system_drive_total_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
|
||||
"expr": "sum by (drive) (rustfs_system_drive_used_bytes{job=~\"$job\", drive=~\"$drive\"}) / sum by (drive)(rustfs_system_drive_total_bytes{job=~\"$job\", drive=~\"$drive\"})",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"legendFormat": "{{server}} | {{drive}} (percent)",
|
||||
"legendFormat": "{{drive}} (percent)",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "System Drive Usage",
|
||||
"type": "timeseries",
|
||||
"description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
@@ -2102,15 +2097,14 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_age_seconds\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
|
||||
"legendFormat": "{{server}} | {{drive}}",
|
||||
"expr": "max by (drive) (rustfs_system_drive_capacity_observation_age_seconds{job=~\"$job\", drive=~\"$drive\"})",
|
||||
"legendFormat": "{{drive}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Drive Capacity Observation Age",
|
||||
"type": "timeseries",
|
||||
"description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
@@ -2196,8 +2190,8 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_state\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",state=\"stale\"}",
|
||||
"legendFormat": "{{server}} | {{drive}} stale",
|
||||
"expr": "max by (drive) (rustfs_system_drive_capacity_observation_state{job=~\"$job\", drive=~\"$drive\", state=\"stale\"})",
|
||||
"legendFormat": "{{drive}} stale",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
@@ -2207,15 +2201,14 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_system_drive_capacity_observation_state\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",state=\"missing\"}",
|
||||
"legendFormat": "{{server}} | {{drive}} missing",
|
||||
"expr": "max by (drive) (rustfs_system_drive_capacity_observation_state{job=~\"$job\", drive=~\"$drive\", state=\"missing\"})",
|
||||
"legendFormat": "{{drive}} missing",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Drive Capacity Observation State",
|
||||
"type": "timeseries",
|
||||
"description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
@@ -4558,7 +4551,7 @@
|
||||
"index": 0,
|
||||
"text": "INACTIVE"
|
||||
},
|
||||
"to": 1e-09
|
||||
"to": 1e-9
|
||||
},
|
||||
"type": "range"
|
||||
}
|
||||
@@ -6570,7 +6563,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_online_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
|
||||
"expr": "rustfs_cluster_health_drives_online_count{job=~\"$job\"}",
|
||||
"legendFormat": "online - {{job}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
@@ -6581,7 +6574,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_offline_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
|
||||
"expr": "rustfs_cluster_health_drives_offline_count{job=~\"$job\"}",
|
||||
"legendFormat": "offline - {{job}}",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
@@ -6592,15 +6585,14 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_health_drives_count\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
|
||||
"expr": "rustfs_cluster_health_drives_count{job=~\"$job\"}",
|
||||
"legendFormat": "total - {{job}}",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"title": "Cluster Drive Health Counts",
|
||||
"type": "timeseries",
|
||||
"description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
@@ -8562,15 +8554,14 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=~\"rustfs_system_drive_.*\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\"}",
|
||||
"legendFormat": "{{server}} | {{source_metric}} | {{drive}}",
|
||||
"expr": "{__name__=~\"rustfs_system_drive_.*\",job=~\"$job\",drive=~\"$drive\"}",
|
||||
"legendFormat": "{{__name__}} | {{drive}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "System Drive (All)",
|
||||
"type": "timeseries",
|
||||
"description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
@@ -8951,15 +8942,14 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "rustfs:storage:current{source_metric=~\"rustfs_cluster_erasure_set_.*\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\"}",
|
||||
"legendFormat": "{{source_metric}}",
|
||||
"expr": "{__name__=~\"rustfs_cluster_erasure_set_.*\",job=~\"$job\"}",
|
||||
"legendFormat": "{{__name__}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Cluster Erasure Set (All)",
|
||||
"type": "timeseries",
|
||||
"description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
@@ -11726,7 +11716,7 @@
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_drive_runtime_state\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\",drive=~\"$drive\"}",
|
||||
"expr": "max by (server, drive, pool_index, set_index, drive_index, state) (rustfs_system_drive_runtime_state{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})",
|
||||
"legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{state}}"
|
||||
},
|
||||
{
|
||||
@@ -11737,13 +11727,12 @@
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"expr": "rustfs:storage:current{source_metric=\"rustfs_cluster_drive_offline_duration_seconds\",collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",observer=\"$storage_observer\",drive=~\"$drive\"}",
|
||||
"expr": "max by (server, drive, pool_index, set_index, drive_index) (rustfs_system_drive_offline_duration_seconds{job=~\"$job\",server=~\"$server\",drive=~\"$drive\"})",
|
||||
"legendFormat": "{{server}} | {{drive}} | offline seconds"
|
||||
}
|
||||
],
|
||||
"title": "Observed Cluster Drive State and Offline Duration",
|
||||
"type": "timeseries",
|
||||
"description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
|
||||
"title": "Drive Runtime State and Offline Duration",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
@@ -11836,13 +11825,12 @@
|
||||
"editorMode": "code",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"expr": "rate(rustfs_system_drive_api_calls_total{collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}[$__rate_interval]) and ignoring(source_metric) rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}",
|
||||
"expr": "sum by (server, drive, pool_index, set_index, drive_index, api) (rate(rustfs_system_drive_api_calls_total{job=~\"$job\",server=~\"$server\",drive=~\"$drive\",api=~\"$drive_api\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{server}} | {{drive}} | p{{pool_index}}/s{{set_index}}/d{{drive_index}} | {{api}}"
|
||||
}
|
||||
],
|
||||
"title": "Drive API Calls by Operation",
|
||||
"type": "timeseries",
|
||||
"description": "Storage snapshot rules are required. Local details come from each drive owner. Global values use the selected fresh cluster observer. Missing or expired observations show no data; select another observer if needed."
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
@@ -12388,50 +12376,6 @@
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"current": {},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values(rustfs:storage_snapshot:fresh, rustfs_cluster_id)",
|
||||
"includeAll": false,
|
||||
"label": "Storage cluster",
|
||||
"multi": false,
|
||||
"name": "storage_cluster",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(rustfs:storage_snapshot:fresh, rustfs_cluster_id)",
|
||||
"refId": "PrometheusVariableQueryEditor-storage_cluster"
|
||||
},
|
||||
"refresh": 2,
|
||||
"regex": "",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"current": {},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "query_result(rustfs:storage_snapshot:fresh{collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\"})",
|
||||
"includeAll": false,
|
||||
"label": "Cluster observer",
|
||||
"multi": false,
|
||||
"name": "storage_observer",
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 3,
|
||||
"query": "query_result(rustfs:storage_snapshot:fresh{collection_scope=\"cluster\",rustfs_cluster_id=\"$storage_cluster\",job=~\"$job\"})",
|
||||
"refId": "PrometheusVariableQueryEditor-storage_observer"
|
||||
},
|
||||
"refresh": 2,
|
||||
"regex": "/observer=\"([^\"]+)\"/",
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
},
|
||||
{
|
||||
"allValue": ".*",
|
||||
"current": {
|
||||
@@ -12442,7 +12386,7 @@
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"definition": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, api)",
|
||||
"definition": "label_values(rustfs_system_drive_api_calls_total,api)",
|
||||
"includeAll": true,
|
||||
"label": "Drive API",
|
||||
"multi": true,
|
||||
@@ -12450,7 +12394,7 @@
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, api)",
|
||||
"query": "label_values(rustfs_system_drive_api_calls_total,api)",
|
||||
"refId": "PrometheusVariableQueryEditor-drive_api"
|
||||
},
|
||||
"refresh": 2,
|
||||
@@ -12511,7 +12455,7 @@
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"definition": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, drive)",
|
||||
"definition": "label_values(rustfs_system_drive_used_bytes,drive)",
|
||||
"includeAll": true,
|
||||
"label": "Drive",
|
||||
"multi": true,
|
||||
@@ -12519,7 +12463,7 @@
|
||||
"options": [],
|
||||
"query": {
|
||||
"qryType": 1,
|
||||
"query": "label_values(rustfs:storage:current{source_metric=\"rustfs_system_drive_used_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"$storage_cluster\"}, drive)",
|
||||
"query": "label_values(rustfs_system_drive_used_bytes,drive)",
|
||||
"refId": "PrometheusVariableQueryEditor-VariableQuery"
|
||||
},
|
||||
"refresh": 2,
|
||||
|
||||
@@ -1,575 +0,0 @@
|
||||
# Preserve OTLP timestamps in the Collector. Apply timestamp() directly to
|
||||
# each raw selector, before label rewriting; functions such as label_replace()
|
||||
# replace the evaluation timestamp and would make a cached value look new.
|
||||
# Compare publication times at Prometheus millisecond precision so points
|
||||
# published and exported within the same millisecond are not withheld.
|
||||
groups:
|
||||
- name: rustfs-storage-snapshots
|
||||
interval: 15s
|
||||
rules:
|
||||
- record: rustfs:storage_snapshot:fresh
|
||||
expr: |
|
||||
rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}
|
||||
and ((time() - rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}) <= rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"})
|
||||
and ((time() - rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"}) >= 0)
|
||||
and (rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id!="",collection_scope=~"local|cluster"} > 0)
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_buckets_total
|
||||
expr: |
|
||||
rustfs_cluster_buckets_total{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_buckets_total{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_capacity_free_bytes
|
||||
expr: |
|
||||
rustfs_cluster_capacity_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_capacity_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_capacity_missing_drives
|
||||
expr: |
|
||||
rustfs_cluster_capacity_missing_drives{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_capacity_missing_drives{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_capacity_raw_total_bytes
|
||||
expr: |
|
||||
rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_capacity_stale_drives
|
||||
expr: |
|
||||
rustfs_cluster_capacity_stale_drives{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_capacity_stale_drives{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_capacity_usable_total_bytes
|
||||
expr: |
|
||||
rustfs_cluster_capacity_usable_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_capacity_usable_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_capacity_used_bytes
|
||||
expr: |
|
||||
rustfs_cluster_capacity_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_capacity_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_drive_capacity_observation_age_seconds
|
||||
expr: |
|
||||
rustfs_cluster_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_drive_capacity_observation_state
|
||||
expr: |
|
||||
rustfs_cluster_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_drive_free_bytes
|
||||
expr: |
|
||||
rustfs_cluster_drive_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_drive_free_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_drive_offline_duration_seconds
|
||||
expr: |
|
||||
rustfs_cluster_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_drive_present
|
||||
expr: |
|
||||
rustfs_cluster_drive_present{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_drive_present{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_drive_runtime_state
|
||||
expr: |
|
||||
rustfs_cluster_drive_runtime_state{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_drive_runtime_state{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_drive_total_bytes
|
||||
expr: |
|
||||
rustfs_cluster_drive_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_drive_total_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_drive_used_bytes
|
||||
expr: |
|
||||
rustfs_cluster_drive_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_drive_used_bytes{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_erasure_set_data_shards
|
||||
expr: |
|
||||
rustfs_cluster_erasure_set_data_shards{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_erasure_set_data_shards{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_erasure_set_healing_drives_count
|
||||
expr: |
|
||||
rustfs_cluster_erasure_set_healing_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_erasure_set_healing_drives_count{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_erasure_set_health
|
||||
expr: |
|
||||
rustfs_cluster_erasure_set_health{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_erasure_set_health{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_erasure_set_online_drives_count
|
||||
expr: |
|
||||
rustfs_cluster_erasure_set_online_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_erasure_set_online_drives_count{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_erasure_set_overall_health
|
||||
expr: |
|
||||
rustfs_cluster_erasure_set_overall_health{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_erasure_set_overall_health{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_erasure_set_overall_write_quorum
|
||||
expr: |
|
||||
rustfs_cluster_erasure_set_overall_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_erasure_set_overall_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_erasure_set_parity
|
||||
expr: |
|
||||
rustfs_cluster_erasure_set_parity{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_erasure_set_parity{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_erasure_set_read_health
|
||||
expr: |
|
||||
rustfs_cluster_erasure_set_read_health{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_erasure_set_read_health{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_erasure_set_read_quorum
|
||||
expr: |
|
||||
rustfs_cluster_erasure_set_read_quorum{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_erasure_set_read_quorum{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_erasure_set_read_tolerance
|
||||
expr: |
|
||||
rustfs_cluster_erasure_set_read_tolerance{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_erasure_set_read_tolerance{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_erasure_set_size
|
||||
expr: |
|
||||
rustfs_cluster_erasure_set_size{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_erasure_set_size{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_erasure_set_write_health
|
||||
expr: |
|
||||
rustfs_cluster_erasure_set_write_health{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_erasure_set_write_health{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_erasure_set_write_quorum
|
||||
expr: |
|
||||
rustfs_cluster_erasure_set_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_erasure_set_write_quorum{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_erasure_set_write_tolerance
|
||||
expr: |
|
||||
rustfs_cluster_erasure_set_write_tolerance{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_erasure_set_write_tolerance{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_health_drives_count
|
||||
expr: |
|
||||
rustfs_cluster_health_drives_count{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_health_drives_count{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_health_drives_offline_count
|
||||
expr: |
|
||||
rustfs_cluster_health_drives_offline_count{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_health_drives_offline_count{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_health_drives_online_count
|
||||
expr: |
|
||||
rustfs_cluster_health_drives_online_count{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_health_drives_online_count{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_cluster_objects_total
|
||||
expr: |
|
||||
rustfs_cluster_objects_total{rustfs_cluster_id!="",collection_scope="cluster"}
|
||||
and (timestamp(rustfs_cluster_objects_total{rustfs_cluster_id!="",collection_scope="cluster"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_node_disk_free_bytes
|
||||
expr: |
|
||||
rustfs_node_disk_free_bytes{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_node_disk_free_bytes{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_node_disk_total_bytes
|
||||
expr: |
|
||||
rustfs_node_disk_total_bytes{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_node_disk_total_bytes{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_node_disk_used_bytes
|
||||
expr: |
|
||||
rustfs_node_disk_used_bytes{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_node_disk_used_bytes{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_api_calls_total
|
||||
expr: |
|
||||
rustfs_system_drive_api_calls_total{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_api_calls_total{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_api_latency_by_api_micros
|
||||
expr: |
|
||||
rustfs_system_drive_api_latency_by_api_micros{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_api_latency_by_api_micros{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_api_latency_micros
|
||||
expr: |
|
||||
rustfs_system_drive_api_latency_micros{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_api_latency_micros{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_availability_errors_total
|
||||
expr: |
|
||||
rustfs_system_drive_availability_errors_total{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_availability_errors_total{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_capacity_observation_age_seconds
|
||||
expr: |
|
||||
rustfs_system_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_capacity_observation_age_seconds{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_capacity_observation_state
|
||||
expr: |
|
||||
rustfs_system_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_capacity_observation_state{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_count
|
||||
expr: |
|
||||
rustfs_system_drive_count{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_count{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_deletes_total
|
||||
expr: |
|
||||
rustfs_system_drive_deletes_total{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_deletes_total{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_free_bytes
|
||||
expr: |
|
||||
rustfs_system_drive_free_bytes{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_free_bytes{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_free_inodes
|
||||
expr: |
|
||||
rustfs_system_drive_free_inodes{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_free_inodes{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_healing
|
||||
expr: |
|
||||
rustfs_system_drive_healing{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_healing{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_health
|
||||
expr: |
|
||||
rustfs_system_drive_health{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_health{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_info
|
||||
expr: |
|
||||
rustfs_system_drive_info{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_info{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_io_errors_total
|
||||
expr: |
|
||||
rustfs_system_drive_io_errors_total{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_io_errors_total{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_offline_count
|
||||
expr: |
|
||||
rustfs_system_drive_offline_count{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_offline_count{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_offline_duration_seconds
|
||||
expr: |
|
||||
rustfs_system_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_offline_duration_seconds{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_online_count
|
||||
expr: |
|
||||
rustfs_system_drive_online_count{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_online_count{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_perc_util
|
||||
expr: |
|
||||
rustfs_system_drive_perc_util{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_perc_util{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_present
|
||||
expr: |
|
||||
rustfs_system_drive_present{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_present{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_reads_await
|
||||
expr: |
|
||||
rustfs_system_drive_reads_await{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_reads_await{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_reads_kb_per_sec
|
||||
expr: |
|
||||
rustfs_system_drive_reads_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_reads_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_reads_per_sec
|
||||
expr: |
|
||||
rustfs_system_drive_reads_per_sec{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_reads_per_sec{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_runtime_state
|
||||
expr: |
|
||||
rustfs_system_drive_runtime_state{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_runtime_state{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_scanning
|
||||
expr: |
|
||||
rustfs_system_drive_scanning{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_scanning{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_timeout_errors_total
|
||||
expr: |
|
||||
rustfs_system_drive_timeout_errors_total{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_timeout_errors_total{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_total_bytes
|
||||
expr: |
|
||||
rustfs_system_drive_total_bytes{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_total_bytes{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_total_inodes
|
||||
expr: |
|
||||
rustfs_system_drive_total_inodes{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_total_inodes{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_used_bytes
|
||||
expr: |
|
||||
rustfs_system_drive_used_bytes{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_used_bytes{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_used_inodes
|
||||
expr: |
|
||||
rustfs_system_drive_used_inodes{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_used_inodes{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_waiting_io
|
||||
expr: |
|
||||
rustfs_system_drive_waiting_io{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_waiting_io{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_writes_await
|
||||
expr: |
|
||||
rustfs_system_drive_writes_await{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_writes_await{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_writes_kb_per_sec
|
||||
expr: |
|
||||
rustfs_system_drive_writes_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_writes_kb_per_sec{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_writes_per_sec
|
||||
expr: |
|
||||
rustfs_system_drive_writes_per_sec{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_writes_per_sec{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
- record: rustfs:storage:current
|
||||
labels:
|
||||
source_metric: rustfs_system_drive_writes_total
|
||||
expr: |
|
||||
rustfs_system_drive_writes_total{rustfs_cluster_id!="",collection_scope="local"}
|
||||
and (timestamp(rustfs_system_drive_writes_total{rustfs_cluster_id!="",collection_scope="local"})
|
||||
>= on (rustfs_cluster_id, observer, collection_scope, job, instance) group_left()
|
||||
(floor(rustfs:storage_snapshot:fresh * 1000) / 1000))
|
||||
@@ -1,189 +0,0 @@
|
||||
rule_files:
|
||||
- ../prometheus-rules/rustfs-storage.yml
|
||||
evaluation_interval: 15s
|
||||
tests:
|
||||
- name: owners, pools, clusters and observer views remain distinct in mixed versions
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 0+60x8
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 180x8
|
||||
- series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",pool_index="0",set_index="0",drive_index="0"}
|
||||
values: 100x8
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 0+60x8
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 180x8
|
||||
- series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",pool_index="0",set_index="0",drive_index="1"}
|
||||
values: 100x8
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 0+60x8
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 180x8
|
||||
- series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n2",collection_scope="local",job="rustfs",instance="collector",server="n2",drive="/data",pool_index="1",set_index="0",drive_index="0"}
|
||||
values: 100x8
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 0+60x8
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 180x8
|
||||
- series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n3",collection_scope="local",job="rustfs",instance="collector",server="n3",drive="/data",pool_index="1",set_index="0",drive_index="1"}
|
||||
values: 100x8
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 0+60x8
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 180x8
|
||||
- series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",pool_index="0",set_index="0",drive_index="0"}
|
||||
values: 1000x8
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 0+60x8
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 180x8
|
||||
- series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",pool_index="0",set_index="0",drive_index="1"}
|
||||
values: 1000x8
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 0+60x8
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 180x8
|
||||
- series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n2",collection_scope="local",job="rustfs",instance="collector",server="n2",drive="/data",pool_index="1",set_index="0",drive_index="0"}
|
||||
values: 1000x8
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 0+60x8
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 180x8
|
||||
- series: rustfs_system_drive_total_bytes{rustfs_cluster_id="b",observer="n3",collection_scope="local",job="rustfs",instance="collector",server="n3",drive="/data",pool_index="1",set_index="0",drive_index="1"}
|
||||
values: 1000x8
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"}
|
||||
values: 0+60x8
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"}
|
||||
values: 180x8
|
||||
- series: rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id="a",observer="n0",collection_scope="cluster",job="rustfs",instance="collector"}
|
||||
values: 400x8
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"}
|
||||
values: 0+60x8
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"}
|
||||
values: 180x8
|
||||
- series: rustfs_cluster_capacity_raw_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="cluster",job="rustfs",instance="collector"}
|
||||
values: 300x8
|
||||
- series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="old",server="n0",drive="/data",job="rustfs",instance="collector"}
|
||||
values: 999x8
|
||||
promql_expr_test:
|
||||
- expr: sum by (rustfs_cluster_id) (rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes",collection_scope="local"})
|
||||
eval_time: 2m
|
||||
exp_samples:
|
||||
- labels: '{rustfs_cluster_id="a"}'
|
||||
value: 400
|
||||
- labels: '{rustfs_cluster_id="b"}'
|
||||
value: 4000
|
||||
- expr: count by (rustfs_cluster_id,pool_index) (rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes"})
|
||||
eval_time: 2m
|
||||
exp_samples:
|
||||
- labels: '{rustfs_cluster_id="a",pool_index="0"}'
|
||||
value: 2
|
||||
- labels: '{rustfs_cluster_id="a",pool_index="1"}'
|
||||
value: 2
|
||||
- labels: '{rustfs_cluster_id="b",pool_index="0"}'
|
||||
value: 2
|
||||
- labels: '{rustfs_cluster_id="b",pool_index="1"}'
|
||||
value: 2
|
||||
- expr: sum(rustfs:storage:current{source_metric="rustfs_cluster_capacity_raw_total_bytes",rustfs_cluster_id="a",observer="n0"})
|
||||
eval_time: 2m
|
||||
exp_samples:
|
||||
- labels: '{}'
|
||||
value: 400
|
||||
- expr: sum(rustfs:storage:current{source_metric="rustfs_cluster_capacity_raw_total_bytes",rustfs_cluster_id="a",observer="n1"})
|
||||
eval_time: 2m
|
||||
exp_samples:
|
||||
- labels: '{}'
|
||||
value: 300
|
||||
- name: removed values and identities cannot rejoin new snapshots; stalled sources expire
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 0+60x8
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 180x8
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 0 60 _ _ _ _ _ _ _
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 180x8
|
||||
- series: rustfs_system_drive_info{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="old"}
|
||||
values: 1 1 _ _ _ _ _ _ _
|
||||
- series: rustfs_system_drive_info{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="new"}
|
||||
values: _ _ 1 1 1 1 1 1 1
|
||||
- series: rustfs_system_drive_waiting_io{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data"}
|
||||
values: 7 7 _ _ _ _ _ _ _
|
||||
- series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data"}
|
||||
values: 100 100 _ _ _ _ _ _ _
|
||||
promql_expr_test:
|
||||
- expr: rustfs:storage:current{source_metric="rustfs_system_drive_info",disk_id="old"}
|
||||
eval_time: 2m
|
||||
exp_samples: []
|
||||
- expr: rustfs:storage:current{source_metric="rustfs_system_drive_waiting_io"}
|
||||
eval_time: 2m
|
||||
exp_samples: []
|
||||
- expr: count(rustfs:storage:current{source_metric="rustfs_system_drive_info",disk_id="new"})
|
||||
eval_time: 2m
|
||||
exp_samples:
|
||||
- labels: '{}'
|
||||
value: 1
|
||||
- expr: rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes"}
|
||||
eval_time: 5m
|
||||
exp_samples: []
|
||||
- expr: rustfs:storage_snapshot:fresh{observer="n1"}
|
||||
eval_time: 5m
|
||||
exp_samples: []
|
||||
- name: counters reset independently; filter versions before rate and sum rates across owners
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 0+60x8
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 180x8
|
||||
- series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",observer="n0",collection_scope="local",job="rustfs",instance="collector",server="n0",drive="/data",disk_id="n0-disk",api="read_all"}
|
||||
values: 0 60 120 30 90 150 210
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 0+60x8
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 180x8
|
||||
- series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",observer="n1",collection_scope="local",job="rustfs",instance="collector",server="n1",drive="/data",disk_id="n1-disk",api="read_all"}
|
||||
values: 0+120x6
|
||||
- series: rustfs_system_drive_api_calls_total{rustfs_cluster_id="a",server="n0",drive="/data",observer="old"}
|
||||
values: 0+999x6
|
||||
promql_expr_test:
|
||||
- expr: sum(resets(rustfs_system_drive_api_calls_total{collection_scope="local"}[5m]))
|
||||
eval_time: 5m
|
||||
exp_samples:
|
||||
- labels: '{}'
|
||||
value: 1
|
||||
- expr: sum(rate(rustfs_system_drive_api_calls_total{collection_scope="local"}[2m]) and ignoring(source_metric) rustfs:storage:current{source_metric="rustfs_system_drive_api_calls_total"})
|
||||
eval_time: 5m
|
||||
exp_samples:
|
||||
- labels: '{}'
|
||||
value: 3
|
||||
- name: completed slow collection has no validity budget
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="slow",collection_scope="cluster",job="rustfs",instance="collector"}
|
||||
values: 0+60x3
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="slow",collection_scope="cluster",job="rustfs",instance="collector"}
|
||||
values: '0x3'
|
||||
promql_expr_test:
|
||||
- expr: rustfs:storage_snapshot:fresh{observer="slow"}
|
||||
eval_time: 2m
|
||||
exp_samples: []
|
||||
- name: submillisecond publication and millisecond OTLP samples share a cutoff
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: rustfs_storage_snapshot_last_success_timestamp_seconds{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 0.0009+60x3
|
||||
- series: rustfs_storage_snapshot_max_age_seconds{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector"}
|
||||
values: 180x3
|
||||
- series: rustfs_system_drive_total_bytes{rustfs_cluster_id="a",observer="submillisecond",collection_scope="local",job="rustfs",instance="collector",server="submillisecond",drive="/data"}
|
||||
values: 100x3
|
||||
promql_expr_test:
|
||||
- expr: count(rustfs:storage:current{source_metric="rustfs_system_drive_total_bytes",observer="submillisecond"})
|
||||
eval_time: 2m15s
|
||||
exp_samples:
|
||||
- labels: '{}'
|
||||
value: 1
|
||||
@@ -111,6 +111,10 @@ runs:
|
||||
shell: bash
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
- name: Check CI paths stay in sync
|
||||
shell: bash
|
||||
run: ./scripts/check_ci_paths_sync.sh
|
||||
|
||||
- name: Check io_uring lane --lib precondition
|
||||
shell: bash
|
||||
run: ./scripts/check_uring_lane_lib_only.sh
|
||||
|
||||
@@ -55,10 +55,6 @@ secret_key = ${S3_SECRET_KEY}
|
||||
|
||||
## replace with key id obtained when secret is created, or delete if KMS not tested
|
||||
#kms_keyid = 01234567-89ab-cdef-0123-456789abcdef
|
||||
#kms_keyid2 = fedcba98-7654-3210-fedc-ba9876543210
|
||||
|
||||
## Expected service default for SSE-KMS requests without a key id; empty means none
|
||||
#kms_default_keyid =
|
||||
|
||||
## Storage classes
|
||||
#storage_classes = "LUKEWARM, FROZEN"
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Reports the existing required checks for paths excluded by ci.yml.
|
||||
# Mixed PRs can trigger both workflows; their Quick Checks jobs use one shared
|
||||
# action to keep validation coverage aligned. Keep this paths list in sync with
|
||||
# ci.yml's pull_request.paths-ignore via scripts/check_ci_paths_sync.sh.
|
||||
|
||||
name: Continuous Integration (docs only)
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [ opened, synchronize, reopened ]
|
||||
branches: [ main, release ]
|
||||
paths:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
- "deploy/**"
|
||||
- "scripts/dev_*.sh"
|
||||
- "scripts/probe.sh"
|
||||
- "LICENSE*"
|
||||
- ".gitignore"
|
||||
- ".dockerignore"
|
||||
- "README*"
|
||||
- "**/*.png"
|
||||
- "**/*.jpg"
|
||||
- "**/*.svg"
|
||||
- ".github/workflows/build.yml"
|
||||
- ".github/workflows/docker.yml"
|
||||
- ".github/workflows/audit.yml"
|
||||
- "flake.lock"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
quick-checks:
|
||||
name: Quick Checks
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run shared quick checks
|
||||
uses: ./.github/actions/quick-checks
|
||||
|
||||
test-and-lint:
|
||||
name: Test and Lint
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Docs-only PRs skip the full code CI, but they are exactly where a
|
||||
# planning-type document could be slipped in (git add -f bypasses
|
||||
# .gitignore). Run the guard here so the required "Test and Lint" check
|
||||
# stays meaningful for docs-only changes.
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
- name: Satisfy required check for docs-only changes
|
||||
run: echo "Docs-only change — code CI is skipped by paths-ignore; planning-docs guard passed, reporting success for the required 'Test and Lint' check."
|
||||
+74
-77
@@ -37,6 +37,25 @@ on:
|
||||
pull_request:
|
||||
types: [ opened, synchronize, reopened, closed ]
|
||||
branches: [ main, release ]
|
||||
# Keep this list in sync with the `paths` list in ci-docs-only.yml, which
|
||||
# reports the required "Test and Lint" check for PRs skipped here.
|
||||
paths-ignore:
|
||||
- "**.md"
|
||||
- "docs/**"
|
||||
- "deploy/**"
|
||||
- "scripts/dev_*.sh"
|
||||
- "scripts/probe.sh"
|
||||
- "LICENSE*"
|
||||
- ".gitignore"
|
||||
- ".dockerignore"
|
||||
- "README*"
|
||||
- "**/*.png"
|
||||
- "**/*.jpg"
|
||||
- "**/*.svg"
|
||||
- ".github/workflows/build.yml"
|
||||
- ".github/workflows/docker.yml"
|
||||
- ".github/workflows/audit.yml"
|
||||
- "flake.lock"
|
||||
merge_group:
|
||||
types: [ checks_requested ]
|
||||
schedule:
|
||||
@@ -69,32 +88,6 @@ jobs:
|
||||
- name: Explain cancellation run
|
||||
run: echo "PR closed; this run only cancels older runs in the same concurrency group."
|
||||
|
||||
classify-changes:
|
||||
name: Select CI scope
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
mode: ${{ steps.scope.outputs.mode }}
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
fetch-depth: 2
|
||||
persist-credentials: false
|
||||
- name: Select scope using the base revision's policy
|
||||
id: scope
|
||||
env:
|
||||
CI_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
elif [[ "$CI_BASE_SHA" =~ ^[0-9a-f]{40}$ ]] && git show "$CI_BASE_SHA:scripts/ci_gate.py" > "$RUNNER_TEMP/ci-gate-base.py"; then
|
||||
python3 -I "$RUNNER_TEMP/ci-gate-base.py" select
|
||||
else
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
|
||||
typos:
|
||||
name: Typos
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
@@ -107,7 +100,7 @@ jobs:
|
||||
- name: Typos check with custom config file
|
||||
uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
|
||||
|
||||
# Fail early with compile-free checks for every pull request.
|
||||
# Fail early with compile-free checks shared with docs-only CI.
|
||||
quick-checks:
|
||||
name: Quick Checks
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
@@ -123,9 +116,9 @@ jobs:
|
||||
uses: ./.github/actions/quick-checks
|
||||
|
||||
test-and-lint:
|
||||
name: Workspace Test and Lint
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
name: Test and Lint
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
@@ -296,6 +289,45 @@ jobs:
|
||||
- name: Run rebalance/decommission migration proofs
|
||||
run: ./scripts/check_migration_gate_count.sh
|
||||
|
||||
# Record the reason before this job completes as FAILURE. A separate
|
||||
# dependent job cancels sibling lanes only after GitHub has preserved this
|
||||
# required check's failure verdict.
|
||||
- name: Annotate early-stop reason
|
||||
if: >-
|
||||
failure() && github.event_name == 'pull_request'
|
||||
&& github.event.pull_request.head.repo.full_name == github.repository
|
||||
run: |
|
||||
{
|
||||
echo "## CI early-stop"
|
||||
echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; a follow-up job will cancel sibling lanes to free runners."
|
||||
echo "Sibling jobs showing **cancelled** were stopped by the early-stop follow-up, not by their own failure."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Preserve the required Test and Lint FAILURE verdict before stopping sibling
|
||||
# lanes. Cancelling from inside test-and-lint changed its own conclusion to
|
||||
# CANCELLED and hid the actionable failure in the PR checks UI.
|
||||
cancel-after-test-and-lint-failure:
|
||||
name: Cancel siblings after Test and Lint failure
|
||||
if: >-
|
||||
failure() && needs.test-and-lint.result == 'failure'
|
||||
&& github.event_name == 'pull_request'
|
||||
&& github.event.pull_request.head.repo.full_name == github.repository
|
||||
needs: [ test-and-lint ]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
actions: write
|
||||
steps:
|
||||
- name: Cancel remaining jobs
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
curl -fsS -X POST \
|
||||
-H "Authorization: Bearer ${GH_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
"${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel"
|
||||
|
||||
# Dedicated serial lane for the ILM / lifecycle integration tests. These tests
|
||||
# drive the object layer through process-global singletons (the GLOBAL_ENV
|
||||
# ECStore, the global tier-config manager, background-expiry workers) and bind
|
||||
@@ -308,8 +340,8 @@ jobs:
|
||||
# See rustfs/backlog#1148 (ilm-1) and #1155.
|
||||
test-ilm-integration-serial:
|
||||
name: ILM Integration (serial)
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
@@ -376,8 +408,8 @@ jobs:
|
||||
|
||||
test-and-lint-rio-v2:
|
||||
name: Test and Lint (rio-v2)
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
@@ -417,8 +449,8 @@ jobs:
|
||||
|
||||
connect-short-credential-boundary:
|
||||
name: Connect Short Credential Boundary
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
@@ -475,8 +507,8 @@ jobs:
|
||||
|
||||
test-and-lint-protocols:
|
||||
name: "Test and Lint (${{ matrix.features.name }})"
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
@@ -529,8 +561,8 @@ jobs:
|
||||
|
||||
build-rustfs-debug-binary:
|
||||
name: Build RustFS Debug Binary
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
@@ -652,8 +684,8 @@ jobs:
|
||||
# job had neither, so each closed/merged PR really ran the whole io_uring
|
||||
# suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
|
||||
# 30662728539) and kept the cancellation run in progress for minutes.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
# GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
|
||||
# a container, applies no seccomp filter that would block io_uring_setup — so
|
||||
# the probe succeeds and the tests exercise the real UringBackend/FdCache/
|
||||
@@ -1181,44 +1213,9 @@ jobs:
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
|
||||
required-checks:
|
||||
name: Test and Lint
|
||||
if: always() && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs:
|
||||
- classify-changes
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- test-ilm-integration-serial
|
||||
- test-and-lint-rio-v2
|
||||
- connect-short-credential-boundary
|
||||
- test-and-lint-protocols
|
||||
- build-rustfs-debug-binary
|
||||
- uring-integration
|
||||
- e2e-tests
|
||||
- s3-implemented-tests
|
||||
- s3-lifecycle-behavior-tests
|
||||
- build-rustfs-debug-binary-rio-v2
|
||||
- e2e-tests-rio-v2
|
||||
- e2e-full
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Require the expected result of every CI lane
|
||||
env:
|
||||
CI_NEEDS: ${{ toJSON(needs) }}
|
||||
shell: bash
|
||||
run: python3 scripts/ci_gate.py verify
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs:
|
||||
- classify-changes
|
||||
- connect-short-credential-boundary
|
||||
- required-checks
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
|
||||
@@ -19,27 +19,17 @@ on:
|
||||
- cron: "7 0 * * *"
|
||||
timezone: "Asia/Shanghai"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
branch:
|
||||
description: 'Branch/ref to build and publish as the nightly (empty = scheduled source, see NIGHTLY_BUILD_REF)'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Scheduled builds follow the NIGHTLY_BRANCH repo variable so the channel can
|
||||
# be pointed at e.g. `release` for the GA cycle and back to `main` afterwards
|
||||
# without touching this file. Manual runs take the `branch` input, falling
|
||||
# back to the branch the run was dispatched from.
|
||||
concurrency:
|
||||
group: nightly-gnu-build-${{ github.event_name }}-${{ github.event_name == 'schedule' && (vars.NIGHTLY_BRANCH || 'main') || (inputs.branch || github.ref_name) }}
|
||||
group: nightly-gnu-build-main-${{ github.event_name }}
|
||||
cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' }}
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
NIGHTLY_BUILD_REF: ${{ github.event_name == 'schedule' && (vars.NIGHTLY_BRANCH || 'main') || (inputs.branch || github.ref_name) }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
@@ -53,7 +43,6 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -163,104 +152,13 @@ jobs:
|
||||
|
||||
fakeroot dpkg-deb --build "${PKG_DIR}"
|
||||
ls -lh "${DEB_FILE}"
|
||||
echo "deb_date=${DEB_DATE}" >> "${GITHUB_OUTPUT}"
|
||||
echo "deb_file=${DEB_FILE}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
# Same packaging scheme as .github/workflows/package.yml (fpm), but from
|
||||
# the locally built nightly binary instead of a release artifact, with a
|
||||
# date-based version that mirrors the DEB.
|
||||
- name: Build RPM package
|
||||
id: rpm
|
||||
shell: bash
|
||||
env:
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if ! command -v fpm >/dev/null 2>&1; then
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} apt-get update -qq && ${SUDO} apt-get install -y -qq ruby ruby-dev build-essential rpm >/dev/null
|
||||
${SUDO} gem install fpm --no-document >/dev/null
|
||||
fi
|
||||
|
||||
RPM_FILE="rustfs-nightly-${DEB_DATE}.rpm"
|
||||
RPM_VERSION="0"
|
||||
RPM_RELEASE="0.nightly.${DEB_DATE//-/.}"
|
||||
|
||||
echo "Building RPM: ${RPM_FILE} (version ${RPM_VERSION}-${RPM_RELEASE})"
|
||||
|
||||
# fpm wants the config file to exist before packaging.
|
||||
mkdir -p ./tmp-pkg/etc/default
|
||||
cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://rustfs.com/docs/ for more information
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ROOT_USER=""
|
||||
# RUSTFS_ROOT_PASSWORD=""
|
||||
ENVEOF
|
||||
|
||||
fpm -s dir -t rpm \
|
||||
--name rustfs \
|
||||
--version "$RPM_VERSION" \
|
||||
--iteration "$RPM_RELEASE" \
|
||||
--architecture x86_64 \
|
||||
--package "$RPM_FILE" \
|
||||
--depends "glibc >= 2.31" \
|
||||
--maintainer "RustFS Team <support@rustfs.com>" \
|
||||
--description "High-performance distributed object storage" \
|
||||
--url "https://rustfs.com" \
|
||||
--license "Apache-2.0" \
|
||||
--after-install <(cat << 'POSTINST'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if ! getent passwd rustfs > /dev/null 2>&1; then
|
||||
useradd -r -s /bin/false -d /opt/rustfs rustfs
|
||||
fi
|
||||
mkdir -p /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
chown rustfs:rustfs /opt/rustfs /data/rustfs /var/log/rustfs
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTINST
|
||||
) \
|
||||
--before-remove <(cat << 'PRERM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ] && systemctl is-active --quiet rustfs; then
|
||||
systemctl stop rustfs
|
||||
fi
|
||||
PRERM
|
||||
) \
|
||||
--after-remove <(cat << 'POSTRM'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
if [ -d /run/systemd/system ]; then
|
||||
systemctl daemon-reload
|
||||
fi
|
||||
POSTRM
|
||||
) \
|
||||
--config-files /etc/default/rustfs \
|
||||
"rustfs-nightly-${DEB_DATE}/usr/bin/rustfs=/usr/bin/rustfs" \
|
||||
./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
|
||||
deploy/build/rustfs.service=/lib/systemd/system/rustfs.service \
|
||||
LICENSE=/usr/share/doc/rustfs/LICENSE \
|
||||
README.md=/usr/share/doc/rustfs/README.md
|
||||
|
||||
[[ -f "$RPM_FILE" ]] || { echo "RPM build failed"; exit 1; }
|
||||
rpm -qpl "$RPM_FILE" | grep -Fx '/usr/bin/rustfs' >/dev/null
|
||||
stat --printf='%n %s bytes\n' "$RPM_FILE"
|
||||
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload DEB artifact
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: ${{ steps.deb.outputs.deb_file }}
|
||||
path: ${{ steps.deb.outputs.deb_file }}
|
||||
- name: Upload RPM artifact
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: ${{ steps.rpm.outputs.rpm_file }}
|
||||
path: ${{ steps.rpm.outputs.rpm_file }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Persist the nightly deb on Cloudflare R2 (same channel as package.yml)
|
||||
@@ -289,10 +187,11 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# The candidate manifest must describe the tree that was actually
|
||||
# built. With a ref override (NIGHTLY_BRANCH / dispatch input) that
|
||||
# is not necessarily GITHUB_SHA, so always advertise HEAD.
|
||||
SOURCE_SHA="$(git rev-parse HEAD)"
|
||||
if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then
|
||||
echo "Checkout SHA does not match the nightly build run" >&2
|
||||
exit 1
|
||||
fi
|
||||
DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
|
||||
CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
|
||||
CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"
|
||||
@@ -348,20 +247,6 @@ jobs:
|
||||
path: ${{ steps.publish.outputs.candidate_file }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
|
||||
# engineers can download and install the nightly directly. The branch is
|
||||
# a single-commit orphan rewritten on every build, which keeps the repo
|
||||
# small while the latest files stay reachable at stable raw URLs.
|
||||
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
|
||||
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
|
||||
- name: Publish packages to auto-testing release assets
|
||||
env:
|
||||
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
|
||||
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
run: bash scripts/release/publish_nightly_assets.sh
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
@@ -399,7 +284,6 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -488,7 +372,6 @@ jobs:
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
|
||||
@@ -16,7 +16,7 @@ name: Windows Filesystem Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, release ]
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- "crates/ecstore/src/disk/**"
|
||||
- "crates/ecstore/src/store/init_format.rs"
|
||||
@@ -26,7 +26,7 @@ on:
|
||||
- ".github/actions/setup/**"
|
||||
- ".github/workflows/windows-filesystem.yml"
|
||||
pull_request:
|
||||
branches: [ main, release ]
|
||||
branches: [ main ]
|
||||
paths:
|
||||
- "crates/ecstore/src/disk/**"
|
||||
- "crates/ecstore/src/store/init_format.rs"
|
||||
|
||||
Generated
+145
-123
@@ -701,9 +701,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "async-compression"
|
||||
version = "0.4.46"
|
||||
version = "0.4.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4f10dafd0c8d2e51ae9a748805777613ed0bbe17bf586b76c8311f45c020a32f"
|
||||
checksum = "515a1f282e33d55983c499d7e9e87082e81cbc32974825bf9032f928392d5844"
|
||||
dependencies = [
|
||||
"compression-codecs",
|
||||
"compression-core",
|
||||
@@ -1031,9 +1031,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.146.0"
|
||||
version = "1.145.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37b5ffaae346b9bd486ebdc3eb7053ec8ab8a1ad0f19a332eefeff036ed559e6"
|
||||
checksum = "f0e6320417a37c8a62f78b443d0b4cf628b57cd340a09b0eb56173d47cc94e93"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1735,18 +1735,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bon"
|
||||
version = "3.10.1"
|
||||
version = "3.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60eafe0d77c3a2fc292c1d1346c3041b33c0a108085a2afabf672b70f69dbbc9"
|
||||
checksum = "9e3fac94a66da67200398458a25412bcc3f9b6443b5119a6cad9cf3ccfcd8cc6"
|
||||
dependencies = [
|
||||
"bon-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bon-macros"
|
||||
version = "3.10.1"
|
||||
version = "3.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd0f9631d8aaaee112c41985d675ef269e02acbd4f33122836af4f0c5f699ff6"
|
||||
checksum = "d4654961ad0494e4774c5c60b4cb4cd0ae9b9d92d039d901638b1dba97ebebf5"
|
||||
dependencies = [
|
||||
"darling 0.24.1",
|
||||
"ident_case",
|
||||
@@ -2220,7 +2220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47"
|
||||
dependencies = [
|
||||
"unicode-segmentation",
|
||||
"unicode-width",
|
||||
"unicode-width 0.2.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2238,9 +2238,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "compression-codecs"
|
||||
version = "0.4.41"
|
||||
version = "0.4.39"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "58a6d0db8759036a783bc7c3f7a07f8cef3bf9470eb1db3bc86e8bcd1c5d0fe8"
|
||||
checksum = "2fe67f2944eef52fc7b106b8c9450d243a88701a0c065f7f57235e76abaed7df"
|
||||
dependencies = [
|
||||
"brotli 8.0.4",
|
||||
"bzip2",
|
||||
@@ -2249,8 +2249,8 @@ dependencies = [
|
||||
"liblzma",
|
||||
"lz4",
|
||||
"memchr",
|
||||
"zstd 0.14.0",
|
||||
"zstd-safe 8.0.0",
|
||||
"zstd 0.13.3",
|
||||
"zstd-safe 7.3.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2408,12 +2408,6 @@ version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||
|
||||
[[package]]
|
||||
name = "core_detect"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
|
||||
|
||||
[[package]]
|
||||
name = "cpp_demangle"
|
||||
version = "0.5.1"
|
||||
@@ -3933,6 +3927,16 @@ dependencies = [
|
||||
"dirs-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-next"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"dirs-sys-next",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys"
|
||||
version = "0.5.0"
|
||||
@@ -3941,10 +3945,21 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"redox_users 0.5.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys-next"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"redox_users 0.4.6",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dispatch2"
|
||||
version = "0.3.1"
|
||||
@@ -4193,17 +4208,11 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.41"
|
||||
version = "0.8.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a"
|
||||
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"core_detect",
|
||||
"multiversion",
|
||||
"multiversion_no_op",
|
||||
"rustversion",
|
||||
"scopeguard",
|
||||
"simdutf8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5062,9 +5071,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
version = "0.12.2"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb"
|
||||
checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248"
|
||||
dependencies = [
|
||||
"hashbrown 0.17.1",
|
||||
]
|
||||
@@ -5278,9 +5287,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath"
|
||||
version = "0.25.1"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "366c5db5a8e4643bdde121a1d6f35cfa035e7850b8b8ea94c6960a139d44e93a"
|
||||
checksum = "2ec7782e005cabd5eaf350febde384cd799faa3a0e624587aa8759c240e0b592"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -5289,7 +5298,6 @@ dependencies = [
|
||||
"crossbeam-channel",
|
||||
"flate2",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"hdrhistogram",
|
||||
"hotpath-macros",
|
||||
@@ -5298,8 +5306,9 @@ dependencies = [
|
||||
"object 0.36.7",
|
||||
"parking_lot",
|
||||
"pin-project-lite",
|
||||
"prettytable-rs",
|
||||
"quanta",
|
||||
"regex-lite",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"reqwest-middleware",
|
||||
"rustc-demangle",
|
||||
@@ -5401,9 +5410,9 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
|
||||
|
||||
[[package]]
|
||||
name = "hybrid-array"
|
||||
version = "0.4.15"
|
||||
version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17"
|
||||
checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b"
|
||||
dependencies = [
|
||||
"ctutils",
|
||||
"subtle",
|
||||
@@ -5702,9 +5711,9 @@ checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96"
|
||||
|
||||
[[package]]
|
||||
name = "io-uring"
|
||||
version = "0.7.15"
|
||||
version = "0.7.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb"
|
||||
checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"cfg-if",
|
||||
@@ -5742,6 +5751,17 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is-terminal"
|
||||
version = "0.4.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_debug"
|
||||
version = "1.1.0"
|
||||
@@ -6001,9 +6021,9 @@ checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37"
|
||||
|
||||
[[package]]
|
||||
name = "lapin"
|
||||
version = "4.11.0"
|
||||
version = "4.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24b99d8cdfe3f6223f9f75e35e7510a8662767bee27c1cf870d503fc3bb2428a"
|
||||
checksum = "0fd20e01fd92597ca352ca7ceed3c589851ebad279dfcada48aa4d24fd3a7caa"
|
||||
dependencies = [
|
||||
"amq-protocol",
|
||||
"async-rs",
|
||||
@@ -6704,33 +6724,6 @@ version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
|
||||
|
||||
[[package]]
|
||||
name = "multiversion"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c"
|
||||
dependencies = [
|
||||
"multiversion-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "multiversion-macros"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rustversion",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "multiversion_no_op"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"
|
||||
|
||||
[[package]]
|
||||
name = "murmur3"
|
||||
version = "0.5.2"
|
||||
@@ -7716,9 +7709,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pe-unwind-info"
|
||||
version = "0.6.1"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e33c6dbf1a8fb7f71742cd70f5e9f0986e60b2d19dc0b28d9ca0d1323259274a"
|
||||
checksum = "97f6fccfd2d9d2df765ca23ff85fe5cc437fb0e6d3e164e4d3cbe09d14780c93"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"bitflags 2.13.1",
|
||||
@@ -8194,6 +8187,19 @@ dependencies = [
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prettytable-rs"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a"
|
||||
dependencies = [
|
||||
"encode_unicode",
|
||||
"is-terminal",
|
||||
"lazy_static",
|
||||
"term",
|
||||
"unicode-width 0.1.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "primefield"
|
||||
version = "0.14.0"
|
||||
@@ -8513,24 +8519,24 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "qrcode-core"
|
||||
version = "2.1.0"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a5c0d85fe467f79319e9f0d0ffe43f8c32defae67263f78d57e33d8b4a638bf"
|
||||
checksum = "79de8ed9b3c65c866e0ca79ba3aa8b05264e5183b0f894f1ea43b3a53146d657"
|
||||
|
||||
[[package]]
|
||||
name = "qrcode-decode"
|
||||
version = "2.1.0"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04cf729caefa8fd0919f8d111a4e6531942ac48742e7649ed4c9c159b41a6fda"
|
||||
checksum = "41c307d82a6b418dc99004a68cdd621c0606874faae8645036eef9168ec5aef4"
|
||||
dependencies = [
|
||||
"qrcode-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "qrcode-eps"
|
||||
version = "2.1.0"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d289a6212464685d1e598c3692e02189a096b2f4abe1acdb1f4fba6a932809d2"
|
||||
checksum = "c1cf7d12b9e539644309c628db9436be7024e0fbe16f440109ce92d7c9787688"
|
||||
dependencies = [
|
||||
"qrcode-core",
|
||||
"qrcode-render",
|
||||
@@ -8538,19 +8544,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "qrcode-html"
|
||||
version = "2.1.0"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a14b98d4aa6278f7e7218ad901837a85cef040a262127d0c4b6f8d3bd51d46e"
|
||||
dependencies = [
|
||||
"qrcode-core",
|
||||
"qrcode-render",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "qrcode-image"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c7923bee1dc13ef96b9778e9299d803ae9f5de38971cb8e55c141022cc296618"
|
||||
checksum = "7fb7d790f2b34ce1edbafdf7a6802e73694aa94798be275c63acb780a39c1e6c"
|
||||
dependencies = [
|
||||
"qrcode-core",
|
||||
"qrcode-render",
|
||||
@@ -8558,15 +8554,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "qrcode-parse"
|
||||
version = "2.1.0"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0726e7e2acd6335b63936ba1d784361765313f53274942efb0636407e882fdc9"
|
||||
checksum = "84f6484a2329e87b5a4b21c983d68bee32a442087287c0679efe782689ee469a"
|
||||
|
||||
[[package]]
|
||||
name = "qrcode-pdf"
|
||||
version = "2.1.0"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41ed9a51cab153e441963f8ec64ffc980d7a40bfd7fa8f6e0edd290424cda206"
|
||||
checksum = "deda8c86193091e31856024288e986869559ace2a71b7b477f59b4caea4dabf4"
|
||||
dependencies = [
|
||||
"qrcode-core",
|
||||
"qrcode-render",
|
||||
@@ -8574,9 +8570,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "qrcode-pic"
|
||||
version = "2.1.0"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "038be675ee01aeae82f20eea4b043054f8232e14d3ccc7836803510ad3ff6524"
|
||||
checksum = "03bda277eb6e11b06e0ebe0cb1b519d71467a3f1d7f6408336af9a979edb1091"
|
||||
dependencies = [
|
||||
"qrcode-core",
|
||||
"qrcode-render",
|
||||
@@ -8584,24 +8580,23 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "qrcode-render"
|
||||
version = "2.1.0"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7af1ddfd0f2496c28276913e26289e8fb245cab16ece7efcf9df46e2027dd3c8"
|
||||
checksum = "7a40771febec53bdf7de3b3de6549950a7c9f54eef58477d99f4de2fd9f114f9"
|
||||
dependencies = [
|
||||
"qrcode-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "qrcode-rs"
|
||||
version = "2.1.0"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "246e31e4a82d59d4714c589e4912f27bc67f5fa205cc4e3bdf916ccc640af010"
|
||||
checksum = "a62237103e62d01195453a4b34d56a35f59cf8c8ca3fa8ddcfca84336d5c4a2a"
|
||||
dependencies = [
|
||||
"qrcode-core",
|
||||
"qrcode-decode",
|
||||
"qrcode-eps",
|
||||
"qrcode-html",
|
||||
"qrcode-image",
|
||||
"qrcode-parse",
|
||||
"qrcode-pdf",
|
||||
"qrcode-pic",
|
||||
@@ -8611,9 +8606,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "qrcode-svg"
|
||||
version = "2.1.0"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19f641cac9a021d31948437c740baec47243c03586c3230c839b76df9e4df084"
|
||||
checksum = "513f1dd2a0e11d1bcc4dd3268887cdad059ed8ab5269d45ce57aada1256c5405"
|
||||
dependencies = [
|
||||
"qrcode-core",
|
||||
"qrcode-render",
|
||||
@@ -8986,6 +8981,17 @@ dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_users"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
|
||||
dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
"libredox",
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_users"
|
||||
version = "0.5.2"
|
||||
@@ -9078,11 +9084,11 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.13.5"
|
||||
version = "0.13.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029"
|
||||
checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"encoding_rs",
|
||||
"futures-channel",
|
||||
@@ -9304,9 +9310,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "russh"
|
||||
version = "0.63.3"
|
||||
version = "0.63.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "036204edbd199552a5b3832f63c60dcdf395dc44c7f06b4af1c0e8139cc11bce"
|
||||
checksum = "8e134e2480f4e86f83e4aa45b4c0a9723f84beaffa694c54bdf056e74efdd7dd"
|
||||
dependencies = [
|
||||
"aes 0.9.3",
|
||||
"aws-lc-rs",
|
||||
@@ -9387,9 +9393,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "russh-sftp"
|
||||
version = "3.0.0"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "093197e526668d92bba562e2bbbe98d1af9831bf080b619c736316ca1fa35101"
|
||||
checksum = "9de67aace74530a29086db0671fa200c470a58eb380081f28ad512ffb0c5356b"
|
||||
dependencies = [
|
||||
"bitflags 2.13.1",
|
||||
"bytes",
|
||||
@@ -9922,7 +9928,6 @@ dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"crc-fast",
|
||||
"futures",
|
||||
"hotpath",
|
||||
@@ -11080,9 +11085,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.44"
|
||||
version = "0.23.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba"
|
||||
checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
@@ -11598,11 +11603,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_with"
|
||||
version = "3.23.0"
|
||||
version = "3.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4"
|
||||
checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"base64 0.22.1",
|
||||
"bs58",
|
||||
"chrono",
|
||||
"hex",
|
||||
@@ -11619,14 +11624,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_with_macros"
|
||||
version = "3.23.0"
|
||||
version = "3.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc"
|
||||
checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46"
|
||||
dependencies = [
|
||||
"darling 0.24.1",
|
||||
"darling 0.23.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.5",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12190,9 +12195,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "suppaftp"
|
||||
version = "12.0.0"
|
||||
version = "11.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c6978f1b04fc86f122f767b88a9d1d89e80cfe3dc97d670598507d713c4ce48"
|
||||
checksum = "46c5095831abc0d7944a2d50d6ec6abcd75b9d165d9377deb3e45798cae2343a"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
@@ -12411,6 +12416,17 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "term"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
|
||||
dependencies = [
|
||||
"dirs-next",
|
||||
"rustversion",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "termcolor"
|
||||
version = "1.4.1"
|
||||
@@ -13177,6 +13193,12 @@ version = "1.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.2.2"
|
||||
@@ -13977,18 +13999,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.57"
|
||||
version = "0.8.56"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873"
|
||||
checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.57"
|
||||
version = "0.8.56"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc"
|
||||
checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
||||
+10
-10
@@ -145,7 +145,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" }
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.46" }
|
||||
async-compression = { version = "0.4.44" }
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.50.0", default-features = false }
|
||||
@@ -156,7 +156,7 @@ futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "6.9.0" }
|
||||
lapin = { default-features = false, version = "4.11.0" }
|
||||
lapin = { default-features = false, version = "4.10.0" }
|
||||
hyper = { version = "1.11.1" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.9" }
|
||||
hyper-util = { version = "0.1.20" }
|
||||
@@ -164,7 +164,7 @@ http = "1.5.0"
|
||||
http-body = "1.1.0"
|
||||
http-body-util = "0.1.5"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
reqwest = "0.13.4"
|
||||
rustfs-kafka-async = { version = "1.3.1" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.53.1" }
|
||||
@@ -211,7 +211,7 @@ openidconnect = { default-features = false, version = "4.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
rustls = { default-features = false, version = "0.23.44" }
|
||||
rustls = { default-features = false, version = "0.23.43" }
|
||||
rustls-native-certs = "0.8"
|
||||
rustls-pki-types = "1.15.1"
|
||||
x509-parser = "0.18.1"
|
||||
@@ -245,7 +245,7 @@ atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.12.0" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.118.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.146.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.145.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.114.0" }
|
||||
aws-smithy-async = { version = "1.3.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
|
||||
@@ -297,7 +297,7 @@ percent-encoding = "2.3.2"
|
||||
# Server-side QR rendering for TOTP enrollment, so neither the console nor the
|
||||
# CLI needs its own QR encoder. No default features: the image/render backends
|
||||
# pull in an image stack this only needs SVG and text output from.
|
||||
qrcode-rs = { version = "2.1.0", default-features = false, features = ["std", "svg"] }
|
||||
qrcode-rs = { version = "2.0.0", default-features = false, features = ["std", "svg"] }
|
||||
pin-project-lite = "0.2.17"
|
||||
pretty_assertions = "1.4.1"
|
||||
rand = { version = "0.10.2" }
|
||||
@@ -362,10 +362,10 @@ pyroscope = { version = "2.1.1" }
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.0.0" }
|
||||
suppaftp = { version = "11.0.0" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.63.3" }
|
||||
russh-sftp = "3.0.0"
|
||||
russh = { version = "0.63.2" }
|
||||
russh-sftp = "2.4.0"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.11.0"
|
||||
@@ -373,7 +373,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.5.3" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.25.1", default-features = false }
|
||||
hotpath = { version = "=0.25.0", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
|
||||
@@ -211,10 +211,7 @@ For developers who want to build RustFS Docker images from source with multi-arc
|
||||
|
||||
```bash
|
||||
# Build multi-architecture images locally
|
||||
./docker-buildx.sh
|
||||
|
||||
# Build a single-platform image locally
|
||||
./docker-buildx.sh -p linux/amd64
|
||||
./docker-buildx.sh --build-arg RELEASE=latest
|
||||
|
||||
# Build and push to registry
|
||||
./docker-buildx.sh --push
|
||||
|
||||
+1
-4
@@ -167,10 +167,7 @@ docker compose -f docker-compose-simple.yml up -d
|
||||
|
||||
```bash
|
||||
# 在本地构建多架构镜像
|
||||
./docker-buildx.sh
|
||||
|
||||
# 在本地构建单平台镜像
|
||||
./docker-buildx.sh -p linux/amd64
|
||||
./docker-buildx.sh --build-arg RELEASE=latest
|
||||
|
||||
# 构建并推送到仓库
|
||||
./docker-buildx.sh --push
|
||||
|
||||
@@ -593,20 +593,6 @@ pub struct DataUsageSnapshotIdentity {
|
||||
pub scanner_epoch: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct DataUsageSegmentInvalidationProof {
|
||||
#[serde(default)]
|
||||
pub process_epoch: String,
|
||||
#[serde(default)]
|
||||
pub generation_start: u64,
|
||||
#[serde(default)]
|
||||
pub generation_end: u64,
|
||||
#[serde(default)]
|
||||
pub producer_identity_coverage_complete: bool,
|
||||
#[serde(default)]
|
||||
pub cold_zero_walk_oracle: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct DataUsageSnapshotSetState {
|
||||
pub pool_index: u64,
|
||||
@@ -621,8 +607,6 @@ pub struct DataUsageSnapshotSetState {
|
||||
pub complete: bool,
|
||||
#[serde(default)]
|
||||
pub tombstone: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub segment_invalidation_proof: Option<DataUsageSegmentInvalidationProof>,
|
||||
}
|
||||
|
||||
impl DataUsageInfo {
|
||||
@@ -3089,7 +3073,6 @@ mod tests {
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: false,
|
||||
tombstone: false,
|
||||
segment_invalidation_proof: None,
|
||||
}];
|
||||
assert!(observed_data_usage_is_newer(&partial, &authoritative));
|
||||
}
|
||||
@@ -3112,7 +3095,6 @@ mod tests {
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: true,
|
||||
tombstone: false,
|
||||
segment_invalidation_proof: None,
|
||||
},
|
||||
DataUsageSnapshotSetState {
|
||||
pool_index: 1,
|
||||
@@ -3122,7 +3104,6 @@ mod tests {
|
||||
scan_plan_digest: Some([2; 32]),
|
||||
complete: false,
|
||||
tombstone: false,
|
||||
segment_invalidation_proof: None,
|
||||
},
|
||||
],
|
||||
..Default::default()
|
||||
@@ -3132,42 +3113,6 @@ mod tests {
|
||||
assert!(partial.is_valid_partial_snapshot());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_state_segment_invalidation_proof_is_additive() {
|
||||
#[derive(Deserialize)]
|
||||
struct LegacySetState {
|
||||
pool_index: u64,
|
||||
set_index: u64,
|
||||
complete: bool,
|
||||
}
|
||||
|
||||
let proof = DataUsageSegmentInvalidationProof {
|
||||
process_epoch: "scanner-process".to_string(),
|
||||
generation_start: 3,
|
||||
generation_end: 5,
|
||||
producer_identity_coverage_complete: true,
|
||||
cold_zero_walk_oracle: true,
|
||||
};
|
||||
let state = DataUsageSnapshotSetState {
|
||||
pool_index: 1,
|
||||
set_index: 2,
|
||||
scanner_cycle: Some(9),
|
||||
scanner_epoch: Some(4),
|
||||
scan_plan_digest: Some([7; 32]),
|
||||
complete: true,
|
||||
tombstone: false,
|
||||
segment_invalidation_proof: Some(proof.clone()),
|
||||
};
|
||||
let encoded = rmp_serde::to_vec_named(&state).expect("set state should encode with additive proof");
|
||||
let legacy: LegacySetState = rmp_serde::from_slice(&encoded).expect("legacy readers should ignore proof metadata");
|
||||
assert_eq!(legacy.pool_index, 1);
|
||||
assert_eq!(legacy.set_index, 2);
|
||||
assert!(legacy.complete);
|
||||
|
||||
let decoded: DataUsageSnapshotSetState = rmp_serde::from_slice(&encoded).expect("new readers should restore proof");
|
||||
assert_eq!(decoded.segment_invalidation_proof, Some(proof));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completeness_marker_requires_a_snapshot_timestamp() {
|
||||
let untimestamped = DataUsageInfo {
|
||||
|
||||
@@ -30,8 +30,6 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
|
||||
The external-tool `storage_metric_ownership_test` validates the OTLP/Collector/Prometheus path, including a rolling upgrade and node failures. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for its required binaries and focused command.
|
||||
|
||||
## How to run
|
||||
|
||||
All commands assume repo root. `cargo test` triggers an on-demand build of the
|
||||
|
||||
@@ -268,7 +268,6 @@ async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<(
|
||||
let rule = CorsRule::builder()
|
||||
.allowed_methods("GET")
|
||||
.allowed_origins("https://example.com")
|
||||
.allowed_headers("*")
|
||||
.build()?;
|
||||
let configuration = CorsConfiguration::builder().cors_rules(rule).build()?;
|
||||
|
||||
@@ -289,60 +288,6 @@ async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<(
|
||||
assert_eq!(rules[0].allowed_methods(), ["GET"]);
|
||||
assert_eq!(rules[0].allowed_origins(), ["https://example.com"]);
|
||||
|
||||
let http = reqwest::Client::builder().no_proxy().build()?;
|
||||
let url = format!("http://{}/{}", cluster.nodes[1].address, BUCKET_METADATA_RELOAD_BUCKET);
|
||||
let without_headers = http
|
||||
.request(reqwest::Method::OPTIONS, &url)
|
||||
.header("Origin", "https://example.com")
|
||||
.header("Access-Control-Request-Method", "GET")
|
||||
.send()
|
||||
.await?;
|
||||
assert!(without_headers.status().is_success());
|
||||
assert!(!without_headers.headers().contains_key("access-control-allow-headers"));
|
||||
assert!(
|
||||
without_headers
|
||||
.headers()
|
||||
.get("vary")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.is_some_and(|value| value.contains("Access-Control-Request-Headers")),
|
||||
"a cached header-free preflight must not suppress a later requested header grant"
|
||||
);
|
||||
let preflight = http
|
||||
.request(reqwest::Method::OPTIONS, &url)
|
||||
.header("Origin", "https://example.com")
|
||||
.header("Access-Control-Request-Method", "GET")
|
||||
.header("Access-Control-Request-Headers", "X-Another-Header, x-could-be-anything")
|
||||
.send()
|
||||
.await?;
|
||||
assert!(preflight.status().is_success(), "peer preflight should succeed: {preflight:?}");
|
||||
assert_eq!(
|
||||
preflight
|
||||
.headers()
|
||||
.get("access-control-allow-headers")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("x-another-header,x-could-be-anything"),
|
||||
"a wildcard rule must return only the headers requested by this preflight"
|
||||
);
|
||||
assert!(
|
||||
preflight
|
||||
.headers()
|
||||
.get("vary")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.is_some_and(|value| value.contains("Access-Control-Request-Headers")),
|
||||
"preflight caches must distinguish the requested header list"
|
||||
);
|
||||
let denied = http
|
||||
.request(reqwest::Method::OPTIONS, &url)
|
||||
.header("Origin", "https://disallowed.example.com")
|
||||
.header("Access-Control-Request-Method", "GET")
|
||||
.header("Access-Control-Request-Headers", "x-another-header")
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
!denied.headers().contains_key("access-control-allow-headers"),
|
||||
"a rejected origin must not receive the requested header grant"
|
||||
);
|
||||
|
||||
writer
|
||||
.delete_bucket_cors()
|
||||
.bucket(BUCKET_METADATA_RELOAD_BUCKET)
|
||||
|
||||
@@ -962,6 +962,27 @@ pub(crate) async fn wait_for_rebalance_active(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_rebalance_running_with_progress(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
expected_id: &str,
|
||||
timeout: Duration,
|
||||
) -> TestResult {
|
||||
let deadline = Instant::now() + timeout;
|
||||
loop {
|
||||
let status = rebalance_status_json(cluster).await?;
|
||||
if rebalance_running_with_progress(&status, expected_id)? {
|
||||
return Ok(());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"rebalance did not become active with non-zero progress within {timeout:?}; last status: {status}"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_rebalance_complete(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
expected_id: &str,
|
||||
|
||||
@@ -26,7 +26,6 @@ use std::collections::{BTreeMap, HashSet};
|
||||
use std::io::{Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use tokio::time::{Instant, sleep};
|
||||
|
||||
const EC84_NODE_COUNT: usize = 3;
|
||||
const EC84_DRIVES_PER_NODE: usize = 4;
|
||||
@@ -34,8 +33,6 @@ const EC84_DATA_BLOCKS: usize = 8;
|
||||
const EC84_PARITY_BLOCKS: usize = 4;
|
||||
const EC84_TARGET_DRIVE_RESTART_CASE: &str = "ec84-target-drive-restart";
|
||||
const EC84_TARGET_DRIVE_RESTART_ORACLE: &str = "ec84-target-drive-restart.json";
|
||||
const EC84_HEAL_CONTROL_READY_TIMEOUT: Duration = Duration::from_secs(45);
|
||||
const EC84_HEAL_CONTROL_RETRY_DELAY: Duration = Duration::from_millis(250);
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ExpectedShard {
|
||||
@@ -214,29 +211,6 @@ fn assert_replaced_drive_empty(drive: &Path, bucket: &str, keys: &[String]) -> T
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn is_cluster_heal_coordination_unavailable(error: &(dyn std::error::Error + Send + Sync)) -> bool {
|
||||
let message = error.to_string();
|
||||
message.contains("500 Internal Server Error") && message.contains("cluster heal coordination unavailable")
|
||||
}
|
||||
|
||||
async fn start_ec84_root_heal_when_control_ready(
|
||||
heal_url: &str,
|
||||
heal_body: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> TestResult {
|
||||
let deadline = Instant::now() + EC84_HEAL_CONTROL_READY_TIMEOUT;
|
||||
loop {
|
||||
match signed_admin_post(heal_url, Some(heal_body), access_key, secret_key).await {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(error) if is_cluster_heal_coordination_unavailable(error.as_ref()) && Instant::now() < deadline => {
|
||||
sleep(EC84_HEAL_CONTROL_RETRY_DELAY).await;
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn put_large_inventory(client: &Client, bucket: &str) -> TestResult<Vec<ExpectedShard>> {
|
||||
let mut expected = Vec::new();
|
||||
for index in 0..4 {
|
||||
@@ -330,7 +304,7 @@ async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_res
|
||||
let heal_body =
|
||||
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
let heal_url = format!("{}/rustfs/admin/v3/heal/{bucket}?forceStart=true", dist.cluster.nodes[0].url);
|
||||
start_ec84_root_heal_when_control_ready(&heal_url, heal_body, &dist.cluster.access_key, &dist.cluster.secret_key).await?;
|
||||
signed_admin_post(&heal_url, Some(heal_body), &dist.cluster.access_key, &dist.cluster.secret_key).await?;
|
||||
|
||||
wait_until(
|
||||
Duration::from_secs(120),
|
||||
@@ -391,23 +365,3 @@ async fn three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_res
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn cluster_heal_coordination_retry_is_exact() {
|
||||
let retryable: Box<dyn std::error::Error + Send + Sync> =
|
||||
"admin POST failed: 500 Internal Server Error cluster heal coordination unavailable".into();
|
||||
assert!(is_cluster_heal_coordination_unavailable(retryable.as_ref()));
|
||||
|
||||
let other_internal: Box<dyn std::error::Error + Send + Sync> =
|
||||
"admin POST failed: 500 Internal Server Error unrelated".into();
|
||||
assert!(!is_cluster_heal_coordination_unavailable(other_internal.as_ref()));
|
||||
|
||||
let wrong_status: Box<dyn std::error::Error + Send + Sync> =
|
||||
"admin POST failed: 503 Service Unavailable cluster heal coordination unavailable".into();
|
||||
assert!(!is_cluster_heal_coordination_unavailable(wrong_status.as_ref()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,9 +14,9 @@
|
||||
|
||||
use super::harness::{
|
||||
DECOMMISSION_POOL_ID, DistCluster, DistLayout, TestResult, assert_inventory, decommission_running_with_progress,
|
||||
decommission_status_json, put_inventory_retrying, rebalance_active, rebalance_status_json, retrying_get_equals, retrying_put,
|
||||
start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
|
||||
wait_for_decommission_running_with_progress, wait_for_rebalance_active, wait_for_rebalance_complete,
|
||||
decommission_status_json, put_inventory_retrying, rebalance_running_with_progress, rebalance_status_json,
|
||||
retrying_get_equals, retrying_put, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
|
||||
wait_for_decommission_running_with_progress, wait_for_rebalance_complete, wait_for_rebalance_running_with_progress,
|
||||
};
|
||||
use crate::common::init_logging;
|
||||
use std::time::Duration;
|
||||
@@ -67,10 +67,7 @@ async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResu
|
||||
assert_inventory(&live, &bucket, &inventory).await?;
|
||||
|
||||
let rebalance_id = start_rebalance(&dist.cluster).await?;
|
||||
// The status API reads persisted progress, whose first periodic save is
|
||||
// after 30 seconds. A shorter run can remain at zero until completion.
|
||||
// Require Started around the S3 operations and nonzero progress at completion.
|
||||
wait_for_rebalance_active(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
|
||||
wait_for_rebalance_running_with_progress(&dist.cluster, &rebalance_id, Duration::from_secs(30)).await?;
|
||||
retrying_put(
|
||||
&live,
|
||||
&bucket,
|
||||
@@ -87,26 +84,11 @@ async fn s3_put_get_list_succeed_during_decommission_and_rebalance() -> TestResu
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
.await?;
|
||||
let listed = live.list_objects_v2().bucket(&bucket).send().await?;
|
||||
assert!(
|
||||
listed
|
||||
.contents()
|
||||
.iter()
|
||||
.any(|object| object.key() == Some("during-rebalance.bin")),
|
||||
"list during rebalance missed the newly written key"
|
||||
);
|
||||
let status = rebalance_status_json(&dist.cluster).await?;
|
||||
if !rebalance_active(&status, &rebalance_id)? {
|
||||
if !rebalance_running_with_progress(&status, &rebalance_id)? {
|
||||
return Err(format!("rebalance did not remain active across the S3 operations: {status}").into());
|
||||
}
|
||||
wait_for_rebalance_complete(&dist.cluster, &rebalance_id, Duration::from_secs(180)).await?;
|
||||
let after = dist.client(1)?;
|
||||
assert_inventory(&after, &bucket, &inventory).await?;
|
||||
for (key, body) in [
|
||||
("during-decommission.bin", b"written-while-decommissioning".as_slice()),
|
||||
("during-rebalance.bin", b"written-while-rebalancing".as_slice()),
|
||||
] {
|
||||
retrying_get_equals(&after, &bucket, key, body, Duration::from_secs(30)).await?;
|
||||
}
|
||||
assert_inventory(&dist.client(1)?, &bucket, &inventory).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -62,38 +62,18 @@ mod tests {
|
||||
unclean_shutdown_marker: bool,
|
||||
topology: EvidenceTopology,
|
||||
storage_class_standard: Option<&'static str>,
|
||||
erasure_set_drive_count: Option<usize>,
|
||||
outage_target_manifest_required: bool,
|
||||
erasure_set_drive_count: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct EvidenceTopology {
|
||||
nodes: usize,
|
||||
drives_per_node: usize,
|
||||
layout: EvidenceTopologyLayout,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum EvidenceTopologyLayout {
|
||||
SinglePool,
|
||||
PerNodePools,
|
||||
}
|
||||
|
||||
impl EvidenceTopology {
|
||||
const fn new(nodes: usize, drives_per_node: usize) -> Self {
|
||||
Self {
|
||||
nodes,
|
||||
drives_per_node,
|
||||
layout: EvidenceTopologyLayout::SinglePool,
|
||||
}
|
||||
}
|
||||
|
||||
const fn per_node_pools(nodes: usize, drives_per_node: usize) -> Self {
|
||||
Self {
|
||||
nodes,
|
||||
drives_per_node,
|
||||
layout: EvidenceTopologyLayout::PerNodePools,
|
||||
}
|
||||
Self { nodes, drives_per_node }
|
||||
}
|
||||
|
||||
fn total_drives(self) -> usize {
|
||||
@@ -101,23 +81,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn cluster_topology(self) -> ClusterTopology {
|
||||
match self.layout {
|
||||
EvidenceTopologyLayout::SinglePool => ClusterTopology::single_pool_multidrive(self.nodes, self.drives_per_node),
|
||||
EvidenceTopologyLayout::PerNodePools => {
|
||||
ClusterTopology::per_node_pools(self.drives_per_node, (0..self.nodes).map(|node| vec![node]).collect())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn pool_count(self) -> usize {
|
||||
match self.layout {
|
||||
EvidenceTopologyLayout::SinglePool => 1,
|
||||
EvidenceTopologyLayout::PerNodePools => self.nodes,
|
||||
}
|
||||
}
|
||||
|
||||
fn set_count(self, erasure_set_drive_count: Option<usize>) -> usize {
|
||||
self.total_drives() / erasure_set_drive_count.unwrap_or_else(|| self.total_drives())
|
||||
ClusterTopology::single_pool_multidrive(self.nodes, self.drives_per_node)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,7 +93,6 @@ mod tests {
|
||||
topology: EvidenceTopology::new(4, 1),
|
||||
storage_class_standard: None,
|
||||
erasure_set_drive_count: None,
|
||||
outage_target_manifest_required: true,
|
||||
};
|
||||
|
||||
const BACKGROUND_TARGET_CRASH_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
@@ -140,7 +103,6 @@ mod tests {
|
||||
topology: EvidenceTopology::new(4, 1),
|
||||
storage_class_standard: None,
|
||||
erasure_set_drive_count: None,
|
||||
outage_target_manifest_required: true,
|
||||
};
|
||||
|
||||
const BACKGROUND_TARGET_RESTART_EC84_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
@@ -150,8 +112,7 @@ mod tests {
|
||||
unclean_shutdown_marker: false,
|
||||
topology: EvidenceTopology::new(3, 4),
|
||||
storage_class_standard: Some("EC:4"),
|
||||
erasure_set_drive_count: Some(12),
|
||||
outage_target_manifest_required: true,
|
||||
erasure_set_drive_count: Some("12"),
|
||||
};
|
||||
|
||||
const BACKGROUND_TARGET_CRASH_EC84_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
@@ -161,30 +122,7 @@ mod tests {
|
||||
unclean_shutdown_marker: true,
|
||||
topology: EvidenceTopology::new(3, 4),
|
||||
storage_class_standard: Some("EC:4"),
|
||||
erasure_set_drive_count: Some(12),
|
||||
outage_target_manifest_required: true,
|
||||
};
|
||||
|
||||
const BACKGROUND_TARGET_RESTART_EC84_MULTI_SET_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
id: "background-target-restart-ec8-4-multi-set",
|
||||
oracle: "background-target-restart-ec8-4-multi-set.json",
|
||||
evidence: "process-restart",
|
||||
unclean_shutdown_marker: false,
|
||||
topology: EvidenceTopology::new(3, 8),
|
||||
storage_class_standard: Some("EC:4"),
|
||||
erasure_set_drive_count: Some(12),
|
||||
outage_target_manifest_required: true,
|
||||
};
|
||||
|
||||
const BACKGROUND_TARGET_CRASH_EC84_MULTI_POOL_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
id: "background-target-crash-ec8-4-multi-pool",
|
||||
oracle: "background-target-crash-ec8-4-multi-pool.json",
|
||||
evidence: "process-crash-restart",
|
||||
unclean_shutdown_marker: true,
|
||||
topology: EvidenceTopology::per_node_pools(3, 12),
|
||||
storage_class_standard: Some("EC:4"),
|
||||
erasure_set_drive_count: Some(12),
|
||||
outage_target_manifest_required: false,
|
||||
erasure_set_drive_count: Some("12"),
|
||||
};
|
||||
|
||||
struct RestartEvidenceContext {
|
||||
@@ -1103,26 +1041,6 @@ mod tests {
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
timeout(
|
||||
Duration::from_secs(600),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetRestartEc84MultiSet),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
timeout(
|
||||
Duration::from_secs(600),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetCrashEc84MultiPool),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_remote_shards_after_coordinator_restart() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
@@ -1162,8 +1080,6 @@ mod tests {
|
||||
BackgroundTargetCrash,
|
||||
BackgroundTargetRestartEc84,
|
||||
BackgroundTargetCrashEc84,
|
||||
BackgroundTargetRestartEc84MultiSet,
|
||||
BackgroundTargetCrashEc84MultiPool,
|
||||
BackgroundCoordinatorRestart,
|
||||
TargetEndpointBlackhole,
|
||||
}
|
||||
@@ -1175,8 +1091,6 @@ mod tests {
|
||||
InterruptionScenario::BackgroundTargetCrash => Some(BACKGROUND_TARGET_CRASH_EVIDENCE),
|
||||
InterruptionScenario::BackgroundTargetRestartEc84 => Some(BACKGROUND_TARGET_RESTART_EC84_EVIDENCE),
|
||||
InterruptionScenario::BackgroundTargetCrashEc84 => Some(BACKGROUND_TARGET_CRASH_EC84_EVIDENCE),
|
||||
InterruptionScenario::BackgroundTargetRestartEc84MultiSet => Some(BACKGROUND_TARGET_RESTART_EC84_MULTI_SET_EVIDENCE),
|
||||
InterruptionScenario::BackgroundTargetCrashEc84MultiPool => Some(BACKGROUND_TARGET_CRASH_EC84_MULTI_POOL_EVIDENCE),
|
||||
_ => None,
|
||||
};
|
||||
let evidence_run = match evidence_case {
|
||||
@@ -1190,18 +1104,12 @@ mod tests {
|
||||
InterruptionScenario::BackgroundTargetCrash => (true, 1, "background_target_crash"),
|
||||
InterruptionScenario::BackgroundTargetRestartEc84 => (true, 1, "background_target_restart_ec8_4"),
|
||||
InterruptionScenario::BackgroundTargetCrashEc84 => (true, 1, "background_target_crash_ec8_4"),
|
||||
InterruptionScenario::BackgroundTargetRestartEc84MultiSet => (true, 1, "background_target_restart_ec8_4_multi_set"),
|
||||
InterruptionScenario::BackgroundTargetCrashEc84MultiPool => (true, 1, "background_target_crash_ec8_4_multi_pool"),
|
||||
InterruptionScenario::BackgroundCoordinatorRestart => (true, 0, "coordinator_restart"),
|
||||
InterruptionScenario::TargetEndpointBlackhole => (false, 1, "target_endpoint_blackhole"),
|
||||
};
|
||||
let topology = evidence_case
|
||||
.map(|case| case.topology)
|
||||
.unwrap_or_else(|| EvidenceTopology::new(4, 1));
|
||||
let erasure_set_drive_count = evidence_case
|
||||
.and_then(|case| case.erasure_set_drive_count)
|
||||
.unwrap_or_else(|| topology.total_drives());
|
||||
let outage_target_manifest_required = evidence_case.map(|case| case.outage_target_manifest_required).unwrap_or(true);
|
||||
init_logging();
|
||||
info!(
|
||||
event = "heal_interruption_started",
|
||||
@@ -1220,10 +1128,8 @@ mod tests {
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", storage_class);
|
||||
}
|
||||
if let Some(erasure_set_drive_count) = evidence_case.and_then(|case| case.erasure_set_drive_count) {
|
||||
cluster.set_env("RUSTFS_ERASURE_SET_DRIVE_COUNT", erasure_set_drive_count.to_string());
|
||||
cluster.set_env("RUSTFS_ERASURE_SET_DRIVE_COUNT", erasure_set_drive_count);
|
||||
}
|
||||
// Capture physical baselines after the PUT rename fanout has drained.
|
||||
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
// Heal control uses the first lexicographically sorted grid host.
|
||||
// Keep that coordinator distinct from the remote target at index 1.
|
||||
cluster.nodes.sort_by(|left, right| left.url.cmp(&right.url));
|
||||
@@ -1362,9 +1268,6 @@ mod tests {
|
||||
}
|
||||
for (drive_index, drive) in node.data_dirs.iter().enumerate() {
|
||||
let census = census_object_version_on_disk(Path::new(drive), bucket, outage_key, None)?;
|
||||
if !census.has_xl_meta {
|
||||
continue;
|
||||
}
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"online node {node_index} drive {drive_index} must hold a complete outage-object shard: {census:?}"
|
||||
@@ -1373,7 +1276,7 @@ mod tests {
|
||||
format!("online node {node_index} drive {drive_index} outage-object shard has no erasure index: {census:?}")
|
||||
})?;
|
||||
assert!(
|
||||
(1..=erasure_set_drive_count).contains(&erasure_index),
|
||||
(1..=topology.total_drives()).contains(&erasure_index),
|
||||
"online node {node_index} drive {drive_index} outage-object erasure index is out of range: {census:?}"
|
||||
);
|
||||
assert!(
|
||||
@@ -1382,26 +1285,19 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
!outage_peer_erasure_indices.is_empty() && outage_peer_erasure_indices.len() <= erasure_set_drive_count,
|
||||
"outage-object must occupy one non-empty erasure set"
|
||||
assert_eq!(
|
||||
outage_peer_erasure_indices.len(),
|
||||
topology.total_drives().saturating_sub(cluster.nodes[1].data_dirs.len()),
|
||||
"every online drive must contribute one unique outage-object erasure index"
|
||||
);
|
||||
if outage_target_manifest_required {
|
||||
let min_online_data_shards = erasure_set_drive_count.saturating_sub(4);
|
||||
assert!(
|
||||
outage_peer_erasure_indices.len() >= min_online_data_shards,
|
||||
"online drives in the selected erasure set must retain at least the EC data quorum"
|
||||
);
|
||||
}
|
||||
let missing_outage_erasure_indices = (1..=erasure_set_drive_count)
|
||||
let missing_outage_erasure_indices = (1..=topology.total_drives())
|
||||
.filter(|index| !outage_peer_erasure_indices.contains(index))
|
||||
.collect::<HashSet<_>>();
|
||||
if outage_target_manifest_required {
|
||||
assert!(
|
||||
!missing_outage_erasure_indices.is_empty(),
|
||||
"the stopped target must account for at least one missing outage-object erasure index"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
missing_outage_erasure_indices.len(),
|
||||
cluster.nodes[1].data_dirs.len(),
|
||||
"the stopped node must account for every missing outage-object erasure index"
|
||||
);
|
||||
|
||||
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
if !background_enabled {
|
||||
@@ -1673,9 +1569,7 @@ mod tests {
|
||||
} else {
|
||||
if matches!(
|
||||
scenario,
|
||||
InterruptionScenario::BackgroundTargetRestart
|
||||
| InterruptionScenario::BackgroundTargetRestartEc84
|
||||
| InterruptionScenario::BackgroundCoordinatorRestart
|
||||
InterruptionScenario::BackgroundTargetRestart | InterruptionScenario::BackgroundTargetRestartEc84
|
||||
) {
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
} else {
|
||||
@@ -1700,9 +1594,7 @@ mod tests {
|
||||
unclean_shutdown_marker_observed = Some(marker_exists);
|
||||
let expected_marker = matches!(
|
||||
scenario,
|
||||
InterruptionScenario::BackgroundTargetCrash
|
||||
| InterruptionScenario::BackgroundTargetCrashEc84
|
||||
| InterruptionScenario::BackgroundTargetCrashEc84MultiPool
|
||||
InterruptionScenario::BackgroundTargetCrash | InterruptionScenario::BackgroundTargetCrashEc84
|
||||
);
|
||||
assert!(
|
||||
marker_exists == expected_marker,
|
||||
@@ -1742,10 +1634,9 @@ mod tests {
|
||||
.unwrap_or(180);
|
||||
let heal_deadline = Instant::now() + Duration::from_secs(heal_timeout_secs);
|
||||
loop {
|
||||
let baseline_recovered = metadata_count(&replaced_disk, bucket, &expected_manifests) == expected_manifests.len();
|
||||
let outage_recovered =
|
||||
!outage_target_manifest_required || object_metadata_exists_on_disk(&replaced_disk, bucket, outage_key);
|
||||
if baseline_recovered && outage_recovered {
|
||||
if metadata_count(&replaced_disk, bucket, &expected_manifests) == expected_manifests.len()
|
||||
&& object_metadata_exists_on_disk(&replaced_disk, bucket, outage_key)
|
||||
{
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
let pool_metadata_matches = match &expected_pool_metadata {
|
||||
@@ -1755,10 +1646,7 @@ mod tests {
|
||||
}
|
||||
None => true,
|
||||
};
|
||||
if matching == expected_manifests.len()
|
||||
&& (!outage_target_manifest_required || outage_census.is_complete())
|
||||
&& pool_metadata_matches
|
||||
{
|
||||
if matching == expected_manifests.len() && outage_census.is_complete() && pool_metadata_matches {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -1803,19 +1691,17 @@ mod tests {
|
||||
);
|
||||
}
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
if outage_target_manifest_required {
|
||||
assert!(
|
||||
outage_census.is_complete(),
|
||||
"outage object must have a complete target shard: {outage_census:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
outage_census
|
||||
.erasure_index
|
||||
.filter(|index| missing_outage_erasure_indices.contains(index)),
|
||||
outage_census.erasure_index,
|
||||
"the outage object must be rebuilt into one of the stopped node's missing erasure slots"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
outage_census.is_complete(),
|
||||
"outage object must have a complete target shard: {outage_census:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
outage_census
|
||||
.erasure_index
|
||||
.filter(|index| missing_outage_erasure_indices.contains(index)),
|
||||
outage_census.erasure_index,
|
||||
"the outage object must be rebuilt into one of the stopped node's missing erasure slots"
|
||||
);
|
||||
|
||||
if let Some(cycle_end) = scanner_cycle_floor {
|
||||
wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
|
||||
@@ -1881,22 +1767,33 @@ mod tests {
|
||||
let task_status_body = signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let task_status: serde_json::Value = serde_json::from_str(&task_status_body)
|
||||
.map_err(|err| format!("heal task status is not JSON ({err}): {task_status_body}"))?;
|
||||
if task_status["summary"].as_str() != Some("finished") {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
}
|
||||
if interruption_node == 0 {
|
||||
// Restart recovery must finish the original durable root request.
|
||||
// Admin tasks are process-local. Physical and queue convergence
|
||||
// above establish recovery; a lost task must not report success.
|
||||
assert_eq!(
|
||||
task_status["summary"].as_str(),
|
||||
Some("notFound"),
|
||||
"interrupted task status: {task_status}"
|
||||
);
|
||||
assert_eq!(
|
||||
task_status["detail"].as_str(),
|
||||
Some("heal task not found or expired"),
|
||||
"interrupted admin task must be explicitly unavailable: {task_status}"
|
||||
);
|
||||
info!(
|
||||
event = "heal_interruption_recovered",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
interruption_node,
|
||||
interruption_kind,
|
||||
task_state = "finished",
|
||||
"Original root heal completed after coordinator restart"
|
||||
task_state = "not_found",
|
||||
"Physical recovery completed after coordinator restart"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
if task_status["summary"].as_str() != Some("finished") {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
}
|
||||
|
||||
if let Some(evidence_context) = evidence_run {
|
||||
let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
|
||||
@@ -1913,14 +1810,6 @@ mod tests {
|
||||
"binary_sha256": evidence_context.run.binary.sha256,
|
||||
"test_binary_sha256": evidence_context.run.test_binary.sha256,
|
||||
"topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
|
||||
"erasure_set_drive_count": erasure_set_drive_count,
|
||||
"sets": topology.set_count(evidence_context.case.erasure_set_drive_count),
|
||||
"pools": topology.pool_count(),
|
||||
"outage_target_manifest_required": outage_target_manifest_required,
|
||||
"distributed_ec_invalidation": true,
|
||||
"peer_count": cluster.nodes.len(),
|
||||
"same_window_remote_proof": true,
|
||||
"all_peers_bound_to_generation_window": true,
|
||||
"pid_before": target_pid, "pid_after": restarted_pid,
|
||||
"unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
|
||||
@@ -109,10 +109,10 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box<dyn std::error::
|
||||
.await;
|
||||
|
||||
let unavailable_error = put_result2.expect_err("a missing Local KMS key directory must reject encrypted writes");
|
||||
assert_eq!(unavailable_error.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
assert_eq!(unavailable_error.raw_response().map(|response| response.status().as_u16()), Some(500));
|
||||
assert_eq!(
|
||||
unavailable_error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("ServiceUnavailable")
|
||||
Some("InternalError")
|
||||
);
|
||||
let unavailable_absence = s3_client
|
||||
.get_object()
|
||||
|
||||
@@ -12,9 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(test)]
|
||||
mod storage_metric_ownership_test;
|
||||
|
||||
mod reliant;
|
||||
mod storage_api;
|
||||
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
//! Regression coverage for anonymous access on multipart control APIs.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use crate::kms::common::LocalKMSTestEnvironment;
|
||||
use async_compression::tokio::write::{BzEncoder, Lz4Encoder, XzEncoder};
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::operation::head_object::HeadObjectOutput;
|
||||
@@ -1466,10 +1465,10 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_s3() -> Result<(), B
|
||||
async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
let env = &kms_env.base_env;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let master_key = local_sse_master_key_value();
|
||||
env.start_rustfs_server_with_env(vec![], &[(LOCAL_SSE_MASTER_KEY_ENV, master_key.as_str())])
|
||||
.await?;
|
||||
|
||||
let bucket = "anon-post-default-sse-kms";
|
||||
let object_key = "post-default-sse-kms-object.txt";
|
||||
@@ -1485,7 +1484,7 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(),
|
||||
.apply_server_side_encryption_by_default(
|
||||
ServerSideEncryptionByDefault::builder()
|
||||
.sse_algorithm(ServerSideEncryption::AwsKms)
|
||||
.kms_master_key_id(default_key_id)
|
||||
.kms_master_key_id("test-key")
|
||||
.build()
|
||||
.expect("default encryption rule should build"),
|
||||
)
|
||||
|
||||
@@ -17,7 +17,6 @@ use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use bytes::Bytes;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::Barrier;
|
||||
use tracing::{info, warn};
|
||||
|
||||
@@ -26,307 +25,6 @@ const KEY: &str = "thumb/79/concurrent-overwrite.jpg";
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
async fn assert_degraded_cluster_publication_guard_errors_are_retryable() -> TestResult {
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2");
|
||||
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
|
||||
cluster.set_env("RUST_LOG", "warn");
|
||||
cluster.start().await?;
|
||||
cluster.create_test_bucket(BUCKET).await?;
|
||||
let clients: Vec<_> = cluster
|
||||
.create_all_clients()?
|
||||
.into_iter()
|
||||
.map(|client| {
|
||||
Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
for alive in (1..=4).rev() {
|
||||
if alive < 4 {
|
||||
cluster.stop_node(alive)?;
|
||||
}
|
||||
for (node, client) in clients.iter().take(alive).enumerate() {
|
||||
let key = format!("publication-put-{alive}-{node}");
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(&key)
|
||||
.body(Bytes::from_static(b"publication guard regression").into())
|
||||
.send()
|
||||
.await;
|
||||
if alive >= 3 {
|
||||
put?;
|
||||
} else {
|
||||
let err = put.expect_err("PUT must reject writes without a write quorum");
|
||||
assert_eq!(
|
||||
err.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(503),
|
||||
"PUT with {alive} nodes alive, requested through node {node}: {err:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
err.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable"),
|
||||
"PUT with {alive} nodes alive, requested through node {node}: {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
let multipart_key = format!("publication-multipart-{alive}-{node}");
|
||||
let multipart = client
|
||||
.create_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(&multipart_key)
|
||||
.send()
|
||||
.await;
|
||||
if alive >= 3 {
|
||||
let upload = multipart?;
|
||||
let upload_id = upload
|
||||
.upload_id()
|
||||
.expect("successful multipart initialization must return an upload ID");
|
||||
client
|
||||
.abort_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(&multipart_key)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await?;
|
||||
} else {
|
||||
let err = multipart.expect_err("multipart initialization must reject writes without a write quorum");
|
||||
assert_eq!(
|
||||
err.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(503),
|
||||
"CreateMultipartUpload with {alive} nodes alive, requested through node {node}: {err:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
err.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable"),
|
||||
"CreateMultipartUpload with {alive} nodes alive, requested through node {node}: {err:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cluster.stop();
|
||||
cluster.start().await?;
|
||||
for client in &clients {
|
||||
for alive in [1, 3, 4] {
|
||||
for node in 0..alive {
|
||||
let key = format!("publication-put-{alive}-{node}");
|
||||
let get = client.get_object().bucket(BUCKET).key(key).send().await;
|
||||
if alive >= 3 {
|
||||
assert_eq!(
|
||||
get?.body.collect().await?.into_bytes().as_ref(),
|
||||
b"publication guard regression",
|
||||
"acknowledged writes must survive restart"
|
||||
);
|
||||
} else {
|
||||
let err = get.expect_err("a rejected publication guard must not publish an object");
|
||||
assert_eq!(err.as_service_error().and_then(|error| error.meta().code()), Some("NoSuchKey"));
|
||||
}
|
||||
}
|
||||
}
|
||||
let uploads = client.list_multipart_uploads().bucket(BUCKET).send().await?;
|
||||
assert!(
|
||||
uploads
|
||||
.uploads()
|
||||
.iter()
|
||||
.all(|upload| upload.key() != Some("publication-multipart-1-0")),
|
||||
"a rejected publication guard must not publish a multipart upload"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_quorum_object_body(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
|
||||
let body = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await?
|
||||
.body
|
||||
.collect()
|
||||
.await?
|
||||
.into_bytes();
|
||||
assert_eq!(body.as_ref(), expected, "quorum read returned incorrect contents for {key}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_quorum_read_admission(clients: &[Client], bucket: &str) -> TestResult {
|
||||
// SIGKILL can orphan a granted lease. Wait for shared metadata-lock
|
||||
// admission before asserting the stable quorum boundary; cold bodies
|
||||
// remain unread throughout this readiness probe.
|
||||
let deadline =
|
||||
tokio::time::Instant::now() + rustfs_lock::fast_lock::DEFAULT_LOCK_TIMEOUT + std::time::Duration::from_secs(15);
|
||||
loop {
|
||||
let mut ready = true;
|
||||
for client in clients {
|
||||
for key in ["warm-small", "warm-large"] {
|
||||
match client.head_object().bucket(bucket).key(key).send().await {
|
||||
Ok(_) => {}
|
||||
Err(error) if error.raw_response().is_some_and(|response| response.status().as_u16() == 503) => {
|
||||
ready = false;
|
||||
break;
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
if !ready {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if ready {
|
||||
return Ok(());
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!("read quorum did not become available after lease convergence for {bucket}").into());
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
for (node_count, parity) in [(4, 2), (6, 3), (6, 2)] {
|
||||
let read_quorum = node_count - parity;
|
||||
let write_quorum = read_quorum + usize::from(read_quorum == parity);
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(node_count).await?;
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", format!("EC:{parity}"));
|
||||
// Wait for every seed fanout before removing any physical shard.
|
||||
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
|
||||
cluster.set_env("RUST_LOG", "warn,rustfs_lock=debug");
|
||||
cluster.start().await?;
|
||||
|
||||
let clients = cluster
|
||||
.create_all_clients()?
|
||||
.into_iter()
|
||||
.map(|client| {
|
||||
Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let bucket = format!("read-quorum-{node_count}-{parity}");
|
||||
clients[0].create_bucket().bucket(&bucket).send().await?;
|
||||
let small = b"read quorum is derived from the erasure layout".to_vec();
|
||||
let large = (0..1_048_576)
|
||||
.map(|index| u8::try_from(index % 251).expect("bounded payload byte"))
|
||||
.collect::<Vec<_>>();
|
||||
for (key, body) in [
|
||||
("warm-small", &small),
|
||||
("warm-large", &large),
|
||||
("cold-small", &small),
|
||||
("cold-large", &large),
|
||||
("below-quorum", &large),
|
||||
] {
|
||||
clients[node_count - 1]
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(Bytes::copy_from_slice(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
for node in &cluster.nodes {
|
||||
for key in ["warm-small", "warm-large", "cold-small", "cold-large", "below-quorum"] {
|
||||
let census =
|
||||
crate::chaos::census_object_version_on_disk(std::path::Path::new(&node.data_dir), &bucket, key, None)?;
|
||||
assert!(census.is_complete(), "seed shard must be complete before fault injection: {census:?}");
|
||||
assert_eq!(census.data_blocks, Some(read_quorum));
|
||||
assert_eq!(census.parity_blocks, Some(parity));
|
||||
}
|
||||
}
|
||||
for client in &clients {
|
||||
assert_quorum_object_body(client, &bucket, "warm-small", &small).await?;
|
||||
assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
|
||||
}
|
||||
|
||||
for offline_node in (read_quorum..node_count).rev() {
|
||||
cluster.stop_node(offline_node)?;
|
||||
wait_for_quorum_read_admission(&clients[..offline_node], &bucket).await?;
|
||||
for client in clients.iter().take(offline_node) {
|
||||
client.head_bucket().bucket(&bucket).send().await?;
|
||||
assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
|
||||
}
|
||||
}
|
||||
|
||||
// Exercise more than the five-second positive bucket-validation TTL.
|
||||
// Every sample must succeed; polling must not hide a transient failure.
|
||||
let validation_deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(6);
|
||||
loop {
|
||||
for client in clients.iter().take(read_quorum) {
|
||||
assert_quorum_object_body(client, &bucket, "warm-small", &small).await?;
|
||||
assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
|
||||
let listing = client.list_objects_v2().bucket(&bucket).send().await?;
|
||||
for key in ["warm-small", "warm-large", "cold-small", "cold-large", "below-quorum"] {
|
||||
assert!(listing.contents().iter().any(|entry| entry.key() == Some(key)), "listing omitted {key}");
|
||||
}
|
||||
}
|
||||
if tokio::time::Instant::now() >= validation_deadline {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
}
|
||||
for client in clients.iter().take(read_quorum) {
|
||||
assert_quorum_object_body(client, &bucket, "cold-small", &small).await?;
|
||||
assert_quorum_object_body(client, &bucket, "cold-large", &large).await?;
|
||||
}
|
||||
|
||||
let write = clients[0]
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key("quorum-write")
|
||||
.body(Bytes::copy_from_slice(&small).into())
|
||||
.send()
|
||||
.await;
|
||||
if read_quorum >= write_quorum {
|
||||
write?;
|
||||
} else {
|
||||
let error = write.expect_err("a read quorum must not authorize a write that needs more votes");
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
|
||||
}
|
||||
|
||||
cluster.stop_node(read_quorum - 1)?;
|
||||
for client in clients.iter().take(read_quorum - 1) {
|
||||
match client.get_object().bucket(&bucket).key("below-quorum").send().await {
|
||||
Ok(response) => assert!(
|
||||
response.body.collect().await.is_err(),
|
||||
"fewer than {read_quorum} valid fragments must not reconstruct an uncached object"
|
||||
),
|
||||
Err(error) => assert_eq!(
|
||||
error.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable"),
|
||||
"a quorum loss must not be mistaken for a missing object"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
for node in 0..read_quorum - 1 {
|
||||
cluster.stop_node(node)?;
|
||||
}
|
||||
cluster.start().await?;
|
||||
for client in &clients {
|
||||
assert_quorum_object_body(client, &bucket, "warm-large", &large).await?;
|
||||
assert_quorum_object_body(client, &bucket, "below-quorum", &large).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn put_object(client: Client, payload: Vec<u8>, writer_id: usize) -> Result<(), String> {
|
||||
client
|
||||
.put_object()
|
||||
@@ -360,9 +58,6 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum()
|
||||
// Keep the regression focused on false quorum-loss errors, not ordinary lock
|
||||
// wait exhaustion under a heavily contended same-key overwrite workload.
|
||||
cluster.set_env("RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT", "20");
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2");
|
||||
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
cluster.set_env("RUSTFS_HEALTH_MINIMAL_RESPONSE_ENABLE", "false");
|
||||
cluster.start().await?;
|
||||
cluster.create_test_bucket(BUCKET).await?;
|
||||
|
||||
@@ -421,181 +116,6 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum()
|
||||
);
|
||||
|
||||
clients[0].delete_object().bucket(BUCKET).key(KEY).send().await?;
|
||||
assert_node_readiness_tracks_quorum(&mut cluster).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvironment) -> TestResult {
|
||||
let clients: Vec<_> = cluster
|
||||
.create_all_clients()?
|
||||
.into_iter()
|
||||
.map(|client| {
|
||||
Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let http = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.timeout(Duration::from_secs(3))
|
||||
.build()?;
|
||||
let seed_key = "readiness-seed";
|
||||
let seed_body = b"readiness quorum regression";
|
||||
clients[0]
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(seed_key)
|
||||
.body(Bytes::from_static(seed_body).into())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
for (phase, survivors) in [4, 3, 2, 1, 4].into_iter().enumerate() {
|
||||
if phase == 4 {
|
||||
cluster.stop();
|
||||
cluster.start().await?;
|
||||
} else if survivors < 4 {
|
||||
cluster.stop_node(survivors)?;
|
||||
}
|
||||
let write_ready = survivors >= 3;
|
||||
let read_quorum = survivors >= 2;
|
||||
let expected_status = if write_ready { 200 } else { 503 };
|
||||
for (idx, client) in clients.iter().enumerate().take(survivors) {
|
||||
let url = &cluster.nodes[idx].url;
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
// Poll health before issuing S3 I/O: idle remote disk handles must
|
||||
// not remain evidence of quorum after their host becomes unreachable.
|
||||
let payload = loop {
|
||||
let response = http.get(format!("{url}/health/ready")).send().await?;
|
||||
let status = response.status().as_u16();
|
||||
let payload: serde_json::Value = response.json().await?;
|
||||
if status == expected_status
|
||||
&& payload["ready"] == write_ready
|
||||
&& payload["details"]["storage"]["ready"] == write_ready
|
||||
&& payload["details"]["storage"]["readQuorum"] == read_quorum
|
||||
&& payload["details"]["storage"]["writeQuorum"] == write_ready
|
||||
&& payload["details"]["poolMetadata"]["ready"] == true
|
||||
&& payload["details"]["iam"]["ready"] == true
|
||||
&& payload["details"]["lock"]["ready"] == write_ready
|
||||
{
|
||||
break payload;
|
||||
}
|
||||
assert!(Instant::now() < deadline, "node {idx}, survivors={survivors}: HTTP {status}, {payload}");
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
};
|
||||
assert_eq!(payload["details"]["storage"]["readinessScope"], "write_quorum_and_pool_metadata");
|
||||
assert_eq!(payload["details"]["storage"]["source"], "local_runtime");
|
||||
assert_eq!(
|
||||
payload["details"]["storage"]["status"],
|
||||
if write_ready { "connected" } else { "disconnected" }
|
||||
);
|
||||
if !write_ready {
|
||||
assert!(
|
||||
payload["degradedReasons"]
|
||||
.as_array()
|
||||
.expect("degraded reasons")
|
||||
.iter()
|
||||
.any(|reason| reason == "storage_and_lock_unavailable")
|
||||
);
|
||||
}
|
||||
|
||||
for path in ["/health/ready", "/minio/health/ready"] {
|
||||
let head = http.head(format!("{url}{path}")).send().await?;
|
||||
assert_eq!(head.status().as_u16(), expected_status, "HEAD {path}, survivors={survivors}");
|
||||
assert!(head.bytes().await?.is_empty());
|
||||
let response = http.get(format!("{url}{path}")).send().await?;
|
||||
assert_eq!(response.status().as_u16(), expected_status);
|
||||
let body: serde_json::Value = response.json().await?;
|
||||
assert_eq!(body["details"]["storage"], payload["details"]["storage"]);
|
||||
assert_eq!(body["details"]["poolMetadata"], payload["details"]["poolMetadata"]);
|
||||
}
|
||||
let live = http.get(format!("{url}/health/live")).send().await?;
|
||||
assert_eq!(live.status().as_u16(), 200);
|
||||
assert!(live.json::<serde_json::Value>().await?.get("details").is_none());
|
||||
for (path, storage_ready, scope) in [
|
||||
("/minio/health/cluster", write_ready, "write_quorum_and_pool_metadata"),
|
||||
("/minio/health/cluster/read", read_quorum, "read_quorum"),
|
||||
] {
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
// Cluster read/write reports have independent caches; allow
|
||||
// each observation to expire before comparing stable states.
|
||||
let body = loop {
|
||||
let response = http.get(format!("{url}{path}")).send().await?;
|
||||
let status = response.status().as_u16();
|
||||
let body: serde_json::Value = response.json().await?;
|
||||
if status == expected_status
|
||||
&& body["details"]["storage"]["ready"] == storage_ready
|
||||
&& body["details"]["lock"]["ready"] == write_ready
|
||||
{
|
||||
break body;
|
||||
}
|
||||
assert!(Instant::now() < deadline, "{path}, survivors={survivors}: HTTP {status}, {body}");
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
};
|
||||
assert_eq!(body["details"]["storage"]["readinessScope"], scope);
|
||||
}
|
||||
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(format!("readiness-phase-{phase}-node-{idx}"))
|
||||
.body(Bytes::from_static(seed_body).into())
|
||||
.send()
|
||||
.await;
|
||||
let put_status = if write_ready {
|
||||
put.expect("a ready node must accept the PUT");
|
||||
200
|
||||
} else {
|
||||
let error = put.expect_err("subquorum node must reject PUT");
|
||||
assert!(
|
||||
error.raw_response().is_some_and(|response| response.status().as_u16() >= 500),
|
||||
"unexpected PUT failure: {error:?}"
|
||||
);
|
||||
error.raw_response().expect("PUT error response").status().as_u16()
|
||||
};
|
||||
let get = client.get_object().bucket(BUCKET).key(seed_key).send().await;
|
||||
let get_status = match get {
|
||||
Ok(object) => {
|
||||
assert_eq!(object.body.collect().await?.into_bytes().as_ref(), seed_body);
|
||||
200
|
||||
}
|
||||
Err(error) => {
|
||||
assert!(!write_ready, "GET must succeed on a ready cluster: {error:?}");
|
||||
let status = error
|
||||
.raw_response()
|
||||
.expect("GET should have an HTTP response")
|
||||
.status()
|
||||
.as_u16();
|
||||
assert!(status >= 500, "unexpected GET failure: {error:?}");
|
||||
status
|
||||
}
|
||||
};
|
||||
let list = client.list_objects_v2().bucket(BUCKET).send().await;
|
||||
let list_status = match list {
|
||||
Ok(result) => {
|
||||
assert!(result.contents().iter().any(|object| object.key() == Some(seed_key)));
|
||||
200
|
||||
}
|
||||
Err(error) => {
|
||||
assert!(!write_ready, "listing must succeed on a ready cluster: {error:?}");
|
||||
let status = error
|
||||
.raw_response()
|
||||
.expect("LIST should have an HTTP response")
|
||||
.status()
|
||||
.as_u16();
|
||||
assert!(status >= 500, "unexpected listing failure: {error:?}");
|
||||
status
|
||||
}
|
||||
};
|
||||
eprintln!(
|
||||
"readiness matrix: survivors={survivors}, node={idx}, ready={write_ready}, read_quorum={read_quorum}, PUT={put_status}, GET={get_status}, LIST={list_status}"
|
||||
);
|
||||
}
|
||||
}
|
||||
cluster.stop();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -605,8 +125,6 @@ async fn assert_node_readiness_tracks_quorum(cluster: &mut RustFSTestClusterEnvi
|
||||
/// Before the fix, `map_namespace_lock_error` wrapped lock timeout/conflict errors as
|
||||
/// `StorageError::other(...)` → `StorageError::Io(...)`, which fell through to
|
||||
/// `S3ErrorCode::InternalError` (500) in the error mapping.
|
||||
/// Also checks PUT and multipart initialization when node failures prevent
|
||||
/// acquiring a table publication guard.
|
||||
#[tokio::test]
|
||||
async fn test_concurrent_put_same_key_never_returns_500() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
@@ -709,6 +227,5 @@ async fn test_concurrent_put_same_key_never_returns_500() -> TestResult {
|
||||
);
|
||||
|
||||
clients[0].delete_object().bucket(BUCKET).key(KEY).send().await?;
|
||||
cluster.stop();
|
||||
assert_degraded_cluster_publication_guard_errors_are_retryable().await
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1137,12 +1137,7 @@ async fn test_odm_admin_config_is_redacted_and_status_counts_match_the_source()
|
||||
let miss = env.raw_get(bucket, miss_key).await?;
|
||||
assert_eq!(miss.status, 404, "{}", String::from_utf8_lossy(&miss.body));
|
||||
}
|
||||
let (listed, _, _) = tokio::try_join!(
|
||||
env.wait_local_listed(bucket, hit_key, SETTLE),
|
||||
env.wait_for_status_counter(bucket, "/counters/pulled_objects_total/inline", 1, SETTLE),
|
||||
env.wait_for_status_counter(bucket, "/counters/pulled_bytes_total", body.len() as u64, SETTLE),
|
||||
)?;
|
||||
assert!(listed);
|
||||
assert!(env.wait_local_listed(bucket, hit_key, SETTLE).await?);
|
||||
|
||||
let status = env.status_json(bucket).await?;
|
||||
assert_eq!(status.pointer("/configured").and_then(Value::as_bool), Some(true), "{status}");
|
||||
|
||||
@@ -1,335 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Native OTLP -> Collector -> Prometheus contract, including a rolling upgrade.
|
||||
|
||||
use crate::common::RustFSTestClusterEnvironment;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use serde_json::Value;
|
||||
use std::fs::{self, File};
|
||||
use std::net::TcpListener;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
type TestResult<T = ()> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
struct ToolProcess(Child);
|
||||
impl Drop for ToolProcess {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
|
||||
fn required_binary(name: &str) -> TestResult<PathBuf> {
|
||||
let path = PathBuf::from(std::env::var(name).map_err(|_| format!("{name} must name a pinned executable"))?);
|
||||
if !path.is_file() {
|
||||
return Err(format!("{name} does not name a file: {}", path.display()).into());
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn free_port() -> TestResult<u16> {
|
||||
Ok(TcpListener::bind("127.0.0.1:0")?.local_addr()?.port())
|
||||
}
|
||||
|
||||
fn start_tool(binary: &Path, args: &[String], log: &Path) -> TestResult<ToolProcess> {
|
||||
let log = File::create(log)?;
|
||||
Ok(ToolProcess(
|
||||
Command::new(binary)
|
||||
.args(args)
|
||||
.env("NO_PROXY", "127.0.0.1,localhost")
|
||||
.env_remove("HTTP_PROXY")
|
||||
.env_remove("HTTPS_PROXY")
|
||||
.stdout(Stdio::from(log.try_clone()?))
|
||||
.stderr(Stdio::from(log))
|
||||
.spawn()?,
|
||||
))
|
||||
}
|
||||
|
||||
async fn query(client: &reqwest::Client, base: &str, expression: &str) -> TestResult<Value> {
|
||||
let mut url = reqwest::Url::parse(&format!("{base}/api/v1/query"))?;
|
||||
url.query_pairs_mut().append_pair("query", expression);
|
||||
let result: Value = client.get(url).send().await?.error_for_status()?.json().await?;
|
||||
if result["status"] != "success" {
|
||||
return Err(format!("PromQL failed: {result}").into());
|
||||
}
|
||||
Ok(result["data"]["result"].clone())
|
||||
}
|
||||
|
||||
async fn await_count(client: &reqwest::Client, base: &str, selector: &str, expected: u64) -> TestResult {
|
||||
let deadline = Instant::now() + Duration::from_secs(120);
|
||||
loop {
|
||||
let result = query(client, base, &format!("count({selector}) or vector(0)")).await;
|
||||
if let Ok(rows) = &result
|
||||
&& rows[0]["value"][1].as_str().and_then(|value| value.parse::<u64>().ok()) == Some(expected)
|
||||
{
|
||||
println!("PASS count={expected}: {selector}");
|
||||
return Ok(());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!("expected {expected} for {selector}; last result: {result:?}").into());
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn validate_dashboard_queries(client: &reqwest::Client, base: &str, observer: &str) -> TestResult {
|
||||
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.docker/observability/grafana/dashboards/rustfs.json");
|
||||
let dashboard: Value = serde_json::from_str(&fs::read_to_string(path)?)?;
|
||||
for name in ["storage_cluster", "storage_observer"] {
|
||||
let variable = dashboard["templating"]["list"]
|
||||
.as_array()
|
||||
.ok_or("dashboard variables")?
|
||||
.iter()
|
||||
.find(|variable| variable["name"] == name)
|
||||
.ok_or("storage selection variable")?;
|
||||
assert_eq!(variable["multi"], false, "storage views must select one {name}");
|
||||
assert_eq!(variable["includeAll"], false, "storage views must select one {name}");
|
||||
}
|
||||
let mut pending = dashboard["panels"]
|
||||
.as_array()
|
||||
.ok_or("dashboard panels")?
|
||||
.iter()
|
||||
.collect::<Vec<_>>();
|
||||
let mut checked = 0;
|
||||
while let Some(panel) = pending.pop() {
|
||||
if let Some(children) = panel["panels"].as_array() {
|
||||
pending.extend(children);
|
||||
}
|
||||
for target in panel["targets"].as_array().into_iter().flatten() {
|
||||
let Some(expression) = target["expr"].as_str() else { continue };
|
||||
if !expression.contains("rustfs:storage:current") {
|
||||
continue;
|
||||
}
|
||||
if expression.contains("collection_scope=\"cluster\"") {
|
||||
assert!(
|
||||
expression.contains("observer=\"$storage_observer\""),
|
||||
"global views must select one observer: {expression}"
|
||||
);
|
||||
}
|
||||
let mut expression = expression.to_string();
|
||||
for (name, value) in [
|
||||
("$__rate_interval", "5m"),
|
||||
("$storage_cluster", "metrics-e2e"),
|
||||
("$storage_observer", observer),
|
||||
("$drive_api", ".*"),
|
||||
("$server", ".*"),
|
||||
("$drive", ".*"),
|
||||
("$job", "rustfs"),
|
||||
] {
|
||||
expression = expression.replace(name, value);
|
||||
}
|
||||
query(client, base, &expression).await?;
|
||||
checked += 1;
|
||||
}
|
||||
}
|
||||
assert!(checked > 0, "the storage dashboard queries must be exercised");
|
||||
println!("PASS: {checked} storage dashboard queries against the live pipeline");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[ignore = "external tools: pinned Collector, Prometheus, previous release and current RustFS binaries"]
|
||||
async fn storage_metric_ownership_pipeline() -> TestResult {
|
||||
let baseline = required_binary("RUSTFS_METRICS_BASELINE_BINARY")?;
|
||||
let current = required_binary("CARGO_BIN_EXE_rustfs")?;
|
||||
let collector = required_binary("RUSTFS_OTELCOL_BINARY")?;
|
||||
let prometheus = required_binary("RUSTFS_PROMETHEUS_BINARY")?;
|
||||
let temp = tempfile::Builder::new().prefix("rustfs-storage-metrics-").tempdir()?;
|
||||
let work = if let Ok(path) = std::env::var("RUSTFS_METRICS_E2E_ARTIFACTS") {
|
||||
let path = PathBuf::from(path);
|
||||
fs::create_dir_all(&path)?;
|
||||
tempfile::Builder::new().prefix("storage-run-").tempdir_in(path)?.keep()
|
||||
} else {
|
||||
temp.path().to_path_buf()
|
||||
};
|
||||
println!("Metrics pipeline logs: {}", work.display());
|
||||
let otlp = free_port()?;
|
||||
let scrape = free_port()?;
|
||||
let prom = free_port()?;
|
||||
let collector_config = work.join("collector.yaml");
|
||||
fs::write(
|
||||
&collector_config,
|
||||
format!(
|
||||
r#"receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 127.0.0.1:{otlp}
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 127.0.0.1:{scrape}
|
||||
send_timestamps: true
|
||||
metric_expiration: 5m
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
service:
|
||||
telemetry:
|
||||
metrics:
|
||||
level: none
|
||||
logs:
|
||||
level: warn
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
exporters: [prometheus]
|
||||
"#
|
||||
),
|
||||
)?;
|
||||
let rules = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../.docker/observability/prometheus-rules/rustfs-storage.yml")
|
||||
.canonicalize()?;
|
||||
let prom_config = work.join("prometheus.yaml");
|
||||
fs::write(
|
||||
&prom_config,
|
||||
format!(
|
||||
r#"global:
|
||||
scrape_interval: 1s
|
||||
evaluation_interval: 1s
|
||||
rule_files:
|
||||
- '{}'
|
||||
scrape_configs:
|
||||
- job_name: rustfs
|
||||
static_configs:
|
||||
- targets: ['127.0.0.1:{scrape}']
|
||||
"#,
|
||||
rules.display()
|
||||
),
|
||||
)?;
|
||||
// Use the shipped expressions, with only the test evaluation interval shortened.
|
||||
let test_rules = work.join("storage-rules.yaml");
|
||||
fs::write(&test_rules, fs::read_to_string(&rules)?.replace("interval: 15s", "interval: 1s"))?;
|
||||
fs::write(
|
||||
&prom_config,
|
||||
fs::read_to_string(&prom_config)?.replace(&rules.display().to_string(), &test_rules.display().to_string()),
|
||||
)?;
|
||||
let _collector = start_tool(
|
||||
&collector,
|
||||
&[format!("--config={}", collector_config.display())],
|
||||
&work.join("collector.log"),
|
||||
)?;
|
||||
let _prometheus = start_tool(
|
||||
&prometheus,
|
||||
&[
|
||||
format!("--config.file={}", prom_config.display()),
|
||||
format!("--web.listen-address=127.0.0.1:{prom}"),
|
||||
format!("--storage.tsdb.path={}", work.join("prometheus-data").display()),
|
||||
],
|
||||
&work.join("prometheus.log"),
|
||||
)?;
|
||||
let http = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.timeout(Duration::from_secs(5))
|
||||
.build()?;
|
||||
let prom_url = format!("http://127.0.0.1:{prom}");
|
||||
await_count(&http, &prom_url, "up{job=\"rustfs\"} == 1", 1).await?;
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
cluster.set_env("NO_PROXY", "127.0.0.1,localhost");
|
||||
cluster.set_env("RUSTFS_OBS_METRIC_ENDPOINT", format!("http://127.0.0.1:{otlp}/v1/metrics"));
|
||||
cluster.set_env("OTEL_RESOURCE_ATTRIBUTES", "rustfs.cluster.id=metrics-e2e");
|
||||
cluster.set_env("RUSTFS_OBS_METER_INTERVAL", "2");
|
||||
cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_OBS_LOGS_EXPORT_ENABLED", "false");
|
||||
cluster.set_env("RUSTFS_OBS_TRACES_EXPORT_ENABLED", "false");
|
||||
cluster.set_env("RUSTFS_METRICS_NODE_INTERVAL", "2");
|
||||
cluster.set_env("RUSTFS_METRICS_CLUSTER_INTERVAL", "5");
|
||||
for index in 0..4 {
|
||||
// Four localhost processes otherwise share the startup resource IP.
|
||||
// Distinct test host IDs model the four hosts in a distributed deployment.
|
||||
cluster.set_node_env(
|
||||
index,
|
||||
"OTEL_RESOURCE_ATTRIBUTES",
|
||||
format!("rustfs.cluster.id=metrics-e2e,host.id=metrics-node-{index}"),
|
||||
)?;
|
||||
cluster.set_node_capture_log_path(index, work.join(format!("node-{index}.log")).display().to_string())?;
|
||||
}
|
||||
cluster.start_with_binary(&baseline).await?;
|
||||
// Reproduce the original four observers x four global drives before fixing it.
|
||||
await_count(&http, &prom_url, "rustfs_system_drive_total_bytes{collection_scope=\"\",drive!=\"\"}", 16).await?;
|
||||
let local = "rustfs:storage:current{source_metric=\"rustfs_system_drive_total_bytes\",collection_scope=\"local\",rustfs_cluster_id=\"metrics-e2e\"}";
|
||||
for index in 0..4 {
|
||||
cluster.stop_node_gracefully(index).await?;
|
||||
cluster.start_node_from_binary(index, ¤t).await?;
|
||||
await_count(&http, &prom_url, local, u64::try_from(index + 1)?).await?;
|
||||
}
|
||||
let rows = query(&http, &prom_url, local).await?;
|
||||
for row in rows.as_array().ok_or("expected a metric vector")? {
|
||||
assert_eq!(
|
||||
row["metric"]["observer"], row["metric"]["server"],
|
||||
"a node must only export its own detailed drives"
|
||||
);
|
||||
}
|
||||
let observer = cluster.nodes[0].address.clone();
|
||||
let inventory = format!(
|
||||
"rustfs:storage:current{{source_metric=\"rustfs_cluster_drive_present\",collection_scope=\"cluster\",rustfs_cluster_id=\"metrics-e2e\",observer=\"{observer}\"}}"
|
||||
);
|
||||
await_count(&http, &prom_url, &inventory, 4).await?;
|
||||
cluster.create_test_bucket("metrics-ownership").await?;
|
||||
let client = cluster.create_s3_client(0)?;
|
||||
for index in 0..8 {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("metrics-ownership")
|
||||
.key(format!("object-{index}"))
|
||||
.body(ByteStream::from(vec![7_u8; 4096]))
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
await_count(
|
||||
&http,
|
||||
&prom_url,
|
||||
"count by (server) (rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\"})",
|
||||
4,
|
||||
).await?;
|
||||
let counters = query(
|
||||
&http,
|
||||
&prom_url,
|
||||
"rustfs:storage:current{source_metric=\"rustfs_system_drive_api_calls_total\",collection_scope=\"local\"}",
|
||||
)
|
||||
.await?;
|
||||
assert!(
|
||||
!counters.as_array().ok_or("expected counters")?.is_empty(),
|
||||
"exercise actual storage counters"
|
||||
);
|
||||
for row in counters.as_array().ok_or("expected counters")? {
|
||||
assert_eq!(row["metric"]["observer"], row["metric"]["server"]);
|
||||
assert!(
|
||||
!row["metric"]["disk_id"].as_str().unwrap_or_default().is_empty(),
|
||||
"counters must carry physical disk identity"
|
||||
);
|
||||
}
|
||||
validate_dashboard_queries(&http, &prom_url, &observer).await?;
|
||||
for index in (1..4).rev() {
|
||||
cluster.stop_node(index)?;
|
||||
// The Collector stays alive; cached samples must not keep stopped owners fresh.
|
||||
await_count(&http, &prom_url, local, u64::try_from(index)?).await?;
|
||||
await_count(&http, &prom_url, &inventory, 4).await?;
|
||||
let unavailable = format!(
|
||||
"rustfs:storage:current{{source_metric=\"rustfs_cluster_drive_runtime_state\",collection_scope=\"cluster\",rustfs_cluster_id=\"metrics-e2e\",observer=\"{observer}\",state=~\"offline|unknown|suspect\"}} == 1"
|
||||
);
|
||||
await_count(&http, &prom_url, &unavailable, u64::try_from(4 - index)?).await?;
|
||||
}
|
||||
cluster.stop();
|
||||
await_count(&http, &prom_url, local, 0).await?;
|
||||
cluster.start_with_binary(¤t).await?;
|
||||
await_count(&http, &prom_url, local, 4).await?;
|
||||
await_count(&http, &prom_url, &inventory, 4).await?;
|
||||
let restored = client.get_object().bucket("metrics-ownership").key("object-0").send().await?;
|
||||
assert_eq!(restored.body.collect().await?.into_bytes().as_ref(), vec![7_u8; 4096].as_slice());
|
||||
println!("PASS: baseline duplication; four rolling upgrades; owner identity; counters; 4 -> 3 -> 2 -> 1 -> 0 -> 4 recovery");
|
||||
Ok(())
|
||||
}
|
||||
@@ -38,16 +38,6 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod lifecycle {
|
||||
pub mod legacy_transition_state_reconcile {
|
||||
pub use crate::bucket::lifecycle::legacy_transition_state_reconcile::{
|
||||
LegacyTransitionStateCopyRepresentation, LegacyTransitionStateMetadataAlias, LegacyTransitionStateReconcileError,
|
||||
LegacyTransitionStateReconcileOutcome, LegacyTransitionStateReconcileReadiness,
|
||||
LegacyTransitionStateReconcileRequest, LegacyTransitionStateReconcileResponse,
|
||||
LegacyTransitionStateReconcileSelector, LegacyTransitionStateSetRepresentation, LegacyTransitionStateSource,
|
||||
LegacyTransitionStateTarget,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod bucket_lifecycle_audit {
|
||||
pub use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
|
||||
}
|
||||
@@ -270,19 +260,18 @@ pub mod bucket {
|
||||
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
|
||||
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
||||
ReplicationState, ReplicationStats, ReplicationStatusType, ReplicationStorage, ReplicationTargetValidationError,
|
||||
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, ScannerDirtyUsageMutationObserver,
|
||||
ScannerDirtyUsageMutationSource, TargetReplicationResyncStatus, VersionPurgeStatusType, XferStats,
|
||||
assign_site_replication_rule_priorities, commit_force_delete_intent, complete_force_delete_intent,
|
||||
delete_replication_state_from_config, delete_replication_version_id, get_global_replication_pool,
|
||||
get_global_replication_stats, get_proxy_targets, init_background_replication,
|
||||
ReplicationType, ResyncOpts, ResyncStatusType, RuntimeReplicationTargetBacklog, TargetReplicationResyncStatus,
|
||||
VersionPurgeStatusType, XferStats, assign_site_replication_rule_priorities, commit_force_delete_intent,
|
||||
complete_force_delete_intent, delete_replication_state_from_config, delete_replication_version_id,
|
||||
get_global_replication_pool, get_global_replication_stats, get_proxy_targets, init_background_replication,
|
||||
invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
|
||||
merge_incoming_replication_config, merge_user_replication_config, persist_force_delete_intent,
|
||||
read_durable_mrf_backlog, replication_state_to_filemeta, replication_status_to_filemeta, replication_statuses_map,
|
||||
replication_target_arn_deployment_id, replication_target_arns, resync_start_conflict_id,
|
||||
set_scanner_dirty_usage_mutation_observer, should_remove_replication_target, should_schedule_delete_replication,
|
||||
should_use_existing_delete_replication_info, should_use_existing_delete_replication_source,
|
||||
site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
|
||||
validate_replication_config_target_arns, version_purge_status_to_filemeta,
|
||||
should_remove_replication_target, should_schedule_delete_replication, should_use_existing_delete_replication_info,
|
||||
should_use_existing_delete_replication_source, site_replication_rule_deployment_id,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
version_purge_status_to_filemeta,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -377,14 +366,6 @@ pub mod config {
|
||||
}
|
||||
|
||||
pub mod data_usage {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::data_movement::SourceCleanupDeleteBarrier;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::data_movement::scanner_backlog::test_util::NativeScannerPauseBacklogWriteFault;
|
||||
pub use crate::data_movement::scanner_backlog::{
|
||||
MAX_SCANNER_PAUSE_BACKLOG_BYTES, ScannerPauseBacklogRetirementPlan, ScannerPauseBacklogRetirementPlanner,
|
||||
ScannerPauseBacklogRetirementReplica, register_scanner_pause_backlog_retirement_planner,
|
||||
};
|
||||
pub use crate::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
|
||||
|
||||
@@ -966,11 +966,6 @@ async fn cleanup_free_version_exact(api: Arc<ECStore>, oi: &ObjectInfo, cancel:
|
||||
if let Some(err) = first_error {
|
||||
return Err(err);
|
||||
}
|
||||
runtime_sources::notify_scanner_dirty_usage_mutation(
|
||||
&oi.bucket,
|
||||
&oi.name,
|
||||
runtime_sources::ScannerDirtyUsageMutationSource::TierExpiration,
|
||||
);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
@@ -4754,11 +4749,6 @@ async fn expire_transitioned_object_with_lock_lost_signal(
|
||||
// Drop any cached restored-copy body so it does not sit resident
|
||||
// until TTL after the copy is expired (ODC-26).
|
||||
crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await;
|
||||
runtime_sources::notify_scanner_dirty_usage_mutation(
|
||||
&oi.bucket,
|
||||
&oi.name,
|
||||
runtime_sources::ScannerDirtyUsageMutationSource::TierExpiration,
|
||||
);
|
||||
//audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn);
|
||||
Ok(dobj)
|
||||
}
|
||||
@@ -4794,11 +4784,6 @@ async fn expire_transitioned_object_with_lock_lost_signal(
|
||||
// The transitioned version is gone; evict any cached body for this object
|
||||
// so it does not linger until TTL (ODC-26).
|
||||
crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await;
|
||||
runtime_sources::notify_scanner_dirty_usage_mutation(
|
||||
&oi.bucket,
|
||||
&oi.name,
|
||||
runtime_sources::ScannerDirtyUsageMutationSource::TierExpiration,
|
||||
);
|
||||
|
||||
//audit_log_lifecycle(oi, ILMExpiry, tags);
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,7 +18,6 @@ mod config_boundary;
|
||||
pub mod core;
|
||||
mod durable_namespace;
|
||||
pub mod evaluator;
|
||||
pub mod legacy_transition_state_reconcile;
|
||||
pub mod manual_transition_job;
|
||||
mod metadata_boundary;
|
||||
pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs, get_lifecycle_config};
|
||||
|
||||
@@ -20,7 +20,6 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::{ExpiryState, TransitionState};
|
||||
use crate::runtime::sources;
|
||||
pub(crate) use crate::runtime::sources::ScannerDirtyUsageMutationSource;
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
use crate::store::ECStore;
|
||||
|
||||
@@ -55,7 +54,3 @@ pub(crate) fn deployment_id() -> Option<String> {
|
||||
pub(crate) async fn bucket_lifecycle_config(bucket: &str) -> Option<BucketLifecycleConfiguration> {
|
||||
sources::bucket_lifecycle_config(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) fn notify_scanner_dirty_usage_mutation(bucket: &str, object: &str, source: ScannerDirtyUsageMutationSource) {
|
||||
sources::notify_scanner_dirty_usage_mutation(bucket, object, source);
|
||||
}
|
||||
|
||||
@@ -1681,13 +1681,9 @@ impl BucketMetadataSys {
|
||||
expected: Option<&Arc<BucketMetadata>>,
|
||||
namespace_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<()> {
|
||||
if !await_bucket_namespace_operation(
|
||||
Some(namespace_guard),
|
||||
bucket,
|
||||
"bucket metadata heal existence check",
|
||||
self.api.bucket_exists_for_heal(bucket),
|
||||
)
|
||||
.await?
|
||||
if !self
|
||||
.bucket_exists(bucket, namespace_guard, "bucket metadata existence check")
|
||||
.await?
|
||||
{
|
||||
if matches!(mode, MetadataLoadMode::Refresh) {
|
||||
let _publish_guard = self
|
||||
|
||||
@@ -92,6 +92,3 @@ pub use replication_target_boundary::SsecPassthroughCapability;
|
||||
pub use replication_target_boundary::VersionIdentityCapability;
|
||||
pub use replication_target_boundary::{ObjectLockIntegrity, object_lock_put_integrity};
|
||||
pub(crate) use replication_target_config_bridge::ReplicationTargetConfigBridge;
|
||||
pub use runtime_boundary::{
|
||||
ScannerDirtyUsageMutationObserver, ScannerDirtyUsageMutationSource, set_scanner_dirty_usage_mutation_observer,
|
||||
};
|
||||
|
||||
@@ -3843,13 +3843,6 @@ async fn persist_replication_state_if_current<S: ReplicationStorage>(
|
||||
match storage.put_object_metadata(&roi.bucket, &roi.name, &write_opts).await {
|
||||
Ok(updated) => {
|
||||
*object_info = updated;
|
||||
if mode == ReplicationStatusWritebackMode::Update {
|
||||
runtime_sources::notify_scanner_dirty_usage_mutation(
|
||||
&roi.bucket,
|
||||
&roi.name,
|
||||
runtime_sources::ScannerDirtyUsageMutationSource::Replication,
|
||||
);
|
||||
}
|
||||
Ok(ReplicationStatePersistOutcome::Updated)
|
||||
}
|
||||
Err(Error::PreconditionFailed) => Ok(ReplicationStatePersistOutcome::Superseded),
|
||||
|
||||
@@ -19,9 +19,6 @@ use super::replication_pool::DynReplicationPool;
|
||||
use super::replication_state::ReplicationStats;
|
||||
use super::replication_storage_boundary::ReplicationObjectStore;
|
||||
use crate::runtime::sources;
|
||||
pub use crate::runtime::sources::{
|
||||
ScannerDirtyUsageMutationObserver, ScannerDirtyUsageMutationSource, set_scanner_dirty_usage_mutation_observer,
|
||||
};
|
||||
|
||||
pub(crate) fn object_store_handle() -> Option<Arc<ReplicationObjectStore>> {
|
||||
sources::object_store_handle()
|
||||
@@ -46,7 +43,3 @@ pub(crate) fn replication_runtime_initialized() -> bool {
|
||||
pub(crate) fn bucket_monitor() -> Option<Arc<ReplicationBucketMonitor>> {
|
||||
sources::bucket_monitor()
|
||||
}
|
||||
|
||||
pub(crate) fn notify_scanner_dirty_usage_mutation(bucket: &str, object: &str, source: ScannerDirtyUsageMutationSource) {
|
||||
sources::notify_scanner_dirty_usage_mutation(bucket, object, source);
|
||||
}
|
||||
|
||||
@@ -51,10 +51,10 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ReplacementRecoveryStatusRequest,
|
||||
ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageSnapshotRequest, ScannerDirtyUsageSnapshotResponse,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
|
||||
ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse, ScannerScopedDirtyUsageEntry, ServerInfoRequest,
|
||||
SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest,
|
||||
TierDailyStatsRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
|
||||
TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageEntry, ServerInfoRequest, SignalServiceRequest,
|
||||
SignalServiceResponse, StartDecommissionRequest, StartProfilingRequest, StopRebalanceRequest, TierDailyStatsRequest,
|
||||
TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse, TierMutationFailureClass,
|
||||
TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
};
|
||||
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
|
||||
@@ -289,20 +289,6 @@ fn scanner_scoped_dirty_usage_ack_payload(
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_response_matches(
|
||||
request: &ScannerScopedDirtyUsageAckRequest,
|
||||
response: &ScannerScopedDirtyUsageAckResponse,
|
||||
) -> bool {
|
||||
let cleared_within_request = u64::try_from(request.entries.len())
|
||||
.is_ok_and(|entry_count| response.cleared <= entry_count && (!request.probe_only || response.cleared == 0));
|
||||
response.protocol_version == rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|
||||
&& response.owner_id == request.owner_id
|
||||
&& response.instance_id == request.instance_id
|
||||
&& response.max_entries == rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_ENTRIES
|
||||
&& response.max_request_bytes == rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|
||||
&& cleared_within_request
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_reconciled(activity: &ScannerPeerActivity, expected_instance_id: &str) -> bool {
|
||||
activity.instance_id == expected_instance_id && activity.dirty_usage_pending == Some(false)
|
||||
}
|
||||
@@ -2269,7 +2255,13 @@ impl PeerRestClient {
|
||||
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
|
||||
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
|
||||
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
|
||||
if !scanner_scoped_dirty_usage_ack_response_matches(&payload, &response) {
|
||||
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|
||||
|| response.owner_id != payload.owner_id
|
||||
|| response.instance_id != payload.instance_id
|
||||
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|
||||
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|
||||
|| response.cleared != 0
|
||||
{
|
||||
return Err(Error::other("scoped dirty usage capability response does not match request"));
|
||||
}
|
||||
if !response.supported {
|
||||
@@ -2305,7 +2297,13 @@ impl PeerRestClient {
|
||||
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
|
||||
.map_err(|_| Error::other("scoped dirty usage acknowledgement response is too large"))?;
|
||||
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
|
||||
if !scanner_scoped_dirty_usage_ack_response_matches(&payload, &response) || !response.supported {
|
||||
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|
||||
|| response.owner_id != payload.owner_id
|
||||
|| response.instance_id != payload.instance_id
|
||||
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|
||||
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|
||||
|| !response.supported
|
||||
{
|
||||
return Err(Error::other("scoped dirty usage acknowledgement response does not match request"));
|
||||
}
|
||||
}
|
||||
@@ -3120,60 +3118,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_response_bounds_cleared_entries_to_request() {
|
||||
use rustfs_protos::scoped_dirty_usage::{
|
||||
SCOPED_DIRTY_USAGE_MAX_ENTRIES, SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES, SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
|
||||
};
|
||||
|
||||
let mut request = scanner_scoped_dirty_usage_ack_payload(
|
||||
"33333333-3333-3333-3333-333333333333",
|
||||
"0123456789abcdef0123456789abcdef",
|
||||
false,
|
||||
vec![
|
||||
ScannerScopedDirtyUsageEntry {
|
||||
bucket: "archive".to_string(),
|
||||
bucket_incarnation: Uuid::from_u128(0x11111111111111111111111111111111).as_bytes().to_vec().into(),
|
||||
generation: 3,
|
||||
},
|
||||
ScannerScopedDirtyUsageEntry {
|
||||
bucket: "photos".to_string(),
|
||||
bucket_incarnation: Uuid::from_u128(0x22222222222222222222222222222222).as_bytes().to_vec().into(),
|
||||
generation: 7,
|
||||
},
|
||||
],
|
||||
)
|
||||
.expect("two ordered entries should form a valid scoped ACK request");
|
||||
let mut response = ScannerScopedDirtyUsageAckResponse {
|
||||
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
|
||||
owner_id: request.owner_id.clone(),
|
||||
instance_id: request.instance_id.clone(),
|
||||
supported: true,
|
||||
max_entries: SCOPED_DIRTY_USAGE_MAX_ENTRIES,
|
||||
max_request_bytes: SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES,
|
||||
cleared: 1,
|
||||
response_proof: Bytes::new(),
|
||||
};
|
||||
|
||||
assert!(scanner_scoped_dirty_usage_ack_response_matches(&request, &response));
|
||||
response.cleared = 2;
|
||||
assert!(scanner_scoped_dirty_usage_ack_response_matches(&request, &response));
|
||||
response.cleared = 3;
|
||||
assert!(
|
||||
!scanner_scoped_dirty_usage_ack_response_matches(&request, &response),
|
||||
"a peer cannot clear more entries than the signed request contains"
|
||||
);
|
||||
|
||||
request.probe_only = true;
|
||||
response.cleared = 1;
|
||||
assert!(
|
||||
!scanner_scoped_dirty_usage_ack_response_matches(&request, &response),
|
||||
"a capability probe cannot report a mutation"
|
||||
);
|
||||
response.cleared = 0;
|
||||
assert!(scanner_scoped_dirty_usage_ack_response_matches(&request, &response));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_reconciliation_requires_same_clean_instance() {
|
||||
let activity = |instance_id: &str, pending| ScannerPeerActivity {
|
||||
|
||||
+78
-2278
File diff suppressed because it is too large
Load Diff
@@ -5256,35 +5256,16 @@ mod decommission_lock_order_tests {
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn data_movement_existing_replica_reconciles_capacity_and_cleans_source() {
|
||||
data_movement_existing_replica_reconciles_capacity_case(false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn data_movement_existing_replica_outside_reservation_uses_reserved_target() {
|
||||
data_movement_existing_replica_reconciles_capacity_case(true);
|
||||
}
|
||||
|
||||
fn data_movement_existing_replica_reconciles_capacity_case(existing_outside_reservation: bool) {
|
||||
run_large_stack_current_thread_async_test("reserved-replica-reconcile", async move || {
|
||||
fn scanner_backlog_native_replica_reconciles_capacity_and_cleans_source() {
|
||||
run_large_stack_current_thread_async_test("scanner-backlog-reconcile", async || {
|
||||
let (_temp_dirs, store, other_store) =
|
||||
test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(None).await;
|
||||
let object = "buckets/reserved-replica-routing.json";
|
||||
let object = "buckets/.scanner-pause-backlog.json";
|
||||
let body = br#"{"schemaVersion":1,"generation":2}"#.to_vec();
|
||||
let old_body = br#"{"schemaVersion":1,"generation":1}"#.to_vec();
|
||||
let source_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(20);
|
||||
let target_time = source_time;
|
||||
let target_pool_index = if existing_outside_reservation { 1 } else { 2 };
|
||||
let mut replicas = vec![(0, body.clone(), source_time), (target_pool_index, old_body, target_time)];
|
||||
if existing_outside_reservation {
|
||||
replicas.push((
|
||||
2,
|
||||
br#"{"schemaVersion":1,"generation":3}"#.to_vec(),
|
||||
source_time + time::Duration::seconds(10),
|
||||
));
|
||||
}
|
||||
for (pool_index, payload, mod_time) in replicas.iter().cloned() {
|
||||
let target_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(10);
|
||||
for (pool_index, payload, mod_time) in [(0, body.clone(), source_time), (2, old_body, target_time)] {
|
||||
store.pools[pool_index]
|
||||
.put_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
@@ -5297,19 +5278,13 @@ mod decommission_lock_order_tests {
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed existing replicas with independent write times");
|
||||
.expect("seed native scanner replicas with independent write times");
|
||||
}
|
||||
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
|
||||
let target_total = body.len() * 8;
|
||||
let capacities = vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, layout, 0, body.len() * 2, body.len() * 2),
|
||||
DecommissionPoolCapacityInfo::for_test(
|
||||
1,
|
||||
layout,
|
||||
if existing_outside_reservation { target_total } else { 0 },
|
||||
target_total,
|
||||
if existing_outside_reservation { 0 } else { target_total },
|
||||
),
|
||||
DecommissionPoolCapacityInfo::for_test(1, layout, 0, target_total, target_total),
|
||||
DecommissionPoolCapacityInfo::for_test(2, layout, target_total, target_total, 0),
|
||||
];
|
||||
set_decommission_capacity_info_overrides_for_test(store.id, vec![capacities.clone()]);
|
||||
@@ -5318,17 +5293,6 @@ mod decommission_lock_order_tests {
|
||||
.await
|
||||
.expect("activate the source reservation");
|
||||
let owner = decommission_capacity_owner(&*store.pool_meta.read().await);
|
||||
let reserved_snapshot = store.pool_meta.read().await.clone();
|
||||
let reservation = reserved_snapshot.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.expect("active source reservation");
|
||||
assert_eq!(
|
||||
reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>(),
|
||||
vec![target_pool_index],
|
||||
"the fixture must reserve exactly one target"
|
||||
);
|
||||
let source_reader = store.pools[0]
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
@@ -5350,91 +5314,12 @@ mod decommission_lock_order_tests {
|
||||
RUSTFS_META_BUCKET.to_string(),
|
||||
source_reader,
|
||||
None,
|
||||
"reserved_replica_conflict",
|
||||
"scanner_backlog_conflict",
|
||||
Some(owner),
|
||||
)
|
||||
.await
|
||||
.expect_err("a different older existing record must retain its source and capacity intent");
|
||||
.expect_err("a different older native ledger must retain its source and capacity intent");
|
||||
assert!(conflict.to_string().contains("Precondition failed"), "unexpected conflict: {conflict}");
|
||||
let reserved_snapshot = store.pool_meta.read().await.clone();
|
||||
let mut selection_opts = ObjectOptions {
|
||||
data_movement: true,
|
||||
src_pool_idx: 0,
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
store
|
||||
.select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true)
|
||||
.await
|
||||
.expect("selection without a capacity owner retains existing-replica routing"),
|
||||
2
|
||||
);
|
||||
owner.apply_to(&mut selection_opts);
|
||||
for stale_owner in [
|
||||
DecommissionCapacityOwner {
|
||||
owner_nonce: uuid::Uuid::new_v4(),
|
||||
..owner
|
||||
},
|
||||
DecommissionCapacityOwner {
|
||||
generation: owner.generation + 1,
|
||||
..owner
|
||||
},
|
||||
] {
|
||||
let mut stale_opts = selection_opts.clone();
|
||||
stale_owner.apply_to(&mut stale_opts);
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &stale_opts, true)
|
||||
.await,
|
||||
Err(crate::error::Error::DecommissionCapacityBlocked { .. })
|
||||
),
|
||||
"a stale owner must not fall back to another target"
|
||||
);
|
||||
}
|
||||
{
|
||||
let mut meta = store.pool_meta.write().await;
|
||||
meta.pools[0]
|
||||
.decommission
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.capacity_reservation
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.expires_at = time::OffsetDateTime::now_utc() - time::Duration::seconds(1);
|
||||
}
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true)
|
||||
.await,
|
||||
Err(crate::error::Error::DecommissionCapacityBlocked { .. })
|
||||
),
|
||||
"an expired owner must not fall back to another target"
|
||||
);
|
||||
*store.pool_meta.write().await = reserved_snapshot.clone();
|
||||
if !existing_outside_reservation {
|
||||
{
|
||||
let mut meta = store.pool_meta.write().await;
|
||||
let target = &mut meta.pools[0]
|
||||
.decommission
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.capacity_reservation
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.targets[0];
|
||||
target.consumed_physical_bytes = target.reserved_physical_bytes;
|
||||
}
|
||||
assert_eq!(
|
||||
store
|
||||
.select_data_movement_pool_idx(RUSTFS_META_BUCKET, object, body.len() as i64, &selection_opts, true)
|
||||
.await
|
||||
.expect("an existing reserved replica can still be selected after capacity was consumed"),
|
||||
target_pool_index
|
||||
);
|
||||
*store.pool_meta.write().await = reserved_snapshot;
|
||||
}
|
||||
let mut persisted = crate::core::pools::PoolMeta::default();
|
||||
persisted
|
||||
.load_no_lock_from_replicas(store.pools.clone())
|
||||
@@ -5451,24 +5336,11 @@ mod decommission_lock_order_tests {
|
||||
.pending_target_physical_bytes,
|
||||
body.len()
|
||||
);
|
||||
for (pool_index, payload, mod_time) in &replicas {
|
||||
let mut reader = store.pools[*pool_index]
|
||||
.get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("a refused existing record replacement must preserve every replica");
|
||||
assert_eq!(reader.object_info.mod_time, Some(*mod_time));
|
||||
let mut actual = Vec::new();
|
||||
reader
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("read the unchanged existing record");
|
||||
assert_eq!(&actual, payload);
|
||||
}
|
||||
let previous = store.pools[target_pool_index]
|
||||
let previous = store.pools[2]
|
||||
.get_object_info(RUSTFS_META_BUCKET, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read the existing writer's CAS revision");
|
||||
let replacement = store.pools[target_pool_index]
|
||||
.expect("read the native writer's CAS revision");
|
||||
let replacement = store.pools[2]
|
||||
.put_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
object,
|
||||
@@ -5484,7 +5356,7 @@ mod decommission_lock_order_tests {
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("existing CAS converges the payload without a migration marker");
|
||||
.expect("native scanner CAS converges the payload without a migration marker");
|
||||
assert!(!data_movement::is_owned_data_movement_target(&replacement));
|
||||
*other_store.pool_meta.write().await = persisted;
|
||||
set_decommission_capacity_info_overrides_for_test(other_store.id, vec![capacities]);
|
||||
@@ -5502,7 +5374,7 @@ mod decommission_lock_order_tests {
|
||||
)
|
||||
.await
|
||||
.expect("replica conflict recovery must be bounded")
|
||||
.expect("identical existing replica should finish migration on the reloaded node");
|
||||
.expect("identical native replica should finish migration on the reloaded node");
|
||||
let mut reconciled = crate::core::pools::PoolMeta::default();
|
||||
reconciled
|
||||
.load_no_lock_from_replicas(other_store.pools.clone())
|
||||
@@ -5532,14 +5404,14 @@ mod decommission_lock_order_tests {
|
||||
.await
|
||||
.expect_err("the source should be cleaned only after equivalent-target capacity reconciliation");
|
||||
assert!(crate::error::is_err_object_not_found(&missing));
|
||||
let mut target_reader = other_store.pools[target_pool_index]
|
||||
let mut target_reader = other_store.pools[2]
|
||||
.get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the surviving replica should remain readable");
|
||||
assert_eq!(
|
||||
target_reader.object_info.mod_time,
|
||||
Some(target_time),
|
||||
"recovery must not overwrite the existing target"
|
||||
"recovery must not overwrite the native target"
|
||||
);
|
||||
let mut actual = Vec::new();
|
||||
target_reader
|
||||
@@ -5547,20 +5419,6 @@ mod decommission_lock_order_tests {
|
||||
.await
|
||||
.expect("read surviving ledger bytes");
|
||||
assert_eq!(actual, body);
|
||||
if existing_outside_reservation {
|
||||
let (_, outside_body, outside_time) = replicas.last().expect("unreserved existing replica");
|
||||
let mut outside = other_store.pools[2]
|
||||
.get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("migration must leave the unreserved existing replica intact");
|
||||
assert_eq!(outside.object_info.mod_time, Some(*outside_time));
|
||||
let mut actual = Vec::new();
|
||||
outside
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("read the untouched unreserved replica");
|
||||
assert_eq!(&actual, outside_body);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
// #730: data-movement migration keeps staged cleanup helpers until copy paths converge.
|
||||
|
||||
pub(crate) mod backpressure;
|
||||
pub(crate) mod scanner_backlog;
|
||||
|
||||
use crate::core::pools::{DecommissionCapacityOwner, decommission_capacity_mutation_id};
|
||||
use crate::error::{
|
||||
@@ -985,6 +984,24 @@ fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target:
|
||||
.is_some_and(|(source_time, target_time)| target_time > source_time)
|
||||
}
|
||||
|
||||
fn is_equivalent_scanner_backlog_replica(source: &ObjectInfo, target: &ObjectInfo, compare_part_checksums: bool) -> bool {
|
||||
// Scanner publishes this exact payload to surviving sets with CAS. Each
|
||||
// set assigns its own write time; that timestamp is not a ledger generation.
|
||||
// Accept only an identical, known unversioned identity, never a different
|
||||
// record based on timestamp ordering or a similarly named user object.
|
||||
source.bucket == crate::disk::RUSTFS_META_BUCKET
|
||||
&& target.bucket == source.bucket
|
||||
&& source.name == "buckets/.scanner-pause-backlog.json"
|
||||
&& target.name == source.name
|
||||
&& is_unversioned_data_movement_object(source)
|
||||
&& is_unversioned_data_movement_object(target)
|
||||
&& !source.delete_marker
|
||||
&& source.mod_time.is_some()
|
||||
&& target.mod_time.is_some()
|
||||
&& source.etag.as_ref().is_some_and(|etag| !etag.is_empty())
|
||||
&& is_equivalent_data_movement_object_identity(source, target, false, compare_part_checksums)
|
||||
}
|
||||
|
||||
fn is_data_movement_upload_takeover_target(source: &ObjectInfo, target: &ObjectInfo, compare_part_checksums: bool) -> bool {
|
||||
let identity = data_movement_upload_identity(source);
|
||||
source.mod_time.is_some()
|
||||
@@ -1200,7 +1217,7 @@ struct SourceCleanupDeleteBarrierState {
|
||||
dead_code,
|
||||
reason = "installed by set_disk object tests behind `--features test-util` (backlog#1823)"
|
||||
)]
|
||||
pub struct SourceCleanupDeleteBarrier {
|
||||
pub(crate) struct SourceCleanupDeleteBarrier {
|
||||
state: Arc<SourceCleanupDeleteBarrierState>,
|
||||
}
|
||||
|
||||
@@ -1214,7 +1231,7 @@ static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock<std::sync::Mutex<Vec<
|
||||
reason = "installed by set_disk object tests behind `--features test-util` (backlog#1823)"
|
||||
)]
|
||||
impl SourceCleanupDeleteBarrier {
|
||||
pub fn install(bucket: &str, object: &str) -> Self {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(SourceCleanupDeleteBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
@@ -1237,7 +1254,7 @@ impl SourceCleanupDeleteBarrier {
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub async fn wait_until_paused(&self) {
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(StdDuration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("source cleanup should reach the pre-delete barrier");
|
||||
@@ -1253,7 +1270,7 @@ impl SourceCleanupDeleteBarrier {
|
||||
self.state.is_paused.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub fn release(&self) {
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
@@ -1432,8 +1449,7 @@ fn resolve_data_movement_overwrite_resume_result_for(
|
||||
target_pool_idx: usize,
|
||||
compare_part_checksums: bool,
|
||||
) -> Result<bool> {
|
||||
if scanner_backlog::is_scanner_pause_backlog(&source.bucket, &source.name)
|
||||
|| !should_check_data_movement_overwrite_resume(err)
|
||||
if !should_check_data_movement_overwrite_resume(err)
|
||||
|| !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx)
|
||||
{
|
||||
return Ok(false);
|
||||
@@ -1455,7 +1471,9 @@ fn resolve_data_movement_overwrite_resume_result_for(
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
Ok(matches!(err, Error::PreconditionFailed) && is_superseding_unversioned_data_movement_object(source, &target))
|
||||
Ok(matches!(err, Error::PreconditionFailed)
|
||||
&& (is_equivalent_scanner_backlog_replica(source, &target, compare_part_checksums)
|
||||
|| is_superseding_unversioned_data_movement_object(source, &target)))
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -1503,27 +1521,9 @@ fn data_movement_part_stage_error(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
part_number: usize,
|
||||
err: Error,
|
||||
err: impl std::fmt::Display,
|
||||
) -> Error {
|
||||
let rendered = format!("{op_label}: {stage} failed for {bucket}/{object} part {part_number}: {err}");
|
||||
if matches!(&err, Error::DecommissionCapacityBlocked { .. }) {
|
||||
return data_movement_context_error(rendered, err);
|
||||
}
|
||||
// A missing target part is not evidence that the source can be deleted.
|
||||
// Keep other part errors opaque to the source-cleanup classifiers.
|
||||
Error::other(rendered)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn data_movement_part_stage_error_for_test(
|
||||
op_label: &str,
|
||||
stage: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
part_number: usize,
|
||||
err: Error,
|
||||
) -> Error {
|
||||
data_movement_part_stage_error(op_label, stage, bucket, object, part_number, err)
|
||||
Error::other(format!("{op_label}: {stage} failed for {bucket}/{object} part {part_number}: {err}"))
|
||||
}
|
||||
|
||||
fn is_data_movement_part_read_error(err: &Error) -> bool {
|
||||
@@ -1628,9 +1628,6 @@ async fn migrate_object_inner(
|
||||
capacity_owner: Option<DecommissionCapacityOwner>,
|
||||
mutation_fence: Option<DecommissionFixedReadAnchor>,
|
||||
) -> Result<()> {
|
||||
if scanner_backlog::is_scanner_pause_backlog(&bucket, &rd.object_info.name) {
|
||||
return Err(Error::other("scanner pause backlog requires native retirement handoff"));
|
||||
}
|
||||
let mut mutation_fence = mutation_fence;
|
||||
let object_info = rd.object_info.clone();
|
||||
let capacity_owner = capacity_owner.map(|owner| {
|
||||
@@ -2431,15 +2428,8 @@ mod tests {
|
||||
let err =
|
||||
data_movement_part_stage_error("rebalance_object", "put_object_part", "bucket-a", "object-a", 7, Error::SlowDown);
|
||||
let message = err.to_string();
|
||||
assert_eq!(
|
||||
message,
|
||||
Error::other(format!(
|
||||
"rebalance_object: put_object_part failed for bucket-a/object-a part 7: {}",
|
||||
Error::SlowDown
|
||||
))
|
||||
.to_string()
|
||||
);
|
||||
assert!(data_movement_stage_source(&err).is_none());
|
||||
assert!(message.contains("rebalance_object: put_object_part failed for bucket-a/object-a part 7"));
|
||||
assert!(message.contains(Error::SlowDown.to_string().as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3339,25 +3329,16 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scanner_backlog_resume_requires_native_cohort_proof_even_for_identical_payload() {
|
||||
fn test_scanner_backlog_resume_accepts_identical_native_replica_with_older_write_time() {
|
||||
let (source, target) = scanner_backlog_replica_pair();
|
||||
assert!(!is_owned_data_movement_target(&target), "native scanner writes are not migration copies");
|
||||
assert!(!is_equivalent_data_movement_object(&source, &target));
|
||||
assert!(
|
||||
!scanner_backlog_precondition_resumes(&source, target),
|
||||
"a single identical replica cannot prove native cohort authority"
|
||||
scanner_backlog_precondition_resumes(&source, target),
|
||||
"identical ledger payloads have replica-local write times, not distinct committed generations"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scanner_backlog_resume_rejects_newer_timestamp_and_full_single_replica_identity() {
|
||||
let (source, mut target) = scanner_backlog_replica_pair();
|
||||
target.mod_time = source.mod_time.map(|time| time + time::Duration::SECOND);
|
||||
target.etag = Some("different-native-ledger".to_string());
|
||||
assert!(!scanner_backlog_precondition_resumes(&source, target));
|
||||
assert!(!scanner_backlog_precondition_resumes(&source, source.clone()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scanner_backlog_resume_rejects_changed_payload_or_metadata() {
|
||||
let (source, target) = scanner_backlog_replica_pair();
|
||||
|
||||
@@ -1,292 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_found};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::object_api::{ObjectInfo, PutObjReader, WriteCompletion};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::storage_api_contracts::object::HTTPPreconditions;
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use futures::future::join_all;
|
||||
use http::HeaderMap;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
pub const MAX_SCANNER_PAUSE_BACKLOG_BYTES: u64 = 64 * 1024;
|
||||
pub(crate) const SCANNER_PAUSE_BACKLOG_PATH: &str = "buckets/.scanner-pause-backlog.json";
|
||||
|
||||
/// A bounded, storage-fenced native replica. Only a confirmed missing object
|
||||
/// has no payload; read failures never enter the Scanner verifier.
|
||||
pub struct ScannerPauseBacklogRetirementReplica {
|
||||
pub pool_index: usize,
|
||||
pub set_index: usize,
|
||||
pub data: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
/// Native records for a membership handoff. Existing durable ledgers are
|
||||
/// preserved; an empty native bootstrap may initialize its first ledger.
|
||||
pub struct ScannerPauseBacklogRetirementPlan {
|
||||
pub seed_record: Option<Vec<u8>>,
|
||||
pub commit_record: Vec<u8>,
|
||||
pub stable_record: Vec<u8>,
|
||||
}
|
||||
|
||||
pub type ScannerPauseBacklogRetirementPlanner =
|
||||
fn(usize, &[ScannerPauseBacklogRetirementReplica]) -> std::result::Result<Option<ScannerPauseBacklogRetirementPlan>, String>;
|
||||
|
||||
static RETIREMENT_PLANNER: OnceLock<ScannerPauseBacklogRetirementPlanner> = OnceLock::new();
|
||||
|
||||
/// Install the stateless native record planner before storage starts workers.
|
||||
/// The scanner runtime switch does not control this storage safety check.
|
||||
pub fn register_scanner_pause_backlog_retirement_planner(planner: ScannerPauseBacklogRetirementPlanner) {
|
||||
RETIREMENT_PLANNER.get_or_init(|| planner);
|
||||
}
|
||||
|
||||
pub(crate) fn is_scanner_pause_backlog(bucket: &str, object: &str) -> bool {
|
||||
bucket == RUSTFS_META_BUCKET && object == SCANNER_PAUSE_BACKLOG_PATH
|
||||
}
|
||||
|
||||
pub(crate) struct ScannerPauseBacklogRetirementRead {
|
||||
pub replica: ScannerPauseBacklogRetirementReplica,
|
||||
pub etag: Option<String>,
|
||||
}
|
||||
|
||||
impl ScannerPauseBacklogRetirementRead {
|
||||
pub(crate) fn preconditions(&self) -> HTTPPreconditions {
|
||||
match &self.etag {
|
||||
Some(etag) => HTTPPreconditions {
|
||||
if_match: Some(etag.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
None => HTTPPreconditions {
|
||||
if_none_match: Some("*".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_replica(set: Arc<SetDisks>) -> Result<ScannerPauseBacklogRetirementRead> {
|
||||
let mut replica = ScannerPauseBacklogRetirementReplica {
|
||||
pool_index: set.pool_index,
|
||||
set_index: set.set_index,
|
||||
data: None,
|
||||
};
|
||||
let reader = match set
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
SCANNER_PAUSE_BACKLOG_PATH,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(reader) => reader,
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
|
||||
return Ok(ScannerPauseBacklogRetirementRead { replica, etag: None });
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let info = &reader.object_info;
|
||||
if info.version_id.is_some_and(|version| !version.is_nil())
|
||||
|| info.delete_marker
|
||||
|| info.is_dir
|
||||
|| info.etag.as_ref().is_none_or(String::is_empty)
|
||||
|| info.size < 0
|
||||
|| info.size > MAX_SCANNER_PAUSE_BACKLOG_BYTES as i64
|
||||
{
|
||||
return Err(Error::other("scanner pause backlog retirement found an unsupported replica identity"));
|
||||
}
|
||||
let etag = info.etag.clone();
|
||||
let expected_size = info.size as usize;
|
||||
let mut data = Vec::new();
|
||||
reader
|
||||
.take(MAX_SCANNER_PAUSE_BACKLOG_BYTES + 1)
|
||||
.read_to_end(&mut data)
|
||||
.await?;
|
||||
if data.len() != expected_size || data.len() > MAX_SCANNER_PAUSE_BACKLOG_BYTES as usize {
|
||||
return Err(Error::other("scanner pause backlog retirement replica has an invalid payload length"));
|
||||
}
|
||||
replica.data = Some(data);
|
||||
Ok(ScannerPauseBacklogRetirementRead { replica, etag })
|
||||
}
|
||||
|
||||
/// The caller retains the fixed object write lock and durable topology read
|
||||
/// fence through both this snapshot and physical source cleanup.
|
||||
pub(crate) async fn read_scanner_pause_backlog_retirement_replicas(
|
||||
source_pool_index: usize,
|
||||
source_set_index: usize,
|
||||
sets: Vec<Arc<SetDisks>>,
|
||||
) -> Result<Vec<ScannerPauseBacklogRetirementRead>> {
|
||||
let replicas = join_all(sets.into_iter().map(read_replica))
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
if !replicas.iter().any(|read| {
|
||||
read.replica.pool_index == source_pool_index && read.replica.set_index == source_set_index && read.replica.data.is_some()
|
||||
}) {
|
||||
return Err(Error::other("scanner pause backlog retirement current source replica is missing"));
|
||||
}
|
||||
Ok(replicas)
|
||||
}
|
||||
|
||||
pub(crate) fn plan_scanner_pause_backlog_retirement(
|
||||
source_pool_index: usize,
|
||||
replicas: &[ScannerPauseBacklogRetirementRead],
|
||||
) -> Result<Option<ScannerPauseBacklogRetirementPlan>> {
|
||||
let planner = RETIREMENT_PLANNER
|
||||
.get()
|
||||
.ok_or_else(|| Error::other("scanner pause backlog native retirement planner is unavailable"))?;
|
||||
let snapshots = replicas
|
||||
.iter()
|
||||
.map(|read| ScannerPauseBacklogRetirementReplica {
|
||||
pool_index: read.replica.pool_index,
|
||||
set_index: read.replica.set_index,
|
||||
data: read.replica.data.clone(),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
planner(source_pool_index, &snapshots).map_err(Error::other)
|
||||
}
|
||||
|
||||
/// The native writer and retirement handoff use the same conditional, full-tail
|
||||
/// write. Their callers retain object and durable membership fences until return.
|
||||
pub(crate) async fn persist_native_scanner_pause_backlog_replica(
|
||||
set: Arc<SetDisks>,
|
||||
data: Vec<u8>,
|
||||
preconditions: HTTPPreconditions,
|
||||
mut opts: ObjectOptions,
|
||||
_phase: &'static str,
|
||||
) -> Result<ObjectInfo> {
|
||||
if data.len() > MAX_SCANNER_PAUSE_BACKLOG_BYTES as usize {
|
||||
return Err(Error::other("scanner pause backlog exceeds its size bound"));
|
||||
}
|
||||
opts.max_parity = true;
|
||||
opts.write_completion = WriteCompletion::TailDrained;
|
||||
opts.http_preconditions = Some(preconditions);
|
||||
#[cfg(feature = "test-util")]
|
||||
let fault = test_util::matching_write(&set, _phase)?;
|
||||
let result = set
|
||||
.put_object(RUSTFS_META_BUCKET, SCANNER_PAUSE_BACKLOG_PATH, &mut PutObjReader::from_vec(data), &opts)
|
||||
.await;
|
||||
#[cfg(feature = "test-util")]
|
||||
if result.is_ok()
|
||||
&& let Some(fault) = fault
|
||||
{
|
||||
fault.arrived.notify_one();
|
||||
fault.release.notified().await;
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use tokio::sync::Notify;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("injected native scanner backlog {phase} write failure")]
|
||||
struct InjectedWriteFailure {
|
||||
phase: &'static str,
|
||||
}
|
||||
|
||||
pub(super) struct WriteFault {
|
||||
set: Arc<SetDisks>,
|
||||
phase: &'static str,
|
||||
remaining: AtomicUsize,
|
||||
fail_before_write: bool,
|
||||
pub(super) arrived: Notify,
|
||||
pub(super) release: Notify,
|
||||
}
|
||||
|
||||
static WRITE_FAULTS: Mutex<Vec<Arc<WriteFault>>> = Mutex::new(Vec::new());
|
||||
|
||||
/// Scope a one-shot fault to the actual set instance, so other stores and
|
||||
/// concurrent tests keep using the ordinary native persistence path.
|
||||
pub struct NativeScannerPauseBacklogWriteFault {
|
||||
state: Arc<WriteFault>,
|
||||
}
|
||||
|
||||
impl NativeScannerPauseBacklogWriteFault {
|
||||
fn install(set: Arc<SetDisks>, phase: &'static str, nth: usize, fail_before_write: bool) -> Self {
|
||||
assert!(nth > 0);
|
||||
let state = Arc::new(WriteFault {
|
||||
set,
|
||||
phase,
|
||||
remaining: AtomicUsize::new(nth),
|
||||
fail_before_write,
|
||||
arrived: Notify::new(),
|
||||
release: Notify::new(),
|
||||
});
|
||||
let mut faults = WRITE_FAULTS.lock().unwrap();
|
||||
assert!(
|
||||
!faults
|
||||
.iter()
|
||||
.any(|fault| Arc::ptr_eq(&fault.set, &state.set) && fault.phase == phase)
|
||||
);
|
||||
faults.push(Arc::clone(&state));
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub fn fail_before_write(set: Arc<SetDisks>, phase: &'static str, nth: usize) -> Self {
|
||||
Self::install(set, phase, nth, true)
|
||||
}
|
||||
|
||||
pub fn pause_after_write(set: Arc<SetDisks>, phase: &'static str) -> Self {
|
||||
Self::install(set, phase, 1, false)
|
||||
}
|
||||
|
||||
pub async fn wait_until_paused(&self) {
|
||||
self.state.arrived.notified().await;
|
||||
}
|
||||
|
||||
pub fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for NativeScannerPauseBacklogWriteFault {
|
||||
fn drop(&mut self) {
|
||||
self.release();
|
||||
WRITE_FAULTS.lock().unwrap().retain(|fault| !Arc::ptr_eq(fault, &self.state));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn matching_write(set: &Arc<SetDisks>, phase: &'static str) -> Result<Option<Arc<WriteFault>>> {
|
||||
let fault = WRITE_FAULTS
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|fault| Arc::ptr_eq(&fault.set, set) && fault.phase == phase)
|
||||
.cloned();
|
||||
let Some(fault) = fault else { return Ok(None) };
|
||||
if fault
|
||||
.remaining
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1))
|
||||
!= Ok(1)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
if fault.fail_before_write {
|
||||
return Err(Error::other(InjectedWriteFailure { phase }));
|
||||
}
|
||||
Ok(Some(fault))
|
||||
}
|
||||
}
|
||||
@@ -3397,7 +3397,6 @@ mod tests {
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: false,
|
||||
tombstone: false,
|
||||
segment_invalidation_proof: None,
|
||||
}];
|
||||
partial.buckets_usage.insert(
|
||||
"bucket".to_string(),
|
||||
@@ -3470,7 +3469,6 @@ mod tests {
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: true,
|
||||
tombstone: false,
|
||||
segment_invalidation_proof: None,
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -627,13 +627,7 @@ impl From<tokio::task::JoinError> for DiskError {
|
||||
impl Clone for DiskError {
|
||||
fn clone(&self) -> Self {
|
||||
match self {
|
||||
DiskError::Io(io_error) => DiskError::Io(
|
||||
rustfs_rio::clone_internode_http_io_error(io_error)
|
||||
.and_then(std::io::Error::into_inner)
|
||||
// The helper derives a kind from the source; Clone must retain the original outer kind.
|
||||
.map(|source| std::io::Error::new(io_error.kind(), source))
|
||||
.unwrap_or_else(|| std::io::Error::new(io_error.kind(), io_error.to_string())),
|
||||
),
|
||||
DiskError::Io(io_error) => DiskError::Io(std::io::Error::new(io_error.kind(), io_error.to_string())),
|
||||
DiskError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
|
||||
DiskError::Unexpected => DiskError::Unexpected,
|
||||
DiskError::CorruptedFormat => DiskError::CorruptedFormat,
|
||||
@@ -1271,49 +1265,6 @@ mod tests {
|
||||
assert!(!bad_request.is_retryable_internode_write_failure());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_internode_http_clone_preserves_retryability_status_and_context() {
|
||||
use http::StatusCode;
|
||||
use rustfs_rio::InternodeHttpErrorKind::{ConnectionRefused, ConnectionReset, HttpStatus, Unknown};
|
||||
|
||||
for (kind, retryable) in [
|
||||
(ConnectionRefused, true),
|
||||
(ConnectionReset, true),
|
||||
(HttpStatus(StatusCode::TOO_MANY_REQUESTS), true),
|
||||
(HttpStatus(StatusCode::SERVICE_UNAVAILABLE), true),
|
||||
(HttpStatus(StatusCode::CONFLICT), true),
|
||||
(Unknown, false),
|
||||
(HttpStatus(StatusCode::BAD_REQUEST), false),
|
||||
(HttpStatus(StatusCode::INTERNAL_SERVER_ERROR), false),
|
||||
] {
|
||||
let original = DiskError::from(rustfs_rio::new_test_internode_http_io_error(kind));
|
||||
assert_eq!(original.internode_http_error_kind(), Some(kind));
|
||||
assert_eq!(original.is_retryable_internode_write_failure(), retryable);
|
||||
|
||||
let cloned = original.clone();
|
||||
assert_eq!(cloned, original, "clone must preserve the error bucket for {kind:?}");
|
||||
assert_eq!(
|
||||
cloned.is_retryable_internode_write_failure(),
|
||||
retryable,
|
||||
"clone changed retryability for {kind:?}"
|
||||
);
|
||||
assert_eq!(cloned.internode_http_error_kind(), Some(kind));
|
||||
if let HttpStatus(status) = kind {
|
||||
assert!(cloned.is_internode_http_status(status.as_u16()));
|
||||
}
|
||||
let DiskError::Io(io_error) = &cloned else {
|
||||
panic!("unmarked internode error must remain Io: {cloned:?}");
|
||||
};
|
||||
let source = io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
|
||||
.expect("clone must retain the structured internode error");
|
||||
assert_eq!(source.context().method(), "PUT");
|
||||
assert_eq!(source.context().target(), "/rustfs/rpc/put_file_stream");
|
||||
assert_eq!(source.context().operation(), Some(INTERNODE_OPERATION_PUT_FILE_STREAM));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_stream_conflict_is_not_a_retryable_put_file_failure() {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
@@ -1358,57 +1309,11 @@ mod tests {
|
||||
!error.is_retryable_internode_write_failure(),
|
||||
"read-operation 409 must not trigger put-file retry"
|
||||
);
|
||||
let cloned = error.clone();
|
||||
let reduced = crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(error)], &[], 1)
|
||||
.expect("the read conflict must remain the dominant error");
|
||||
for preserved in [&cloned, &reduced] {
|
||||
assert!(
|
||||
!preserved.is_retryable_internode_write_failure(),
|
||||
"cloning or reducing a read conflict must not turn it into a PUT retry"
|
||||
);
|
||||
assert!(preserved.is_internode_http_status(409));
|
||||
let DiskError::Io(io_error) = preserved else {
|
||||
panic!("read conflict must remain Io: {preserved:?}");
|
||||
};
|
||||
let source = io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
|
||||
.expect("read conflict must retain its request context");
|
||||
assert_eq!(source.context().method(), "GET");
|
||||
assert_eq!(source.context().target(), "/rustfs/rpc/read_file_stream");
|
||||
assert_eq!(
|
||||
source.context().operation(),
|
||||
Some(rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_READ_FILE_STREAM)
|
||||
);
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("isolated read-conflict test must finish within its budget");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_internode_http_clone_preserves_outer_io_kind_and_message() {
|
||||
let source = rustfs_rio::new_test_internode_http_io_error(InternodeHttpErrorKind::ConnectionReset)
|
||||
.into_inner()
|
||||
.expect("the internode helper must provide a typed source");
|
||||
let original_io = io::Error::new(io::ErrorKind::InvalidData, source);
|
||||
let message = original_io.to_string();
|
||||
let original = DiskError::from(original_io);
|
||||
assert_eq!(original.internode_http_error_kind(), Some(InternodeHttpErrorKind::ConnectionReset));
|
||||
assert!(original.is_retryable_internode_write_failure());
|
||||
|
||||
let cloned = original.clone();
|
||||
let reduced = crate::disk::error_reduce::reduce_write_quorum_errs(&[Some(original)], &[], 1)
|
||||
.expect("the wrapped internode error must remain the dominant error");
|
||||
for preserved in [&cloned, &reduced] {
|
||||
let DiskError::Io(io_error) = preserved else {
|
||||
panic!("the wrapped error must remain Io: {preserved:?}");
|
||||
};
|
||||
assert_eq!(io_error.kind(), io::ErrorKind::InvalidData);
|
||||
assert_eq!(io_error.to_string(), message);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_internode_missing_errors_preserve_disk_error_types() {
|
||||
let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error());
|
||||
@@ -1420,17 +1325,6 @@ mod tests {
|
||||
assert_eq!(file_missing, DiskError::FileNotFound);
|
||||
assert_eq!(volume_missing, DiskError::VolumeNotFound);
|
||||
assert!(matches!(unmarked_server_error, DiskError::Io(_)));
|
||||
for missing in [file_missing, volume_missing] {
|
||||
assert_eq!(missing.clone(), missing);
|
||||
assert_eq!(
|
||||
crate::disk::error_reduce::reduce_write_quorum_errs(
|
||||
&[Some(missing.clone()), Some(missing.clone()), None],
|
||||
&[],
|
||||
2
|
||||
),
|
||||
Some(missing)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -226,78 +226,6 @@ mod tests {
|
||||
assert_eq!(res, Some(quorum_err));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_quorum_reduction_preserves_internode_http_identity() {
|
||||
use http::StatusCode;
|
||||
use rustfs_rio::InternodeHttpErrorKind::{ConnectionRefused, HttpStatus, Unknown};
|
||||
|
||||
for (kind, retryable) in [
|
||||
(ConnectionRefused, true),
|
||||
(HttpStatus(StatusCode::SERVICE_UNAVAILABLE), true),
|
||||
(HttpStatus(StatusCode::CONFLICT), true),
|
||||
(Unknown, false),
|
||||
(HttpStatus(StatusCode::BAD_REQUEST), false),
|
||||
] {
|
||||
// Construct both producer errors independently: the reducer owns the first clone.
|
||||
let first = Error::from(rustfs_rio::new_test_internode_http_io_error(kind));
|
||||
let second = Error::from(rustfs_rio::new_test_internode_http_io_error(kind));
|
||||
assert_eq!(first.internode_http_error_kind(), Some(kind));
|
||||
assert_eq!(second.internode_http_error_kind(), Some(kind));
|
||||
assert_eq!(first.is_retryable_internode_write_failure(), retryable);
|
||||
let errors = [Some(first), Some(second), None];
|
||||
let reduced = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, 2)
|
||||
.expect("two equal producer errors must dominate one successful write");
|
||||
|
||||
assert_eq!(Some(&reduced), errors[0].as_ref());
|
||||
assert_eq!(
|
||||
reduced.is_retryable_internode_write_failure(),
|
||||
retryable,
|
||||
"quorum reduction changed retryability for {kind:?}"
|
||||
);
|
||||
assert_eq!(reduced.internode_http_error_kind(), Some(kind));
|
||||
if let HttpStatus(status) = kind {
|
||||
assert!(reduced.is_internode_http_status(status.as_u16()));
|
||||
}
|
||||
let Error::Io(io_error) = &reduced else {
|
||||
panic!("the dominant error must remain Io: {reduced:?}");
|
||||
};
|
||||
let source = io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<rustfs_rio::InternodeHttpError>())
|
||||
.expect("quorum reduction must retain the structured internode error");
|
||||
assert_eq!(source.context().method(), "PUT");
|
||||
assert_eq!(source.context().target(), "/rustfs/rpc/put_file_stream");
|
||||
assert_eq!(
|
||||
source.context().operation(),
|
||||
Some(rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clone_and_write_quorum_do_not_promote_non_retryable_errors() {
|
||||
use http::StatusCode;
|
||||
use rustfs_rio::InternodeHttpErrorKind::{HttpStatus, Unknown};
|
||||
|
||||
for original in [
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(Unknown)),
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::BAD_REQUEST))),
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::FORBIDDEN))),
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(StatusCode::NOT_FOUND))),
|
||||
Error::from(rustfs_rio::new_test_internode_http_io_error(HttpStatus(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
))),
|
||||
err_io("internode connection reset: PUT /rustfs/rpc/put_file_stream"),
|
||||
] {
|
||||
assert!(!original.is_retryable_internode_write_failure());
|
||||
let cloned = original.clone();
|
||||
let reduced =
|
||||
reduce_write_quorum_errs(&[Some(original)], &[], 1).expect("a non-retryable error must remain an error");
|
||||
assert!(!cloned.is_retryable_internode_write_failure());
|
||||
assert!(!reduced.is_retryable_internode_write_failure());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_count_errs() {
|
||||
let e1 = err_io("a");
|
||||
|
||||
@@ -219,7 +219,6 @@ fn restore_part_transaction_file(current: &Path, backup: &Path, absent: &Path, r
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn write_metadata_rollback_backup(object_dir: &Path, rollback_dir: Uuid, data: &[u8]) -> Result<()> {
|
||||
write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, data, None).await
|
||||
}
|
||||
@@ -251,7 +250,6 @@ async fn write_delete_rollback_file(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn restore_metadata_backup(
|
||||
object_dir: &Path,
|
||||
xl_path: &Path,
|
||||
@@ -279,6 +277,15 @@ async fn restore_metadata_backup_with_namespace_owner(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn restore_delete_rollback(
|
||||
object_dir: &Path,
|
||||
xl_path: &Path,
|
||||
rollback_dir: Uuid,
|
||||
publication_root: &os::PublicationRoot,
|
||||
) -> Result<()> {
|
||||
restore_delete_rollback_with_namespace_owner(object_dir, xl_path, rollback_dir, publication_root, None).await
|
||||
}
|
||||
|
||||
async fn restore_delete_rollback_with_namespace_owner(
|
||||
object_dir: &Path,
|
||||
xl_path: &Path,
|
||||
@@ -5850,8 +5857,6 @@ impl LocalDisk {
|
||||
check_path_length(file_path.to_string_lossy().as_ref())?;
|
||||
|
||||
let xl_path = path_join(&[file_path.as_path(), Path::new(STORAGE_FORMAT_FILE)]);
|
||||
let namespace_owner: Option<Arc<dyn Send + Sync>> =
|
||||
Some(os::acquire_metadata_mutation_lease(&self.get_object_path(volume, path)?, namespace_owner).await);
|
||||
if opts.old_data_dir.is_some() && opts.undo_write {
|
||||
return self.undo_write(file_path.as_path(), &fi, &opts, namespace_owner).await;
|
||||
}
|
||||
@@ -6709,8 +6714,6 @@ impl LocalDisk {
|
||||
|
||||
async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> {
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
let object_path = self.get_object_path(volume, path)?;
|
||||
let namespace_owner: Option<Arc<dyn Send + Sync>> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await);
|
||||
let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?;
|
||||
let object_dir = xlpath
|
||||
.parent()
|
||||
@@ -6720,24 +6723,10 @@ impl LocalDisk {
|
||||
&& opts.undo_write
|
||||
{
|
||||
if opts.undo_delete {
|
||||
return restore_delete_rollback_with_namespace_owner(
|
||||
object_dir,
|
||||
&xlpath,
|
||||
rollback_dir,
|
||||
&self.publication_root,
|
||||
namespace_owner.clone(),
|
||||
)
|
||||
.await;
|
||||
return restore_delete_rollback(object_dir, &xlpath, rollback_dir, &self.publication_root).await;
|
||||
}
|
||||
|
||||
return restore_metadata_backup_with_namespace_owner(
|
||||
object_dir,
|
||||
&xlpath,
|
||||
rollback_dir,
|
||||
&self.publication_root,
|
||||
namespace_owner.clone(),
|
||||
)
|
||||
.await;
|
||||
return restore_metadata_backup(object_dir, &xlpath, rollback_dir, &self.publication_root).await;
|
||||
}
|
||||
|
||||
let (data, _) = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await {
|
||||
@@ -6749,14 +6738,7 @@ impl LocalDisk {
|
||||
return Err(DiskError::FileNotFound);
|
||||
};
|
||||
return self
|
||||
.write_missing_delete_marker(
|
||||
volume,
|
||||
path,
|
||||
delete_marker,
|
||||
object_dir,
|
||||
opts.old_data_dir,
|
||||
namespace_owner.clone(),
|
||||
)
|
||||
.write_missing_delete_marker(volume, path, delete_marker, object_dir, opts.old_data_dir, None)
|
||||
.await;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
@@ -6772,8 +6754,7 @@ impl LocalDisk {
|
||||
let rollback_dir = opts.old_data_dir;
|
||||
let mut reserved_version_delete = false;
|
||||
if let Some(rollback_dir) = rollback_dir {
|
||||
write_delete_rollback_file(object_dir, rollback_dir, STORAGE_FORMAT_FILE_BACKUP, &data, namespace_owner.clone())
|
||||
.await?;
|
||||
write_metadata_rollback_backup(object_dir, rollback_dir, &data).await?;
|
||||
}
|
||||
|
||||
for fi in fis.iter() {
|
||||
@@ -6787,16 +6768,13 @@ impl LocalDisk {
|
||||
|
||||
if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
|
||||
return Err(self
|
||||
.abort_reserved_version_delete_with_failure(
|
||||
.abort_reserved_version_delete(
|
||||
object_dir,
|
||||
rollback_dir,
|
||||
volume,
|
||||
path,
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_metadata_update",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
"delete_versions_metadata_update",
|
||||
err,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
@@ -6809,7 +6787,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_metadata_update",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -6826,16 +6804,13 @@ impl LocalDisk {
|
||||
Err(err) => {
|
||||
if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
|
||||
return Err(self
|
||||
.abort_reserved_version_delete_with_failure(
|
||||
.abort_reserved_version_delete(
|
||||
object_dir,
|
||||
rollback_dir,
|
||||
volume,
|
||||
path,
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_data_path",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
"delete_versions_data_path",
|
||||
err,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
@@ -6848,7 +6823,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_data_path",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -6861,16 +6836,13 @@ impl LocalDisk {
|
||||
let err: DiskError = to_file_error(err).into();
|
||||
if reserved_version_delete {
|
||||
return Err(self
|
||||
.abort_reserved_version_delete_with_failure(
|
||||
.abort_reserved_version_delete(
|
||||
object_dir,
|
||||
rollback_dir,
|
||||
volume,
|
||||
path,
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_rollback_dir",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
"delete_versions_rollback_dir",
|
||||
err,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
@@ -6883,29 +6855,23 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_rollback_dir",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
let reserved = match self
|
||||
.reserve_version_delete_with_namespace_owner(volume, path, dir, rollback_dir, namespace_owner.clone())
|
||||
.await
|
||||
{
|
||||
let reserved = match self.reserve_version_delete(volume, path, dir, rollback_dir).await {
|
||||
Ok(reserved) => reserved,
|
||||
Err(err) => {
|
||||
return Err(self
|
||||
.abort_reserved_version_delete_with_failure(
|
||||
.abort_reserved_version_delete(
|
||||
object_dir,
|
||||
rollback_dir,
|
||||
volume,
|
||||
path,
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_reserve_data",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
"delete_versions_reserve_data",
|
||||
err,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
@@ -6913,12 +6879,11 @@ impl LocalDisk {
|
||||
reserved_version_delete |= reserved;
|
||||
let rollback_data_path = rollback_path.join(dir.to_string());
|
||||
if !reserved
|
||||
&& let Err(err) = os::rename_all_ignore_missing_source_with_owner(
|
||||
&& let Err(err) = rename_all_ignore_missing_source(
|
||||
&dir_path,
|
||||
&rollback_data_path,
|
||||
&rollback_path,
|
||||
&self.publication_root,
|
||||
namespace_owner.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -6931,7 +6896,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_stage_data",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -6940,16 +6905,13 @@ impl LocalDisk {
|
||||
if should_fail_after_delete_data_staged(path) {
|
||||
if reserved_version_delete {
|
||||
return Err(self
|
||||
.abort_reserved_version_delete_with_failure(
|
||||
.abort_reserved_version_delete(
|
||||
object_dir,
|
||||
rollback_dir,
|
||||
volume,
|
||||
path,
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_test_after_stage",
|
||||
error: DiskError::Unexpected,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
"delete_versions_test_after_stage",
|
||||
DiskError::Unexpected,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
@@ -6962,15 +6924,13 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_test_after_stage",
|
||||
error: DiskError::Unexpected,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
} else if let Err(err) = self
|
||||
.move_to_trash_with_namespace_owner(&dir_path, true, false, namespace_owner.clone())
|
||||
.await
|
||||
} else if let Err(err) = self.move_to_trash(&dir_path, true, false).await
|
||||
&& !(err == DiskError::FileNotFound || err == DiskError::VolumeNotFound)
|
||||
{
|
||||
return Err(err);
|
||||
@@ -6985,22 +6945,16 @@ impl LocalDisk {
|
||||
|
||||
// Remove xl.meta when no versions remain
|
||||
if fm.versions.is_empty() {
|
||||
if let Err(err) = self
|
||||
.delete_file_with_namespace_owner(&volume_dir, &xlpath, true, false, namespace_owner.clone())
|
||||
.await
|
||||
{
|
||||
if let Err(err) = self.delete_file(&volume_dir, &xlpath, true, false).await {
|
||||
if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
|
||||
return Err(self
|
||||
.abort_reserved_version_delete_with_failure(
|
||||
.abort_reserved_version_delete(
|
||||
object_dir,
|
||||
rollback_dir,
|
||||
volume,
|
||||
path,
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_commit_delete",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
"delete_versions_commit_delete",
|
||||
err,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
@@ -7013,7 +6967,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_commit_delete",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -7021,22 +6975,10 @@ impl LocalDisk {
|
||||
}
|
||||
if reserved_version_delete
|
||||
&& let Some(rollback_dir) = rollback_dir
|
||||
&& let Err(err) = self
|
||||
.commit_reserved_version_delete_with_namespace_owner(volume, path, rollback_dir, namespace_owner.clone())
|
||||
.await
|
||||
&& let Err(err) = self.commit_reserved_version_delete(volume, path, rollback_dir).await
|
||||
{
|
||||
return Err(self
|
||||
.abort_reserved_version_delete_with_failure(
|
||||
object_dir,
|
||||
rollback_dir,
|
||||
volume,
|
||||
path,
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_commit_intent",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
)
|
||||
.abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_intent", err)
|
||||
.await);
|
||||
}
|
||||
if should_fail_after_delete_commit(self.root.as_path(), path) {
|
||||
@@ -7053,16 +6995,13 @@ impl LocalDisk {
|
||||
let err: DiskError = err.into();
|
||||
if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
|
||||
return Err(self
|
||||
.abort_reserved_version_delete_with_failure(
|
||||
.abort_reserved_version_delete(
|
||||
object_dir,
|
||||
rollback_dir,
|
||||
volume,
|
||||
path,
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_metadata_encode",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
"delete_versions_metadata_encode",
|
||||
err,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
@@ -7075,7 +7014,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_metadata_encode",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -7084,28 +7023,12 @@ impl LocalDisk {
|
||||
};
|
||||
|
||||
if let Err(err) = self
|
||||
.write_all_meta_with_namespace_owner(
|
||||
volume,
|
||||
format!("{path}/{STORAGE_FORMAT_FILE}").as_str(),
|
||||
&buf,
|
||||
true,
|
||||
namespace_owner.clone(),
|
||||
)
|
||||
.write_all_meta(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), &buf, true)
|
||||
.await
|
||||
{
|
||||
if reserved_version_delete && let Some(rollback_dir) = rollback_dir {
|
||||
return Err(self
|
||||
.abort_reserved_version_delete_with_failure(
|
||||
object_dir,
|
||||
rollback_dir,
|
||||
volume,
|
||||
path,
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_commit_write",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
)
|
||||
.abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_write", err)
|
||||
.await);
|
||||
}
|
||||
return Err(restore_delete_rollback_after_error(
|
||||
@@ -7117,7 +7040,7 @@ impl LocalDisk {
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_commit_write",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
namespace_owner: None,
|
||||
},
|
||||
&self.publication_root,
|
||||
)
|
||||
@@ -7126,22 +7049,10 @@ impl LocalDisk {
|
||||
|
||||
if reserved_version_delete
|
||||
&& let Some(rollback_dir) = rollback_dir
|
||||
&& let Err(err) = self
|
||||
.commit_reserved_version_delete_with_namespace_owner(volume, path, rollback_dir, namespace_owner.clone())
|
||||
.await
|
||||
&& let Err(err) = self.commit_reserved_version_delete(volume, path, rollback_dir).await
|
||||
{
|
||||
return Err(self
|
||||
.abort_reserved_version_delete_with_failure(
|
||||
object_dir,
|
||||
rollback_dir,
|
||||
volume,
|
||||
path,
|
||||
DeleteRollbackFailure {
|
||||
stage: "delete_versions_commit_intent",
|
||||
error: err,
|
||||
namespace_owner: namespace_owner.clone(),
|
||||
},
|
||||
)
|
||||
.abort_reserved_version_delete(object_dir, rollback_dir, volume, path, "delete_versions_commit_intent", err)
|
||||
.await);
|
||||
}
|
||||
|
||||
@@ -7152,107 +7063,6 @@ impl LocalDisk {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn reconcile_transition_state_metadata(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
version_id: Option<Uuid>,
|
||||
condition: &super::TransitionStateReconcileCondition,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
authority: Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>,
|
||||
) -> Result<()> {
|
||||
condition.target.validate()?;
|
||||
if !authority.is_current()
|
||||
|| [
|
||||
condition.expected_metadata_digest.as_str(),
|
||||
condition.unchanged_metadata_digest.as_str(),
|
||||
]
|
||||
.iter()
|
||||
.any(|digest| {
|
||||
digest.len() != 64
|
||||
|| !digest
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
})
|
||||
{
|
||||
return Err(DiskError::OutdatedXLMeta);
|
||||
}
|
||||
let metadata_path = format!("{object}/{STORAGE_FORMAT_FILE}");
|
||||
let original = self.read_all(volume, &metadata_path).await?;
|
||||
let original_digest = rustfs_utils::crypto::hex(<sha2::Sha256 as sha2::Digest>::digest(&original));
|
||||
let mut metadata = FileMeta::load(&original)?;
|
||||
let (_, selected) = metadata.find_version(version_id)?;
|
||||
if selected.into_fileinfo(volume, object, true)?.transition_tier != condition.tier {
|
||||
return Err(DiskError::OutdatedXLMeta);
|
||||
}
|
||||
let generation = metadata.transition_reconcile_generation(version_id)?;
|
||||
if rustfs_utils::crypto::hex(<sha2::Sha256 as sha2::Digest>::digest(&generation)) != condition.unchanged_metadata_digest {
|
||||
return Err(DiskError::OutdatedXLMeta);
|
||||
}
|
||||
let changed = metadata.reconcile_transition_state(version_id, &condition.target)?;
|
||||
if !changed {
|
||||
return Ok(());
|
||||
}
|
||||
if condition.verify_only || original_digest != condition.expected_metadata_digest {
|
||||
return Err(DiskError::OutdatedXLMeta);
|
||||
}
|
||||
// fsync_dir_std is a no-op outside Unix, so those platforms cannot
|
||||
// yet prove this repair's durable publication requirement.
|
||||
if !cfg!(unix) || !effective_durability(volume).syncs_commit_metadata() {
|
||||
return Err(DiskError::other(
|
||||
"transition reconciliation requires Unix directory sync and enabled bucket metadata durability",
|
||||
));
|
||||
}
|
||||
// An outstanding rollback can still restore an older whole xl.meta.
|
||||
// Its preparation and execution share this mutation domain; refuse
|
||||
// repair until that transaction has settled and removed its backup.
|
||||
let mut entries = fs::read_dir(self.io_get_object_path(volume, object)?)
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
let mut remaining = 4096usize;
|
||||
while let Some(entry) = entries.next_entry().await.map_err(to_file_error)? {
|
||||
remaining = remaining.checked_sub(1).ok_or(DiskError::OutdatedXLMeta)?;
|
||||
if Uuid::parse_str(&entry.file_name().to_string_lossy()).is_err() {
|
||||
continue;
|
||||
}
|
||||
for marker in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
|
||||
if fs::try_exists(entry.path().join(marker)).await.map_err(to_file_error)? {
|
||||
return Err(DiskError::OutdatedXLMeta);
|
||||
}
|
||||
}
|
||||
}
|
||||
let replacement = metadata.marshal_msg()?;
|
||||
let tmp_volume = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
|
||||
let tmp_file = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &Uuid::new_v4().to_string())?;
|
||||
// Admission above requires metadata durability. Keep rename and its
|
||||
// directory sync in the same owned executor even after cancellation.
|
||||
self.write_all_internal(&tmp_file, InternalBuf::Ref(&replacement), SyncMode::FileOnly, &tmp_volume)
|
||||
.await?;
|
||||
if crash_inject::should_crash_at(CrashPoint::MetaWriteAfterTmpBeforeRename, &metadata_path) {
|
||||
return Err(DiskError::Unexpected);
|
||||
}
|
||||
os::rename_reconciled_metadata(
|
||||
tmp_file,
|
||||
self.io_get_object_path(volume, &metadata_path)?,
|
||||
self.io_get_bucket_path(volume)?,
|
||||
self.publication_root.clone(),
|
||||
namespace_owner.clone(),
|
||||
authority,
|
||||
)
|
||||
.await?;
|
||||
// Keep the same mutation lease through strong readback. Response loss
|
||||
// leaves a monotonic subset for the coordinator's exact-copy retry.
|
||||
let committed = self.read_all(volume, &metadata_path).await?;
|
||||
let mut committed = FileMeta::load(&committed)?;
|
||||
if committed.reconcile_transition_state(version_id, &condition.target)?
|
||||
|| committed.transition_reconcile_generation(version_id)? != generation
|
||||
{
|
||||
return Err(DiskError::OutdatedXLMeta);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn write_all_meta(&self, volume: &str, path: &str, buf: &[u8], sync: bool) -> Result<()> {
|
||||
self.write_all_meta_with_namespace_owner(volume, path, buf, sync, None).await
|
||||
}
|
||||
@@ -8511,7 +8321,6 @@ impl LocalDisk {
|
||||
Ok(Arc::new(QuotaMutationFenceClaim { state }))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn reserve_version_delete(&self, volume: &str, object: &str, data_dir: Uuid, rollback_dir: Uuid) -> Result<bool> {
|
||||
self.reserve_version_delete_with_namespace_owner(volume, object, data_dir, rollback_dir, None)
|
||||
.await
|
||||
@@ -8564,7 +8373,6 @@ impl LocalDisk {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn commit_reserved_version_delete(&self, volume: &str, object: &str, rollback_dir: Uuid) -> Result<()> {
|
||||
self.commit_reserved_version_delete_with_namespace_owner(volume, object, rollback_dir, None)
|
||||
.await
|
||||
@@ -8656,6 +8464,29 @@ impl LocalDisk {
|
||||
first_err.map_or(Ok(found), Err)
|
||||
}
|
||||
|
||||
async fn abort_reserved_version_delete(
|
||||
&self,
|
||||
object_dir: &Path,
|
||||
rollback_dir: Uuid,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
stage: &'static str,
|
||||
err: DiskError,
|
||||
) -> DiskError {
|
||||
self.abort_reserved_version_delete_with_failure(
|
||||
object_dir,
|
||||
rollback_dir,
|
||||
volume,
|
||||
object,
|
||||
DeleteRollbackFailure {
|
||||
stage,
|
||||
error: err,
|
||||
namespace_owner: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn abort_reserved_version_delete_with_failure(
|
||||
&self,
|
||||
object_dir: &Path,
|
||||
@@ -10200,23 +10031,6 @@ impl DiskAPI for LocalDisk {
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
|
||||
let object_path = self.get_object_path(volume, path)?;
|
||||
if let Some(condition) = &opts.transition_reconcile {
|
||||
if !fi.metadata.is_empty() || opts.no_persistence || opts.replace_user_metadata {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let authority =
|
||||
crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority::for_disk(
|
||||
condition,
|
||||
)
|
||||
.await?;
|
||||
let owner: Option<Arc<dyn Send + Sync>> = Some(authority.clone());
|
||||
let owner: Option<Arc<dyn Send + Sync>> = Some(os::acquire_metadata_mutation_lease(&object_path, owner).await);
|
||||
return self
|
||||
.reconcile_transition_state_metadata(volume, path, fi.version_id, condition, owner, authority)
|
||||
.await;
|
||||
}
|
||||
let namespace_owner: Option<Arc<dyn Send + Sync>> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await);
|
||||
if !fi.metadata.is_empty() {
|
||||
let file_path = self.io_get_object_path(volume, path)?;
|
||||
|
||||
@@ -10244,13 +10058,7 @@ impl DiskAPI for LocalDisk {
|
||||
let wbuf = xl_meta.marshal_msg()?;
|
||||
|
||||
return self
|
||||
.write_all_meta_with_namespace_owner(
|
||||
volume,
|
||||
format!("{path}/{STORAGE_FORMAT_FILE}").as_str(),
|
||||
&wbuf,
|
||||
!opts.no_persistence,
|
||||
namespace_owner,
|
||||
)
|
||||
.write_all_meta(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str(), &wbuf, !opts.no_persistence)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -10258,10 +10066,7 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
|
||||
let object_path = self.get_object_path(volume, path)?;
|
||||
let namespace_owner: Option<Arc<dyn Send + Sync>> = Some(os::acquire_metadata_mutation_lease(&object_path, None).await);
|
||||
self.write_metadata_with_namespace_owner(volume, path, fi, namespace_owner)
|
||||
.await
|
||||
self.write_metadata_with_namespace_owner(volume, path, fi, None).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
|
||||
@@ -279,14 +279,11 @@ impl LocalDisk {
|
||||
Some(token) => Some(self.claim_quota_mutation_fence(dst_volume, dst_path, token).await?),
|
||||
None => None,
|
||||
};
|
||||
// Quota admission -> metadata RMW -> namespace/volume publication.
|
||||
let metadata_lease =
|
||||
os::acquire_metadata_mutation_lease(&self.get_object_path(dst_volume, dst_path)?, state.namespace_owner.take()).await;
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&destination_object_path,
|
||||
Some(metadata_lease),
|
||||
state.namespace_owner.take(),
|
||||
)
|
||||
.await;
|
||||
if let Some(claim) = quota_fence_claim {
|
||||
|
||||
@@ -1277,25 +1277,6 @@ pub struct CheckPartsResp {
|
||||
pub struct UpdateMetadataOpts {
|
||||
pub no_persistence: bool,
|
||||
pub replace_user_metadata: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub transition_reconcile: Option<Box<TransitionStateReconcileCondition>>,
|
||||
}
|
||||
|
||||
/// An exact-copy precondition for the single-version tier repair protocol.
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TransitionStateReconcileCondition {
|
||||
pub expected_metadata_digest: String,
|
||||
pub unchanged_metadata_digest: String,
|
||||
pub target: rustfs_filemeta::TransitionStateReconcileTarget,
|
||||
pub tier: String,
|
||||
pub topology_generation: String,
|
||||
pub verify_only: bool,
|
||||
/// Local ownership is never accepted from the wire. A remote disk acquires
|
||||
/// its own fleet and backend leases before entering the mutation domain.
|
||||
#[serde(skip)]
|
||||
pub(crate) authority:
|
||||
Option<Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>>,
|
||||
}
|
||||
|
||||
pub struct DiskLocation {
|
||||
|
||||
+39
-196
@@ -452,13 +452,13 @@ pub(crate) mod windows_rename_test_hooks {
|
||||
|
||||
/// Test-only hooks into the destination-parent walk of rename preparation.
|
||||
///
|
||||
/// Pruning and Windows sharing races live between syscalls inside
|
||||
/// The prune race lives between two syscalls inside
|
||||
/// [`mkdir_all_below_existing_base_std`], so only an injection at that exact
|
||||
/// point reproduces it deterministically. Hooks are keyed by the absolute path
|
||||
/// of the component just opened and queued per path: a retrying preparation
|
||||
/// visits the same component again, so a test models a pruner that keeps
|
||||
/// walking upward by queueing one hook per visit.
|
||||
#[cfg(all(test, any(unix, windows)))]
|
||||
#[cfg(all(test, unix))]
|
||||
pub(crate) mod prepare_rename_test_hooks {
|
||||
use super::*;
|
||||
|
||||
@@ -1449,38 +1449,6 @@ fn disk_namespace_mutation_lock(path: &Path) -> Arc<NamespaceMutationLock> {
|
||||
lock
|
||||
}
|
||||
|
||||
static DISK_METADATA_MUTATION_LOCKS: LazyLock<Mutex<NamespaceMutationLockRegistry>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
/// Serializes the complete xl.meta read/modify/commit transaction. This domain
|
||||
/// precedes namespace/volume publication locks, whose narrower syscall leases
|
||||
/// may retain it after cancellation of the async caller.
|
||||
pub(crate) struct MetadataMutationLease {
|
||||
_guard: OwnedMutexGuard<()>,
|
||||
_owner: Option<Arc<dyn Send + Sync>>,
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_metadata_mutation_lease(
|
||||
object: &Path,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Arc<MetadataMutationLease> {
|
||||
let lock = {
|
||||
let mut locks = DISK_METADATA_MUTATION_LOCKS.lock();
|
||||
locks.retain(|_, lock| lock.strong_count() > 0);
|
||||
if let Some(lock) = locks.get(object).and_then(Weak::upgrade) {
|
||||
lock
|
||||
} else {
|
||||
let lock = Arc::new(AsyncMutex::new(()));
|
||||
locks.insert(object.to_path_buf(), Arc::downgrade(&lock));
|
||||
lock
|
||||
}
|
||||
};
|
||||
Arc::new(MetadataMutationLease {
|
||||
_guard: lock.lock_owned().await,
|
||||
_owner: owner,
|
||||
})
|
||||
}
|
||||
|
||||
/// Keeps a namespace transaction serialized even when its async waiter is
|
||||
/// cancelled while a blocking filesystem call is still running.
|
||||
pub(crate) struct NamespaceMutationLease {
|
||||
@@ -2144,41 +2112,6 @@ pub(crate) async fn rename_all_with_lease(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Publish a conditional repair and sync its directory in one owned executor.
|
||||
/// Cancellation cannot release its metadata/fleet/tier leases between rename
|
||||
/// and fsync. The last authority check runs after destination preparation.
|
||||
pub(crate) async fn rename_reconciled_metadata(
|
||||
source: PathBuf,
|
||||
destination: PathBuf,
|
||||
base_dir: PathBuf,
|
||||
publication_root: PublicationRoot,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
authority: Arc<crate::bucket::lifecycle::legacy_transition_state_reconcile::TransitionStateReconcileAuthority>,
|
||||
) -> Result<()> {
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&destination, owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
let preparation = prepare_rename_with_retry(&source, &destination, &base_dir, &publication_root)?;
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &destination);
|
||||
if !authority.is_current() {
|
||||
return Err(io::Error::new(io::ErrorKind::WouldBlock, "transition reconciliation authority expired"));
|
||||
}
|
||||
rename_prepared(&source, &destination, &preparation)?;
|
||||
if let Some(parent) = destination.parent() {
|
||||
fsync_dir_std(parent)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|err| {
|
||||
if err.kind() == io::ErrorKind::WouldBlock {
|
||||
DiskError::OutdatedXLMeta
|
||||
} else {
|
||||
to_file_error(err).into()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub(crate) async fn rename_all_with_commit_guard(
|
||||
@@ -4275,14 +4208,11 @@ pub(crate) fn mkdir_all_below_existing_base_std(
|
||||
let mut handles = Vec::with_capacity(capacity);
|
||||
handles.push(publication_root.directory.clone());
|
||||
let mut guard = ExistingBaseDirectoryGuard::new(handles);
|
||||
let mut components = base_relative
|
||||
.components()
|
||||
.map(|component| (component, FILE_OPEN))
|
||||
.chain(relative.components().map(|component| (component, FILE_OPEN_IF)))
|
||||
.filter(|(component, _)| !matches!(component, Component::CurDir))
|
||||
.peekable();
|
||||
while let Some((component, disposition)) = components.next() {
|
||||
for component in base_relative.components() {
|
||||
let Component::Normal(component) = component else {
|
||||
if matches!(component, Component::CurDir) {
|
||||
continue;
|
||||
}
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"rename base directory contains an invalid path component",
|
||||
@@ -4292,22 +4222,40 @@ pub(crate) fn mkdir_all_below_existing_base_std(
|
||||
.handles
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("Windows publication root guard is empty"))?;
|
||||
// The kernel opens the final parent for write during a relative
|
||||
// rename. Share writes from its first open: a temporary read-only
|
||||
// share would block another rename into the same trash directory.
|
||||
// Ancestors stay strict and no handle shares delete access, keeping
|
||||
// every retained directory identity pinned.
|
||||
let share_access = if components.peek().is_none() {
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE
|
||||
} else {
|
||||
FILE_SHARE_READ
|
||||
};
|
||||
let child = open_windows_relative_directory_component(parent, component, disposition, share_access)?;
|
||||
let child = open_windows_directory_component(parent, component, FILE_OPEN)?;
|
||||
guard.handles.push(child);
|
||||
#[cfg(test)]
|
||||
if components.peek().is_none() {
|
||||
prepare_rename_test_hooks::run_after_component_opened(dir_path);
|
||||
}
|
||||
}
|
||||
for component in relative.components() {
|
||||
let Component::Normal(component) = component else {
|
||||
continue;
|
||||
};
|
||||
let parent = guard
|
||||
.handles
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("Windows base directory guard is empty"))?;
|
||||
let child = open_windows_directory_component(parent, component, FILE_OPEN_IF)?;
|
||||
guard.handles.push(child);
|
||||
}
|
||||
|
||||
// Windows resolves a handle-relative rename by opening the target for
|
||||
// write. Keep every ancestor strict, but let that internal open share
|
||||
// the final parent. Delete sharing remains omitted, so the retained
|
||||
// directory entry cannot be renamed or removed during publication.
|
||||
if guard.handles.len() > 1 {
|
||||
let component = dir_path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination parent must have a name"))?;
|
||||
let parent_index = guard.handles.len() - 2;
|
||||
let parent = guard
|
||||
.handles
|
||||
.get(parent_index)
|
||||
.ok_or_else(|| io::Error::other("Windows destination guard lost its parent handle"))?;
|
||||
let rename_parent =
|
||||
open_windows_relative_directory_component(parent, component, FILE_OPEN, FILE_SHARE_READ | FILE_SHARE_WRITE)?;
|
||||
*guard
|
||||
.handles
|
||||
.last_mut()
|
||||
.ok_or_else(|| io::Error::other("Windows destination guard is empty"))? = rename_parent;
|
||||
}
|
||||
|
||||
Ok(guard)
|
||||
@@ -5498,111 +5446,6 @@ mod tests {
|
||||
assert_eq!(std::fs::read(dst).expect("read committed metadata"), b"metadata");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn rename_all_concurrent_trash_renames_remove_every_rollback_directory() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let trash = temp_dir.path().join(".rustfs.sys/tmp/.trash");
|
||||
std::fs::create_dir_all(&trash).expect("create trash directory");
|
||||
let publication_root = PublicationRoot::new(temp_dir.path()).expect("open publication root");
|
||||
let sources: Vec<_> = (0..16)
|
||||
.map(|index| {
|
||||
let source = temp_dir
|
||||
.path()
|
||||
.join(format!("bucket/{index}.mp4"))
|
||||
.join(uuid::Uuid::new_v4().to_string());
|
||||
std::fs::create_dir_all(&source).expect("create rollback directory");
|
||||
std::fs::write(source.join("xl.meta.bkp"), b"rollback metadata").expect("write metadata backup");
|
||||
source
|
||||
})
|
||||
.collect();
|
||||
|
||||
let results = futures::future::join_all(sources.iter().enumerate().map(|(index, source)| {
|
||||
super::rename_all_ignore_missing_source(source, trash.join(index.to_string()), &trash, &publication_root)
|
||||
}))
|
||||
.await;
|
||||
|
||||
for (index, (source, result)) in sources.iter().zip(results).enumerate() {
|
||||
result.expect("concurrent rollback cleanup must reach the shared trash directory");
|
||||
assert!(!source.exists(), "rollback cleanup must not leave a directory in the bucket");
|
||||
assert_eq!(
|
||||
std::fs::read(trash.join(index.to_string()).join("xl.meta.bkp")).expect("read moved metadata backup"),
|
||||
b"rollback metadata",
|
||||
"trash staging must retain the complete backup"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn windows_trash_rename_succeeds_during_concurrent_parent_preparation() {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use windows_sys::Win32::{
|
||||
Foundation::{ERROR_SHARING_VIOLATION, GENERIC_WRITE},
|
||||
Storage::FileSystem::{
|
||||
DELETE, FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_DELETE, FILE_SHARE_READ,
|
||||
FILE_SHARE_WRITE,
|
||||
},
|
||||
};
|
||||
|
||||
// Exercise the final parent both in the existing base walk and in the
|
||||
// creatable suffix walk. Neither may briefly deny write sharing.
|
||||
for nested_parent in [false, true] {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let tmp = temp_dir.path().join(".rustfs.sys/tmp");
|
||||
let trash = tmp.join(".trash");
|
||||
std::fs::create_dir_all(&trash).expect("create trash directory");
|
||||
let base = if nested_parent { &tmp } else { &trash };
|
||||
let publication_root = PublicationRoot::new(temp_dir.path()).expect("open publication root");
|
||||
let sources = ["first", "second"].map(|name| {
|
||||
let source = temp_dir
|
||||
.path()
|
||||
.join("bucket")
|
||||
.join(name)
|
||||
.join(uuid::Uuid::new_v4().to_string());
|
||||
std::fs::create_dir_all(&source).expect("create rollback directory");
|
||||
std::fs::write(source.join("xl.meta.bkp"), name.as_bytes()).expect("write metadata backup");
|
||||
source
|
||||
});
|
||||
let destinations = [trash.join("first"), trash.join("second")];
|
||||
let first_preparation = prepare_rename_with_retry(&sources[0], &destinations[0], base, &publication_root)
|
||||
.expect("prepare the first trash rename");
|
||||
let first_source = sources[0].clone();
|
||||
let first_destination = destinations[0].clone();
|
||||
let interleaved = Arc::new(AtomicBool::new(false));
|
||||
let interleaved_hook = Arc::clone(&interleaved);
|
||||
prepare_rename_test_hooks::queue_after_component_opened(&trash, move || {
|
||||
interleaved_hook.store(true, Ordering::Release);
|
||||
rename_prepared(&first_source, &first_destination, &first_preparation)
|
||||
.expect("another preparation's first parent handle must allow the pending trash rename");
|
||||
});
|
||||
|
||||
let second_preparation = prepare_rename_with_retry(&sources[1], &destinations[1], base, &publication_root)
|
||||
.expect("prepare the second trash rename");
|
||||
assert!(interleaved.load(Ordering::Acquire), "the competing parent-open window must be exercised");
|
||||
for (path, access) in [(&tmp, GENERIC_WRITE), (&trash, DELETE)] {
|
||||
let err = std::fs::OpenOptions::new()
|
||||
.access_mode(access)
|
||||
.share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
|
||||
.open(path)
|
||||
.expect_err("ancestor writes and final-parent deletion must remain excluded");
|
||||
assert_eq!(
|
||||
err.raw_os_error(),
|
||||
Some(i32::try_from(ERROR_SHARING_VIOLATION).expect("Windows error code must fit i32"))
|
||||
);
|
||||
}
|
||||
rename_prepared(&sources[1], &destinations[1], &second_preparation).expect("publish the second trash rename");
|
||||
for (index, payload) in [b"first".as_slice(), b"second".as_slice()].into_iter().enumerate() {
|
||||
assert!(!sources[index].exists(), "both rollback directories must leave the bucket");
|
||||
assert_eq!(
|
||||
std::fs::read(destinations[index].join("xl.meta.bkp")).expect("read the staged backup"),
|
||||
payload
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[tokio::test]
|
||||
async fn windows_rename_all_supports_same_parent_publication() {
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
sync::{Arc, LazyLock, OnceLock, RwLock as StdRwLock},
|
||||
sync::{Arc, OnceLock},
|
||||
time::SystemTime,
|
||||
};
|
||||
|
||||
@@ -57,13 +57,6 @@ use uuid::Uuid;
|
||||
const TEST_RPC_SECRET: &str = "test-rpc-secret";
|
||||
|
||||
pub(crate) type WorkloadSnapshotProviderRef = Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>;
|
||||
pub type ScannerDirtyUsageMutationObserver = Arc<dyn Fn(&str, &str, ScannerDirtyUsageMutationSource) + Send + Sync + 'static>;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ScannerDirtyUsageMutationSource {
|
||||
Replication,
|
||||
TierExpiration,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct LockRegistry {
|
||||
@@ -95,8 +88,6 @@ impl LockRegistry {
|
||||
}
|
||||
|
||||
static WORKLOAD_ADMISSION_SNAPSHOT_PROVIDER: OnceLock<WorkloadSnapshotProviderRef> = OnceLock::new();
|
||||
static SCANNER_DIRTY_USAGE_MUTATION_OBSERVER: LazyLock<StdRwLock<Option<ScannerDirtyUsageMutationObserver>>> =
|
||||
LazyLock::new(|| StdRwLock::new(None));
|
||||
|
||||
pub(crate) fn set_workload_admission_snapshot_provider(
|
||||
provider: WorkloadSnapshotProviderRef,
|
||||
@@ -108,28 +99,6 @@ pub(crate) fn workload_admission_snapshot_provider() -> Option<WorkloadSnapshotP
|
||||
WORKLOAD_ADMISSION_SNAPSHOT_PROVIDER.get().cloned()
|
||||
}
|
||||
|
||||
pub fn set_scanner_dirty_usage_mutation_observer(
|
||||
observer: Option<ScannerDirtyUsageMutationObserver>,
|
||||
) -> Option<ScannerDirtyUsageMutationObserver> {
|
||||
let mut slot = SCANNER_DIRTY_USAGE_MUTATION_OBSERVER
|
||||
.write()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
std::mem::replace(&mut *slot, observer)
|
||||
}
|
||||
|
||||
pub(crate) fn notify_scanner_dirty_usage_mutation(bucket: &str, object: &str, source: ScannerDirtyUsageMutationSource) {
|
||||
if bucket.is_empty() || object.is_empty() {
|
||||
return;
|
||||
}
|
||||
let observer = SCANNER_DIRTY_USAGE_MUTATION_OBSERVER
|
||||
.read()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clone();
|
||||
if let Some(observer) = observer {
|
||||
observer(bucket, object, source);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_erasure_write_quorum_failure(stage: &'static str, dominant_error: &'static str) {
|
||||
global_internode_metrics().record_erasure_write_quorum_failure(stage, dominant_error);
|
||||
}
|
||||
@@ -611,16 +580,12 @@ pub(crate) async fn init_tier_config_mgr(store: Arc<ECStore>) -> Result<()> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
LockRegistry, ScannerDirtyUsageMutationSource, clear_local_disk_id_map_for_test, local_disk_path_by_id, local_node_name,
|
||||
notify_scanner_dirty_usage_mutation, reconcile_local_disk_ids, replace_local_disk_id, set_local_node_name,
|
||||
set_scanner_dirty_usage_mutation_observer,
|
||||
LockRegistry, clear_local_disk_id_map_for_test, local_disk_path_by_id, local_node_name, reconcile_local_disk_ids,
|
||||
replace_local_disk_id, set_local_node_name,
|
||||
};
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use rustfs_lock::{LocalClient, LockClient};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use uuid::Uuid;
|
||||
|
||||
fn url_endpoint(raw: &str) -> Endpoint {
|
||||
@@ -655,42 +620,6 @@ mod tests {
|
||||
assert!(Arc::ptr_eq(&clients[1], &client_b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(scanner_dirty_usage_mutation_observer)]
|
||||
fn scanner_dirty_usage_mutation_observer_filters_empty_identity_and_preserves_source() {
|
||||
let observed = Arc::new(Mutex::new(Vec::new()));
|
||||
let observed_clone = Arc::clone(&observed);
|
||||
let previous = set_scanner_dirty_usage_mutation_observer(Some(Arc::new(move |bucket, object, source| {
|
||||
observed_clone.lock().expect("observer lock should not be poisoned").push((
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
source,
|
||||
));
|
||||
})));
|
||||
|
||||
notify_scanner_dirty_usage_mutation("photos", "2026/image.jpg", ScannerDirtyUsageMutationSource::Replication);
|
||||
notify_scanner_dirty_usage_mutation("", "2026/empty-bucket.jpg", ScannerDirtyUsageMutationSource::TierExpiration);
|
||||
notify_scanner_dirty_usage_mutation("photos", "", ScannerDirtyUsageMutationSource::TierExpiration);
|
||||
notify_scanner_dirty_usage_mutation("archive", "expired.bin", ScannerDirtyUsageMutationSource::TierExpiration);
|
||||
set_scanner_dirty_usage_mutation_observer(previous);
|
||||
|
||||
assert_eq!(
|
||||
*observed.lock().expect("observer lock should not be poisoned"),
|
||||
vec![
|
||||
(
|
||||
"photos".to_string(),
|
||||
"2026/image.jpg".to_string(),
|
||||
ScannerDirtyUsageMutationSource::Replication
|
||||
),
|
||||
(
|
||||
"archive".to_string(),
|
||||
"expired.bin".to_string(),
|
||||
ScannerDirtyUsageMutationSource::TierExpiration
|
||||
),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn local_node_name_round_trips_through_common_runtime_helper() {
|
||||
|
||||
@@ -67,9 +67,12 @@ const DECOMMISSION_TARGET_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
|
||||
// Keep this synchronized with the version served by node_service. Including
|
||||
// the local member in the minimum prevents an older coordinator from
|
||||
// self-authorizing a policy implemented only by newer remote peers.
|
||||
const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 5;
|
||||
/// Version 5 preserves explicit transition state/destination bindings and
|
||||
/// supports exact-generation metadata repair with strong all-copy readback.
|
||||
const LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION: u32 = 4;
|
||||
/// Version 5 is reserved for a fleet whose every metadata writer preserves
|
||||
/// explicit transition version state and destination identity, and implements
|
||||
/// conditional per-generation `xl.meta` writes with strong readback. The node
|
||||
/// service must not advertise this version until the conditional writer from
|
||||
/// rustfs/backlog#684 is available.
|
||||
const LEGACY_TRANSITION_STATE_RECONCILE_POLICY_SUPPORTED_VERSION: u32 = 5;
|
||||
|
||||
fn resolve_admin_peer_probe_timeout_secs(configured: Option<u64>) -> u64 {
|
||||
@@ -565,10 +568,6 @@ pub(crate) fn tier_delete_journal_topology_generation(proof: &TierDeleteJournalF
|
||||
stable_tier_delete_journal_topology_generation(&proof.token.topology_fingerprint)
|
||||
}
|
||||
|
||||
pub(crate) fn cross_pool_fence_topology_generation(proof: &CrossPoolFenceFleetProofToken) -> String {
|
||||
stable_tier_delete_journal_topology_generation(&proof.0.topology_fingerprint)
|
||||
}
|
||||
|
||||
/// Acquire one non-cloneable authority that must span the complete reconcile
|
||||
/// effect window, including its final strong readback.
|
||||
pub async fn acquire_legacy_transition_state_reconcile_fleet_proof() -> Option<LegacyTransitionStateReconcileFleetProofToken> {
|
||||
@@ -604,10 +603,6 @@ fn acquire_legacy_transition_state_reconcile_fleet_proof_from(
|
||||
}
|
||||
|
||||
async fn observe_legacy_transition_state_reconcile_fleet(expected_topology: &str) -> Option<BTreeMap<String, Uuid>> {
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
if let Ok(observation) = LEGACY_RECONCILE_TEST_OBSERVATION.try_with(Clone::clone) {
|
||||
return Some(observation);
|
||||
}
|
||||
let notification_sys = get_global_notification_sys()?;
|
||||
let (peer_epochs, minimum_version) = timeout(
|
||||
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
|
||||
@@ -620,34 +615,6 @@ async fn observe_legacy_transition_state_reconcile_fleet(expected_topology: &str
|
||||
reconcile_result.ok()
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
tokio::task_local! {
|
||||
static LEGACY_RECONCILE_TEST_OBSERVATION: BTreeMap<String, Uuid>;
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) async fn with_legacy_transition_state_fleet_proof_for_test<F: std::future::Future>(future: F) -> F::Output {
|
||||
struct Revoke;
|
||||
impl Drop for Revoke {
|
||||
fn drop(&mut self) {
|
||||
revoke_fleet_capability_proof(legacy_transition_state_reconcile_fleet_proof_slot());
|
||||
}
|
||||
}
|
||||
let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get().expect("test store topology");
|
||||
assert!(
|
||||
publish_fleet_capability_probe_result(
|
||||
legacy_transition_state_reconcile_fleet_proof_slot(),
|
||||
topology,
|
||||
Ok(BTreeMap::new()),
|
||||
Instant::now(),
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
let _revoke = Revoke;
|
||||
let _remote_version = install_current_remote_version_state_fleet_proof_for_test();
|
||||
LEGACY_RECONCILE_TEST_OBSERVATION.scope(BTreeMap::new(), future).await
|
||||
}
|
||||
|
||||
/// Revalidate the exact fleet generation captured by a reconcile token with a
|
||||
/// fresh synchronous observation. Callers must await this before each
|
||||
/// conditional metadata write and after the final strong readback.
|
||||
@@ -666,18 +633,6 @@ pub async fn legacy_transition_state_reconcile_fleet_proof_matches(
|
||||
.await
|
||||
}
|
||||
|
||||
/// Final local check in the disk publication executor. The corresponding
|
||||
/// counted permit remains owned until the filesystem operation has drained.
|
||||
pub(crate) fn legacy_transition_state_reconcile_fleet_proof_current(
|
||||
proof: &LegacyTransitionStateReconcileFleetProofToken,
|
||||
) -> bool {
|
||||
let Some(topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else { return false };
|
||||
let state = legacy_transition_state_reconcile_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
legacy_transition_state_reconcile_fleet_proof_matches_at(&state, proof, topology, Instant::now())
|
||||
}
|
||||
|
||||
pub async fn acquire_ilm_recovery_export_fleet_proof() -> Option<IlmRecoveryExportFleetProofToken> {
|
||||
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
|
||||
let proof = {
|
||||
@@ -1165,14 +1120,6 @@ pub(crate) fn install_remote_version_state_fleet_proof_for_test(topology_fingerp
|
||||
RemoteVersionStateFleetProofGuard
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) fn install_current_remote_version_state_fleet_proof_for_test() -> RemoteVersionStateFleetProofGuard {
|
||||
let topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY
|
||||
.get()
|
||||
.expect("the test store must bind its fleet topology before installing a writer proof");
|
||||
install_remote_version_state_fleet_proof_for_test(topology)
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) struct TransitionTransactionCompactionFleetProofGuard;
|
||||
|
||||
@@ -3967,11 +3914,15 @@ mod tests {
|
||||
assert!(decommission_v3.is_err(), "v3 members do not understand the per-target decommission fence");
|
||||
assert!(reconcile_v3.is_err());
|
||||
|
||||
let (generic_v4, journal_v4, decommission_v4, reconcile_v4) = cross_pool_fence_policy_results(peers.clone(), 4);
|
||||
let (generic_v4, journal_v4, decommission_v4, reconcile_v4) =
|
||||
cross_pool_fence_policy_results(peers.clone(), LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
|
||||
assert!(generic_v4.is_ok());
|
||||
assert!(journal_v4.is_ok());
|
||||
assert!(decommission_v4.is_ok(), "an all-v4 fleet may create sticky per-target reservations");
|
||||
assert!(reconcile_v4.is_err(), "v4 does not support conditional transition metadata writes");
|
||||
assert!(
|
||||
reconcile_v4.is_err(),
|
||||
"the current local policy lacks the conditional xl.meta writer required by reconcile"
|
||||
);
|
||||
|
||||
let (generic_v5, journal_v5, decommission_v5, reconcile_v5) = cross_pool_fence_policy_results(peers, 5);
|
||||
assert!(generic_v5.is_ok());
|
||||
@@ -4649,7 +4600,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_transition_state_reconcile_single_node_advertises_conditional_writer() {
|
||||
async fn legacy_transition_state_reconcile_single_node_stays_closed_before_local_cas_support() {
|
||||
let notification_sys = NotificationSys {
|
||||
peer_clients: Vec::new(),
|
||||
all_peer_clients: vec![None],
|
||||
@@ -4665,8 +4616,8 @@ mod tests {
|
||||
assert_eq!(minimum_version, LOCAL_CROSS_POOL_FENCE_POLICY_SUPPORTED_VERSION);
|
||||
let (_, _, _, reconcile_result) = cross_pool_fence_policy_results(peers, minimum_version);
|
||||
assert!(
|
||||
reconcile_result.is_ok(),
|
||||
"the current node implements the conditional writer and preserves repaired bindings"
|
||||
reconcile_result.is_err(),
|
||||
"the current node must not self-authorize reconcile before the conditional writer lands"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -572,7 +572,7 @@ impl ECStore {
|
||||
where
|
||||
S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
|
||||
{
|
||||
// Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin.
|
||||
// Lock order: pool_meta_save_gate -> pool.bin -> rebalance.bin.
|
||||
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
|
||||
pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
|
||||
// Classify the durable rebalance record while holding both namespace
|
||||
|
||||
@@ -50,11 +50,6 @@ fn ensure_rebalance_entry_active(cancel: &CancellationToken) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
static REBALANCE_ENTRY_RUN_FENCE_BARRIER: (Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>);
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct RebalanceEntryTarget {
|
||||
bucket: String,
|
||||
@@ -261,15 +256,9 @@ impl ECStore {
|
||||
.sort_by_key(|v| (v.mod_time.is_none(), std::cmp::Reverse(v.mod_time)));
|
||||
|
||||
// Entry lock order is bucket incarnation -> activation_gate -> rebalance.bin -> movement gate.
|
||||
// Target capacity admission can then acquire pool.bin under the run fence.
|
||||
// Stop waits for in-flight entries through cleanup, but not for entries admitted later.
|
||||
ensure_rebalance_entry_active(&cancel)?;
|
||||
let run_guard = self.rebalance_run_guard(rebalance_id.as_ref(), "rebalance entry").await?;
|
||||
#[cfg(test)]
|
||||
if let Ok((arrived, release)) = REBALANCE_ENTRY_RUN_FENCE_BARRIER.try_with(Clone::clone) {
|
||||
arrived.notify_one();
|
||||
release.notified().await;
|
||||
}
|
||||
let lock_lost_signal = run_guard.lock_lost_signal();
|
||||
#[cfg(test)]
|
||||
let _run_signal_test_fence = lock_lost_signal
|
||||
@@ -1248,130 +1237,6 @@ mod tests {
|
||||
assert_eq!(pool_stats.cleanup_warnings.count, 1, "deferred cleanup must not add a permanent warning");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn real_rebalance_entry_progresses_while_peer_activation_waits_for_run_fence() {
|
||||
const REBALANCE_ID: &str = "rebalance-peer-activation-lock-order";
|
||||
let (_temp_dirs, store, peer) = crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(Some(
|
||||
active_rebalance_meta(REBALANCE_ID),
|
||||
))
|
||||
.await;
|
||||
assert!(!Arc::ptr_eq(&store.ctx, &peer.ctx), "node-local movement gates must be independent");
|
||||
{
|
||||
let mut meta = peer.rebalance_meta.write().await;
|
||||
let meta = meta.as_mut().expect("peer should know the durable run");
|
||||
meta.activation_gate = Arc::default();
|
||||
meta.cancel = None;
|
||||
}
|
||||
let bucket = crate::disk::RUSTFS_META_BUCKET;
|
||||
let object = "rebalance-peer-activation-object";
|
||||
let version_id = uuid::Uuid::new_v4();
|
||||
let payload = b"entry must drain before peer activation takes the pool fence".repeat(1024);
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let target_set = store.pools[1].get_disks_by_key(object);
|
||||
let opts = ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version_id.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let mut writer = PutObjReader::from_vec(payload.clone());
|
||||
let source_before = source_set
|
||||
.put_object(bucket, object, &mut writer, &opts)
|
||||
.await
|
||||
.expect("source version should be written");
|
||||
let entry = metacache_entry_from_source(&source_set, bucket, object).await;
|
||||
let arrived = Arc::new(tokio::sync::Notify::new());
|
||||
let release = Arc::new(tokio::sync::Notify::new());
|
||||
// JoinSet aborts both scoped tasks if an assertion or timeout fails.
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
let entry_store = Arc::clone(&store);
|
||||
tasks.spawn(
|
||||
REBALANCE_ENTRY_RUN_FENCE_BARRIER.scope((Arc::clone(&arrived), Arc::clone(&release)), async move {
|
||||
entry_store
|
||||
.rebalance_entry(
|
||||
RebalanceEntryTarget {
|
||||
bucket: bucket.to_string(),
|
||||
pool_index: 0,
|
||||
},
|
||||
entry,
|
||||
source_set,
|
||||
Arc::new(RebalanceBucketConfigs::default()),
|
||||
Arc::from(REBALANCE_ID),
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
}),
|
||||
);
|
||||
tokio::time::timeout(StdDuration::from_secs(30), arrived.notified())
|
||||
.await
|
||||
.expect("real entry must acquire its persisted run read fence");
|
||||
|
||||
let attempted = Arc::new(tokio::sync::Notify::new());
|
||||
let peer_pool = Arc::clone(&peer.pools[0]);
|
||||
let (activation_done, activation_result) = tokio::sync::oneshot::channel();
|
||||
tasks.spawn(
|
||||
crate::core::pools::REBALANCE_ACTIVATION_LOCK_ATTEMPT.scope(Arc::clone(&attempted), async move {
|
||||
let result = peer.fence_rebalance_worker_activation(peer_pool, REBALANCE_ID).await;
|
||||
let result = result.map(|fence| match fence {
|
||||
super::super::control::RebalanceWorkerActivationFence::Ready(fence) => {
|
||||
fence.ensure_held().expect("peer activation must retain both fences");
|
||||
}
|
||||
super::super::control::RebalanceWorkerActivationFence::NotStartedTerminal => {
|
||||
panic!("the paused entry's run must still require activation");
|
||||
}
|
||||
});
|
||||
activation_done.send(result).expect("activation receiver should remain alive");
|
||||
Ok(RebalanceEntryOutcome::Completed)
|
||||
}),
|
||||
);
|
||||
tokio::time::timeout(StdDuration::from_secs(30), attempted.notified())
|
||||
.await
|
||||
.expect("peer activation must attempt the persisted rebalance write fence");
|
||||
release.notify_one();
|
||||
|
||||
tokio::time::timeout(StdDuration::from_secs(30), async {
|
||||
while let Some(result) = tasks.join_next().await {
|
||||
assert!(matches!(
|
||||
result
|
||||
.expect("scoped task must not panic")
|
||||
.expect("entry must not fail or defer"),
|
||||
RebalanceEntryOutcome::Completed
|
||||
));
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("entry and peer activation must both make progress");
|
||||
activation_result
|
||||
.await
|
||||
.expect("peer activation result should be sent")
|
||||
.expect("peer activation must not time out behind the entry it blocks");
|
||||
|
||||
let mut reader = target_set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("the exact target version must be readable");
|
||||
let mut actual = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("target body should drain completely");
|
||||
assert_eq!(actual, payload);
|
||||
assert_eq!(reader.object_info.version_id, source_before.version_id);
|
||||
assert_eq!(reader.object_info.etag, source_before.etag);
|
||||
assert_eq!(reader.object_info.mod_time, source_before.mod_time);
|
||||
let source_error = store.pools[0]
|
||||
.get_object_info(bucket, object, &opts)
|
||||
.await
|
||||
.expect_err("completed entry must clean up the source version");
|
||||
assert!(crate::error::is_err_object_not_found(&source_error) || crate::error::is_err_version_not_found(&source_error));
|
||||
let meta = store.rebalance_meta.read().await;
|
||||
let stats = &meta.as_ref().expect("local run must remain installed").pool_stats[0];
|
||||
assert_eq!(stats.num_objects, 1);
|
||||
assert_eq!(stats.num_versions, 1);
|
||||
assert_eq!(stats.cleanup_warnings.count, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn real_rebalance_run_fence_loss_before_target_commit_preserves_target_and_source() {
|
||||
|
||||
@@ -1907,124 +1907,6 @@ fn test_is_transient_rebalance_error_accepts_wrapped_disk_timeout() {
|
||||
assert!(is_transient_rebalance_error(&Error::Io(std::io::Error::other(DiskError::Timeout))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rebalance_stage_wrapped_transient_errors_remain_retryable() {
|
||||
let cases = [
|
||||
Error::Lock(rustfs_lock::LockError::timeout(".rustfs.sys/pool.bin@latest", Duration::from_secs(5))),
|
||||
Error::Lock(rustfs_lock::LockError::network(
|
||||
"peer unavailable",
|
||||
std::io::Error::from(std::io::ErrorKind::ConnectionReset),
|
||||
)),
|
||||
Error::SlowDown,
|
||||
Error::ErasureReadQuorum,
|
||||
Error::ErasureWriteQuorum,
|
||||
Error::Io(std::io::Error::other(DiskError::Timeout)),
|
||||
Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
|
||||
];
|
||||
for mut error in cases {
|
||||
for depth in 0..=3 {
|
||||
assert!(is_transient_rebalance_error(&error), "transient source lost at depth {depth}: {error:?}");
|
||||
assert!(
|
||||
should_defer_rebalance_entry_failure(&error),
|
||||
"exhausted transient entries must be deferred"
|
||||
);
|
||||
assert!(should_retry_rebalance_listing(&error, 0, 3));
|
||||
assert!(
|
||||
!should_retry_rebalance_listing(&error, 2, 3),
|
||||
"wrapping must not bypass the attempt limit"
|
||||
);
|
||||
error = data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance_object",
|
||||
"put_object",
|
||||
"bucket",
|
||||
"baseline/00042.bin",
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rebalance_stage_wrapped_terminal_errors_remain_terminal() {
|
||||
let cases = [
|
||||
Error::FileAccessDenied,
|
||||
Error::FileCorrupt,
|
||||
Error::OperationCanceled,
|
||||
Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string()),
|
||||
Error::Lock(rustfs_lock::LockError::already_locked("bucket/object", "owner")),
|
||||
Error::other("permission denied"),
|
||||
];
|
||||
for mut error in cases {
|
||||
for depth in 0..=3 {
|
||||
assert!(
|
||||
!is_transient_rebalance_error(&error),
|
||||
"terminal source must survive depth {depth}: {error:?}"
|
||||
);
|
||||
assert!(!should_defer_rebalance_entry_failure(&error));
|
||||
// Object names are untrusted context, not evidence of a transient failure.
|
||||
error = data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance_object",
|
||||
"put_object",
|
||||
"bucket",
|
||||
"remote lock rpc timed out",
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_rebalance_stage_wrapped_lock_timeout_retries_real_migration_loop() {
|
||||
for succeeds_on_retry in [true, false] {
|
||||
let backend = MigrationBackendSpy::new(None, None);
|
||||
let attempts = AtomicUsize::new(0);
|
||||
let waits = AtomicUsize::new(0);
|
||||
let mut transfer = |_, _, _| {
|
||||
let attempt = attempts.fetch_add(1, Ordering::SeqCst);
|
||||
async move {
|
||||
if succeeds_on_retry && attempt > 0 {
|
||||
return Ok(());
|
||||
}
|
||||
Err(data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance_object",
|
||||
"put_object",
|
||||
"bucket",
|
||||
"baseline/00042.bin",
|
||||
Error::Lock(rustfs_lock::LockError::timeout(".rustfs.sys/pool.bin@latest", Duration::from_secs(5))),
|
||||
))
|
||||
}
|
||||
};
|
||||
let version = version_normal();
|
||||
let result = migrate_entry_version_with_retry_wait(
|
||||
&backend,
|
||||
"bucket".to_string(),
|
||||
0,
|
||||
&version,
|
||||
None,
|
||||
3,
|
||||
false,
|
||||
&mut transfer,
|
||||
|_: String, _: String, _: ObjectOptions| async { Ok::<_, Error>(ObjectInfo::default()) },
|
||||
|_| {
|
||||
waits.fetch_add(1, Ordering::SeqCst);
|
||||
std::future::ready(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result.moved, succeeds_on_retry);
|
||||
assert_eq!(result.failed, !succeeds_on_retry);
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), if succeeds_on_retry { 2 } else { 3 });
|
||||
assert_eq!(backend.get_calls(), attempts.load(Ordering::SeqCst));
|
||||
assert_eq!(waits.load(Ordering::SeqCst), attempts.load(Ordering::SeqCst) - 1);
|
||||
if !succeeds_on_retry {
|
||||
assert_eq!(result.stage, Some("write_target"));
|
||||
assert!(should_defer_rebalance_entry_failure(
|
||||
result.error.as_ref().expect("exhaustion must retain its source error")
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_transient_rebalance_error_accepts_io_timeout_message() {
|
||||
assert!(is_transient_rebalance_error(&Error::Io(std::io::Error::other("timeout"))));
|
||||
|
||||
@@ -244,7 +244,6 @@ pub(super) fn resolve_rebalance_bucket_result(
|
||||
}
|
||||
|
||||
pub(super) fn is_transient_rebalance_error(err: &Error) -> bool {
|
||||
let err = rebalance_error_source(err);
|
||||
match err {
|
||||
Error::SlowDown
|
||||
| Error::ErasureReadQuorum
|
||||
@@ -257,15 +256,6 @@ pub(super) fn is_transient_rebalance_error(err: &Error) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
fn rebalance_error_source(mut err: &Error) -> &Error {
|
||||
// Stage context contains object names, so classify the preserved source,
|
||||
// not timeout-like text supplied by an object name. Iterate nested stages.
|
||||
while let Some(source) = crate::data_movement::data_movement_stage_source(err) {
|
||||
err = source;
|
||||
}
|
||||
err
|
||||
}
|
||||
|
||||
fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
|
||||
match err {
|
||||
rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::Network { .. } => true,
|
||||
@@ -319,7 +309,6 @@ pub(super) fn rebalance_listing_retry_delay(attempt: usize) -> Duration {
|
||||
}
|
||||
|
||||
fn is_rebalance_lock_or_rpc_timeout(err: &Error) -> bool {
|
||||
let err = rebalance_error_source(err);
|
||||
match err {
|
||||
Error::Lock(rustfs_lock::LockError::Timeout { .. }) | Error::Lock(rustfs_lock::LockError::Network { .. }) => true,
|
||||
Error::Io(io_err) => is_rebalance_lock_or_rpc_timeout_message(&io_err.to_string()),
|
||||
@@ -596,48 +585,3 @@ impl SetDisks {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod error_source_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn stage_wrapped_errors_select_the_source_backoff_policy() {
|
||||
let cases = [
|
||||
(
|
||||
Error::Lock(rustfs_lock::LockError::timeout(".rustfs.sys/pool.bin@latest", Duration::from_secs(5))),
|
||||
true,
|
||||
),
|
||||
(
|
||||
Error::Lock(rustfs_lock::LockError::network(
|
||||
"peer unavailable",
|
||||
std::io::Error::from(std::io::ErrorKind::ConnectionReset),
|
||||
)),
|
||||
true,
|
||||
),
|
||||
(Error::other("remote lock rpc timed out"), true),
|
||||
(Error::SlowDown, false),
|
||||
(Error::Io(std::io::Error::other(DiskError::Timeout)), false),
|
||||
(Error::FileAccessDenied, false),
|
||||
];
|
||||
for (mut error, lock_backoff) in cases {
|
||||
for depth in 0..=3 {
|
||||
assert_eq!(
|
||||
is_rebalance_lock_or_rpc_timeout(&error),
|
||||
lock_backoff,
|
||||
"wrong backoff at depth {depth}: {error:?}"
|
||||
);
|
||||
if !lock_backoff {
|
||||
assert_eq!(rebalance_migration_retry_delay(1, &error), REBALANCE_MIGRATION_RETRY_BASE_DELAY * 2);
|
||||
}
|
||||
error = crate::data_movement::data_movement_stage_error_for_test(
|
||||
"rebalance_object",
|
||||
"put_object",
|
||||
"bucket",
|
||||
"remote lock rpc timed out",
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -653,26 +653,6 @@ impl MockWarmBackend {
|
||||
|
||||
#[async_trait]
|
||||
impl WarmBackend for MockWarmBackend {
|
||||
async fn probe_legacy_metadata(
|
||||
&self,
|
||||
object: &str,
|
||||
remote_version: Option<&str>,
|
||||
) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
|
||||
use super::warm_backend::LegacyTransitionStateProbe as Probe;
|
||||
let candidate = match remote_version {
|
||||
Some(version) if !version.is_empty() => self.probe_transition_version(object, version).await?,
|
||||
_ => self.probe_transition_candidate(object).await?,
|
||||
};
|
||||
Ok(match candidate {
|
||||
TransitionCandidateProbe::Missing => Probe::Missing,
|
||||
TransitionCandidateProbe::UnversionedPresent => Probe::UnversionedPresent,
|
||||
TransitionCandidateProbe::VersionedPresent(version) if version == "null" => Probe::SuspendedNullPresent,
|
||||
TransitionCandidateProbe::VersionedPresent(version) => Probe::VersionedPresent(version),
|
||||
TransitionCandidateProbe::Ambiguous => Probe::Ambiguous,
|
||||
TransitionCandidateProbe::Unsupported => Probe::Unsupported,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
|
||||
if remote_version_id.is_empty() {
|
||||
return Ok(());
|
||||
@@ -894,9 +874,8 @@ pub async fn register_mock_tier_backend(handle: &Arc<RwLock<TierConfigMgr>>, tie
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
drop(tier_config_mgr);
|
||||
TierConfigMgr::install_test_driver_in(handle, tier_name, Box::new(backend))
|
||||
.await
|
||||
tier_config_mgr
|
||||
.install_test_driver(tier_name, Box::new(backend))
|
||||
.expect("mock tier driver should install");
|
||||
}
|
||||
|
||||
|
||||
@@ -2322,14 +2322,6 @@ struct SharedWarmBackendProxy(SharedWarmBackend);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for SharedWarmBackendProxy {
|
||||
async fn probe_legacy_metadata(
|
||||
&self,
|
||||
object: &str,
|
||||
remote_version: Option<&str>,
|
||||
) -> io::Result<crate::services::tier::warm_backend::LegacyTransitionStateProbe> {
|
||||
self.0.probe_legacy_metadata(object, remote_version).await
|
||||
}
|
||||
|
||||
async fn validate(&self) -> io::Result<()> {
|
||||
self.0.validate().await
|
||||
}
|
||||
@@ -2498,27 +2490,6 @@ impl TierOperationLease {
|
||||
self.inner.driver.probe_transition_version(object, remote_version_id).await
|
||||
}
|
||||
|
||||
pub(crate) async fn probe_legacy_transition_state(
|
||||
&self,
|
||||
object: &str,
|
||||
remote_version: Option<&str>,
|
||||
) -> io::Result<crate::services::tier::warm_backend::LegacyTransitionStateProbe> {
|
||||
let Some(reconciler) = self
|
||||
.inner
|
||||
.reconciler
|
||||
.get_or_try_init(|| async {
|
||||
crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config)
|
||||
.await
|
||||
.map(|reconciler| reconciler.map(Arc::from))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| io::Error::other(err.message))?
|
||||
else {
|
||||
return self.inner.driver.probe_legacy_metadata(object, remote_version).await;
|
||||
};
|
||||
reconciler.probe_legacy_transition_state(object, remote_version).await
|
||||
}
|
||||
|
||||
pub(crate) fn is_current_generation(&self) -> bool {
|
||||
lock_unpoisoned(&self.runtime)
|
||||
.generations
|
||||
@@ -6042,19 +6013,6 @@ impl TierConfigMgr {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(crate) async fn install_test_driver_in(
|
||||
handle: &Arc<RwLock<Self>>,
|
||||
tier_name: &str,
|
||||
driver: WarmBackendImpl,
|
||||
) -> std::result::Result<(), AdminError> {
|
||||
let mut manager = handle.write().await;
|
||||
// Register the generation runtime before installing the mock so its
|
||||
// explicit lack of a network reconciler survives the first lease.
|
||||
tier_driver_runtime(handle, &manager);
|
||||
manager.install_test_driver(tier_name, driver)
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(crate) fn install_test_driver(
|
||||
&mut self,
|
||||
|
||||
@@ -89,18 +89,6 @@ pub enum TransitionCandidateProbe {
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
/// Live evidence for repairing legacy metadata. Ordinary candidate GETs do not
|
||||
/// establish the bucket's versioning model and cannot supply this authority.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum LegacyTransitionStateProbe {
|
||||
Missing,
|
||||
UnversionedPresent,
|
||||
SuspendedNullPresent,
|
||||
VersionedPresent(String),
|
||||
Ambiguous,
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) struct TransitionCandidateIdentity {
|
||||
pub transaction_id: uuid::Uuid,
|
||||
@@ -109,14 +97,6 @@ pub(crate) struct TransitionCandidateIdentity {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait TransitionCandidateReconciler {
|
||||
async fn probe_legacy_transition_state(
|
||||
&self,
|
||||
_object: &str,
|
||||
_remote_version: Option<&str>,
|
||||
) -> Result<LegacyTransitionStateProbe, std::io::Error> {
|
||||
Ok(LegacyTransitionStateProbe::Unsupported)
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
@@ -126,14 +106,6 @@ pub(crate) trait TransitionCandidateReconciler {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait WarmBackend {
|
||||
async fn probe_legacy_metadata(
|
||||
&self,
|
||||
_object: &str,
|
||||
_remote_version: Option<&str>,
|
||||
) -> Result<LegacyTransitionStateProbe, std::io::Error> {
|
||||
Ok(LegacyTransitionStateProbe::Unsupported)
|
||||
}
|
||||
|
||||
async fn validate(&self) -> Result<(), std::io::Error> {
|
||||
Ok(())
|
||||
}
|
||||
@@ -476,18 +448,6 @@ impl MeteredWarmBackend {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl WarmBackend for MeteredWarmBackend {
|
||||
async fn probe_legacy_metadata(
|
||||
&self,
|
||||
object: &str,
|
||||
remote_version: Option<&str>,
|
||||
) -> Result<LegacyTransitionStateProbe, std::io::Error> {
|
||||
let result = self.inner.probe_legacy_metadata(object, remote_version).await;
|
||||
if matches!(result, Ok(LegacyTransitionStateProbe::Unsupported)) {
|
||||
return result;
|
||||
}
|
||||
Self::record(TierRequestOperation::Probe, result)
|
||||
}
|
||||
|
||||
/// Delegated without a counter: only one backend issues a remote request
|
||||
/// here, and every other one takes the trait default, so a `validate`
|
||||
/// counter would mostly record requests that never happened.
|
||||
@@ -564,18 +524,6 @@ struct MeteredTransitionCandidateReconciler {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TransitionCandidateReconciler for MeteredTransitionCandidateReconciler {
|
||||
async fn probe_legacy_transition_state(
|
||||
&self,
|
||||
object: &str,
|
||||
remote_version: Option<&str>,
|
||||
) -> Result<LegacyTransitionStateProbe, std::io::Error> {
|
||||
let result = self.inner.probe_legacy_transition_state(object, remote_version).await;
|
||||
if matches!(result, Ok(LegacyTransitionStateProbe::Unsupported)) {
|
||||
return result;
|
||||
}
|
||||
MeteredWarmBackend::record(TierRequestOperation::Probe, result)
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
|
||||
@@ -101,19 +101,6 @@ impl WarmBackend for WarmBackendMinIO {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendMinIO {
|
||||
async fn probe_legacy_transition_state(
|
||||
&self,
|
||||
object: &str,
|
||||
remote_version: Option<&str>,
|
||||
) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_legacy_transition_state(
|
||||
&self.0,
|
||||
object,
|
||||
remote_version,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
|
||||
@@ -146,19 +146,6 @@ impl WarmBackend for WarmBackendRustFS {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendRustFS {
|
||||
async fn probe_legacy_transition_state(
|
||||
&self,
|
||||
object: &str,
|
||||
remote_version: Option<&str>,
|
||||
) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
|
||||
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_legacy_transition_state(
|
||||
&self.0,
|
||||
object,
|
||||
remote_version,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
|
||||
@@ -376,6 +376,7 @@ struct TransitionCandidateVersions {
|
||||
}
|
||||
|
||||
impl TransitionCandidateVersions {
|
||||
#[cfg(test)]
|
||||
fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
|
||||
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
|
||||
if self.version_id.is_some() {
|
||||
@@ -511,20 +512,6 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn candidate_probe_fixture() -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
|
||||
scripted_probe_fixture([
|
||||
"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx",
|
||||
"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\n<Error><Code>InvalidRange</Code><Message>empty version</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\n<Error><Code>NoSuchVersion</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
|
||||
].into_iter().map(str::to_owned).collect()).await
|
||||
}
|
||||
|
||||
async fn scripted_probe_fixture(responses: Vec<String>) -> Option<(WarmBackendS3, tokio::task::JoinHandle<Vec<String>>)> {
|
||||
let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await {
|
||||
Ok(listener) => listener,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
|
||||
@@ -535,6 +522,17 @@ mod tests {
|
||||
.expect("listener local address should be available")
|
||||
.to_string();
|
||||
let fixture = tokio::spawn(async move {
|
||||
let responses = [
|
||||
"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: opaque-version\r\nConnection: close\r\n\r\nx",
|
||||
"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nConnection: close\r\n\r\nx",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 66\r\nConnection: close\r\n\r\n<Error><Code>NoSuchObject</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 403 Forbidden\r\nContent-Type: application/xml\r\nContent-Length: 65\r\nConnection: close\r\n\r\n<Error><Code>AccessDenied</Code><Message>denied</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 416 Range Not Satisfiable\r\nContent-Type: application/xml\r\nContent-Length: 72\r\nConnection: close\r\n\r\n<Error><Code>InvalidRange</Code><Message>empty version</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 67\r\nConnection: close\r\n\r\n<Error><Code>NoSuchVersion</Code><Message>missing</Message></Error>",
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Type: application/xml\r\nContent-Length: 63\r\nConnection: close\r\n\r\n<Error><Code>NoSuchKey</Code><Message>missing</Message></Error>",
|
||||
];
|
||||
let mut requests = Vec::new();
|
||||
for response in responses {
|
||||
let (mut stream, _) = listener.accept().await.expect("fixture should accept candidate GET");
|
||||
@@ -675,117 +673,6 @@ mod tests {
|
||||
assert!(requests[8].to_ascii_lowercase().contains("?versionid=historical-version"));
|
||||
}
|
||||
|
||||
fn legacy_probe_xml_response(body: &str) -> String {
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/xml\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
)
|
||||
}
|
||||
|
||||
fn legacy_probe_versioning_response(status: &str) -> String {
|
||||
let state = if status.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("<Status>{status}</Status>")
|
||||
};
|
||||
legacy_probe_xml_response(&format!(
|
||||
"<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">{state}</VersioningConfiguration>"
|
||||
))
|
||||
}
|
||||
|
||||
fn legacy_probe_versions_response(versions: &[&str]) -> String {
|
||||
let versions = versions.iter().map(|version| format!(
|
||||
"<Version><Key>archive/object</Key><VersionId>{version}</VersionId><IsLatest>true</IsLatest><LastModified>2026-09-01T00:00:00Z</LastModified><ETag>\"legacy-etag\"</ETag><Size>7</Size><StorageClass>STANDARD</StorageClass></Version>"
|
||||
)).collect::<String>();
|
||||
legacy_probe_xml_response(&format!(
|
||||
"<ListVersionsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>bucket</Name><Prefix>archive/object</Prefix><KeyMarker/><VersionIdMarker/><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>{versions}</ListVersionsResult>"
|
||||
))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_transition_state_probe_verifies_disabled_suspended_and_enabled_responses() {
|
||||
use super::super::warm_backend::LegacyTransitionStateProbe as Probe;
|
||||
for (initial, confirmed, version, expected) in [
|
||||
("", "", "null", Probe::UnversionedPresent),
|
||||
("Suspended", "Suspended", "null", Probe::SuspendedNullPresent),
|
||||
("Enabled", "Enabled", "version-a", Probe::VersionedPresent("version-a".to_string())),
|
||||
("Enabled", "Enabled", "null", Probe::SuspendedNullPresent),
|
||||
("", "", "unexpected-version", Probe::Ambiguous),
|
||||
("Suspended", "Enabled", "null", Probe::Ambiguous),
|
||||
] {
|
||||
let responses = vec![
|
||||
legacy_probe_versioning_response(initial),
|
||||
legacy_probe_versions_response(&[version]),
|
||||
legacy_probe_versioning_response(confirmed),
|
||||
];
|
||||
let (backend, fixture) = scripted_probe_fixture(responses)
|
||||
.await
|
||||
.expect("legacy probe loopback fixture");
|
||||
let result =
|
||||
tokio::time::timeout(Duration::from_secs(10), backend.probe_legacy_transition_state("archive/object", None))
|
||||
.await
|
||||
.expect("legacy probe must finish")
|
||||
.expect("legacy probe should decode provider XML");
|
||||
assert_eq!(result, expected, "initial={initial} confirmed={confirmed} version={version}");
|
||||
let requests = fixture.await.expect("legacy probe fixture should finish");
|
||||
assert_eq!(requests.len(), 3);
|
||||
assert!(requests.iter().all(|request| request.starts_with("GET ")));
|
||||
assert!(requests[0].lines().next().expect("request line").contains("versioning"));
|
||||
assert!(requests[1].lines().next().expect("request line").contains("versions"));
|
||||
assert!(requests[2].lines().next().expect("request line").contains("versioning"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_transition_state_probe_preserves_historical_exact_version() {
|
||||
use super::super::warm_backend::LegacyTransitionStateProbe as Probe;
|
||||
let responses = vec![
|
||||
legacy_probe_versioning_response("Enabled"),
|
||||
"HTTP/1.1 206 Partial Content\r\nContent-Length: 1\r\nx-amz-version-id: historical-version\r\nConnection: close\r\n\r\nx".to_string(),
|
||||
legacy_probe_versioning_response("Enabled"),
|
||||
];
|
||||
let (backend, fixture) = scripted_probe_fixture(responses).await.expect("exact legacy probe fixture");
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_legacy_transition_state("archive/object", Some("historical-version"))
|
||||
.await
|
||||
.expect("exact version proof"),
|
||||
Probe::VersionedPresent("historical-version".to_string())
|
||||
);
|
||||
let requests = fixture.await.expect("exact probe fixture should finish");
|
||||
assert!(
|
||||
requests[1]
|
||||
.lines()
|
||||
.next()
|
||||
.expect("request line")
|
||||
.contains("versionId=historical-version")
|
||||
);
|
||||
assert!(requests[1].to_ascii_lowercase().contains("range: bytes=0-0"));
|
||||
assert!(requests.iter().all(|request| request.starts_with("GET ")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_transition_state_probe_retains_multiple_candidates() {
|
||||
use super::super::warm_backend::LegacyTransitionStateProbe as Probe;
|
||||
let responses = vec![
|
||||
legacy_probe_versioning_response("Enabled"),
|
||||
legacy_probe_versions_response(&["version-a", "version-b"]),
|
||||
];
|
||||
let (backend, fixture) = scripted_probe_fixture(responses)
|
||||
.await
|
||||
.expect("ambiguous legacy probe fixture");
|
||||
assert_eq!(
|
||||
backend
|
||||
.probe_legacy_transition_state("archive/object", None)
|
||||
.await
|
||||
.expect("ambiguous proof"),
|
||||
Probe::Ambiguous
|
||||
);
|
||||
let requests = fixture.await.expect("ambiguous fixture should finish");
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert!(requests.iter().all(|request| request.starts_with("GET ")));
|
||||
}
|
||||
|
||||
fn list_versions(versions: &[(&str, &str)], delete_markers: &[(&str, &str)], is_truncated: bool) -> ListVersionsResult {
|
||||
ListVersionsResult {
|
||||
versions: versions
|
||||
@@ -1034,56 +921,6 @@ impl WarmBackend for WarmBackendS3 {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TransitionCandidateReconciler for WarmBackendS3 {
|
||||
async fn probe_legacy_transition_state(
|
||||
&self,
|
||||
object: &str,
|
||||
remote_version: Option<&str>,
|
||||
) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
|
||||
use super::warm_backend::LegacyTransitionStateProbe as Probe;
|
||||
|
||||
let initial_versioning = self.remote_bucket_versioning().await?;
|
||||
let candidate = if let Some(version) = remote_version.filter(|version| !version.is_empty()) {
|
||||
validate_remote_version_id(version)?;
|
||||
match self.probe_transition_version(object, version).await? {
|
||||
TransitionCandidateProbe::VersionedPresent(actual) if actual == version => Some(actual),
|
||||
TransitionCandidateProbe::Missing => return Ok(Probe::Missing),
|
||||
_ => return Ok(Probe::Ambiguous),
|
||||
}
|
||||
} else {
|
||||
let remote_object = self.get_dest(object);
|
||||
let mut opts = ListObjectsOptions::default();
|
||||
opts.set("prefix", &remote_object);
|
||||
opts.set("max-keys", "1000");
|
||||
let mut key_marker = String::new();
|
||||
let mut version_marker = String::new();
|
||||
let mut candidates = TransitionCandidateVersions::default();
|
||||
let mut complete = false;
|
||||
// This is one synchronous record inspection, not an unbounded
|
||||
// remote history scan. The caller also bounds the whole probe.
|
||||
for _ in 0..128 {
|
||||
let page = self
|
||||
.client
|
||||
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_marker, "")
|
||||
.await?;
|
||||
candidates.extend(&remote_object, &page);
|
||||
if candidates.ambiguous {
|
||||
return Ok(Probe::Ambiguous);
|
||||
}
|
||||
if !page.is_truncated {
|
||||
complete = true;
|
||||
break;
|
||||
}
|
||||
advance_version_markers(&mut key_marker, &mut version_marker, &page)?;
|
||||
}
|
||||
if !complete {
|
||||
return Ok(Probe::Ambiguous);
|
||||
}
|
||||
candidates.version_id
|
||||
};
|
||||
let confirmed_versioning = self.remote_bucket_versioning().await?;
|
||||
classify_legacy_transition_state(candidate.as_deref(), initial_versioning, confirmed_versioning)
|
||||
}
|
||||
|
||||
async fn probe_transition_candidate_for(
|
||||
&self,
|
||||
object: &str,
|
||||
@@ -1094,32 +931,3 @@ impl TransitionCandidateReconciler for WarmBackendS3 {
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_legacy_transition_state(
|
||||
candidate: Option<&str>,
|
||||
initial: RemoteBucketVersioning,
|
||||
confirmed: RemoteBucketVersioning,
|
||||
) -> Result<super::warm_backend::LegacyTransitionStateProbe, std::io::Error> {
|
||||
use super::warm_backend::LegacyTransitionStateProbe as Probe;
|
||||
if initial != confirmed {
|
||||
return Ok(Probe::Ambiguous);
|
||||
}
|
||||
let Some(version) = candidate else {
|
||||
return Ok(Probe::Missing);
|
||||
};
|
||||
if !version.is_empty() {
|
||||
validate_remote_version_id(version)?;
|
||||
if uuid::Uuid::parse_str(version).is_ok_and(|id| id.is_nil()) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"legacy tier probe returned a nil version identifier",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(match (confirmed, version) {
|
||||
(RemoteBucketVersioning::Disabled, "" | "null") => Probe::UnversionedPresent,
|
||||
(RemoteBucketVersioning::Disabled, _) | (_, "") => Probe::Ambiguous,
|
||||
(_, "null") => Probe::SuspendedNullPresent,
|
||||
(_, version) => Probe::VersionedPresent(version.to_string()),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -51,12 +51,12 @@ use super::super::{
|
||||
can_try_inline_data_shards_direct, capacity_scope_from_disks, codec_streaming_rollout_applies, coding,
|
||||
collect_inline_data_shard_fileinfos_by_index_or_reason, current_dirty_generation, debug, disk,
|
||||
file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info, inline_erasure_shard_file_offset,
|
||||
inline_erasure_shard_size, is_get_metadata_data_read_early_stop_enabled, is_get_metadata_early_stop_bounded_fanout_enabled,
|
||||
is_get_metadata_early_stop_enabled, is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling,
|
||||
is_version_early_stop_enabled, issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure,
|
||||
merge_file_meta_versions, object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs,
|
||||
reduce_write_quorum_errs, send_heal_request_with_admission, should_prevent_write, to_object_err,
|
||||
try_read_inline_data_shards_direct, warn,
|
||||
inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found, is_get_metadata_data_read_early_stop_enabled,
|
||||
is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled,
|
||||
is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling, is_version_early_stop_enabled,
|
||||
issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure, merge_file_meta_versions,
|
||||
object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs, reduce_write_quorum_errs,
|
||||
send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(in crate::set_disk) use super::metadata_quorum::MetadataEarlyStopDecision;
|
||||
@@ -3086,61 +3086,21 @@ impl SetDisks {
|
||||
let read_quorum = disks.len().div_ceil(2).max(1);
|
||||
let (raw_fileinfos, errs) = Self::read_all_raw_file_info(&disks, bucket, disk_object.as_str(), false).await;
|
||||
|
||||
if let Some(err) = errs
|
||||
.iter()
|
||||
.flatten()
|
||||
.find(|err| !matches!(err, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound))
|
||||
{
|
||||
return Err(to_object_err(err.clone().into(), vec![bucket, object]));
|
||||
}
|
||||
// A minority live owner must not disappear behind majority absence.
|
||||
// Only explicit absence on every readable disk proves no ownership.
|
||||
if raw_fileinfos.iter().all(Option::is_none) {
|
||||
return Ok(None);
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
let object_err = to_object_err(err.into(), vec![bucket, object]);
|
||||
if is_err_object_not_found(&object_err) || is_err_version_not_found(&object_err) {
|
||||
return Ok(None);
|
||||
}
|
||||
return Err(object_err);
|
||||
}
|
||||
|
||||
let mut shallow_versions = Vec::with_capacity(raw_fileinfos.len());
|
||||
type TransitionCopy = (FileInfo, Option<crate::services::tier::tier::TierDestinationId>);
|
||||
let mut transition_copies: std::collections::HashMap<Option<Uuid>, Vec<TransitionCopy>> =
|
||||
std::collections::HashMap::new();
|
||||
let decode_error = |err| Error::other(format!("exact object versions decode failed for {bucket}/{object}: {err}"));
|
||||
for raw_fileinfo in raw_fileinfos.into_iter().flatten() {
|
||||
let meta = FileMeta::load(&raw_fileinfo.buf)
|
||||
.map_err(|err| Error::other(format!("exact object metadata decode failed for {bucket}/{object}: {err}")))?;
|
||||
let versions = meta.get_all_file_info_versions(bucket, object, true).map_err(decode_error)?;
|
||||
for version in versions.versions.into_iter().chain(versions.free_versions) {
|
||||
if version.transition_status != rustfs_filemeta::TRANSITION_COMPLETE {
|
||||
continue;
|
||||
}
|
||||
let destination =
|
||||
crate::services::tier::tier::tier_destination_id_from_metadata(&version.metadata).map_err(Error::other)?;
|
||||
transition_copies
|
||||
.entry(version.version_id.filter(|id| !id.is_nil()))
|
||||
.or_default()
|
||||
.push((version, destination));
|
||||
}
|
||||
shallow_versions.push(meta.versions);
|
||||
}
|
||||
|
||||
// Exact cleanup/recovery reads must not select a repaired majority
|
||||
// while another physical copy still carries legacy absence. Missing
|
||||
// copies permit deletion retries; an unreadable disk proves nothing.
|
||||
for copies in transition_copies
|
||||
.values()
|
||||
.filter(|copies| copies.iter().any(|(_, destination)| destination.is_some()))
|
||||
{
|
||||
let (first, destination) = &copies[0];
|
||||
if copies.iter().any(|(copy, identity)| {
|
||||
identity != destination
|
||||
|| copy.transition_version_state != first.transition_version_state
|
||||
|| copy.transition_version != first.transition_version
|
||||
|| copy.transition_tier != first.transition_tier
|
||||
|| copy.transitioned_objname != first.transitioned_objname
|
||||
}) {
|
||||
return Err(Error::other("exact transition metadata has not converged across physical copies"));
|
||||
}
|
||||
}
|
||||
|
||||
if shallow_versions.len() < read_quorum {
|
||||
return Err(to_object_err(StorageError::ErasureReadQuorum, vec![bucket, object]));
|
||||
}
|
||||
@@ -3157,7 +3117,7 @@ impl SetDisks {
|
||||
..Default::default()
|
||||
}
|
||||
.get_all_file_info_versions(bucket, object, true)
|
||||
.map_err(decode_error)?;
|
||||
.map_err(|err| Error::other(format!("exact object versions decode failed for {bucket}/{object}: {err}")))?;
|
||||
|
||||
for file_info in file_info_versions
|
||||
.versions
|
||||
@@ -5855,17 +5815,17 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
if let Some(disk) = disks[i].as_ref() {
|
||||
// A failed version-only copy owns only its new version.
|
||||
// Removing the whole xl.meta would also erase existing
|
||||
// versions and any concurrently reconciled tier binding.
|
||||
let mut rollback = FileInfo {
|
||||
version_id: files[i].version_id,
|
||||
..Default::default()
|
||||
};
|
||||
rollback.set_skip_tier_free_version();
|
||||
let path = path_join_buf(&[prefix, STORAGE_FORMAT_FILE]);
|
||||
revert_futures.push(async move {
|
||||
if let Err(err) = disk
|
||||
.delete_version(bucket, prefix, rollback, false, DeleteOptions::default())
|
||||
.delete(
|
||||
bucket,
|
||||
&path,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!("write meta revert err {:?}", err);
|
||||
@@ -12132,9 +12092,7 @@ mod tests {
|
||||
let bucket = "write-unique-bucket";
|
||||
let object = "object";
|
||||
let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
|
||||
let mut fi = metadata_test_fileinfo(object);
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
let files = vec![fi.clone(), fi];
|
||||
let files = vec![metadata_test_fileinfo(object), metadata_test_fileinfo(object)];
|
||||
|
||||
let result = SetDisks::write_unique_file_info(&[Some(disk.clone()), None], bucket, bucket, object, &files, 2).await;
|
||||
|
||||
@@ -12148,53 +12106,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_unique_file_info_rollback_preserves_existing_reconciled_version() {
|
||||
let bucket = "write-unique-existing";
|
||||
let object = "object";
|
||||
let (_dir, disk) = read_multiple_test_disk(bucket, &[]).await;
|
||||
let mut original = metadata_test_fileinfo(object);
|
||||
original.version_id = Some(Uuid::from_u128(1));
|
||||
original.data_dir = Some(Uuid::from_u128(3));
|
||||
original.mod_time = Some(OffsetDateTime::now_utc());
|
||||
original.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_string();
|
||||
original.transition_tier = "WARM".to_string();
|
||||
original.transitioned_objname = "remote-original".to_string();
|
||||
original.transition_version_state = rustfs_filemeta::TransitionVersionState::KnownDisabled;
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut original.metadata,
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
"ab".repeat(32),
|
||||
);
|
||||
disk.write_metadata(bucket, bucket, object, original.clone())
|
||||
.await
|
||||
.expect("existing reconciled source");
|
||||
let raw = disk
|
||||
.read_all(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
|
||||
.await
|
||||
.expect("original metadata");
|
||||
let before = FileMeta::load(&raw).unwrap().find_version(original.version_id).unwrap().1;
|
||||
let mut added = original.clone();
|
||||
added.version_id = Some(Uuid::from_u128(2));
|
||||
let result =
|
||||
SetDisks::write_unique_file_info(&[Some(disk.clone()), None], bucket, bucket, object, &[added.clone(), added], 2)
|
||||
.await;
|
||||
assert!(result.is_err());
|
||||
let raw = disk
|
||||
.read_all(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
|
||||
.await
|
||||
.expect("preserved xl.meta");
|
||||
let after = FileMeta::load(&raw).expect("preserved metadata");
|
||||
assert_eq!(
|
||||
after
|
||||
.find_version(original.version_id)
|
||||
.expect("original source survives rollback")
|
||||
.1,
|
||||
before
|
||||
);
|
||||
assert!(after.find_version(Some(Uuid::from_u128(2))).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_object_meta_handles_empty_metadata_and_missing_quorum() {
|
||||
let set = io_primitives_test_set(vec![None, None], 1).await;
|
||||
|
||||
@@ -326,15 +326,14 @@ impl SetDisks {
|
||||
let parity_blocks = Self::common_parity(&parities, default_parity_count as i32);
|
||||
|
||||
if parity_blocks < 0 {
|
||||
// A consistent layout can require more replies than the initial
|
||||
// half-set probe. Reaching that probe alone is not corruption;
|
||||
// only invalid or conflicting healthy replies establish that.
|
||||
// No parity value reached read quorum. Distinguish two cases:
|
||||
// enough disks answered with valid-looking metadata that simply
|
||||
// cannot be reconciled (corrupt/foreign entries — retrying cannot
|
||||
// help, and heal should see Corrupt, rustfs#5801) versus too few
|
||||
// healthy answers (a genuine quorum condition where retry may
|
||||
// succeed once disks recover).
|
||||
let healthy_replies = errs.iter().filter(|err| err.is_none()).count();
|
||||
let consistent_parity = parities
|
||||
.iter()
|
||||
.find(|&&parity| parity >= 0)
|
||||
.filter(|&&parity| parities.iter().filter(|&&candidate| candidate == parity).count() == healthy_replies);
|
||||
if healthy_replies >= expected_rquorum && consistent_parity.is_none() {
|
||||
if healthy_replies >= expected_rquorum {
|
||||
error!(
|
||||
"object_quorum_from_meta: irreconcilable parity across {healthy_replies} healthy replies (corrupt metadata), errs={errs:?}"
|
||||
);
|
||||
@@ -1653,40 +1652,6 @@ mod tests {
|
||||
assert_eq!(err, DiskError::FileCorrupt);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consistent_parity_below_its_data_shard_quorum_is_not_corruption() {
|
||||
for (drive_count, parity) in [(6, 2), (8, 2), (12, 4)] {
|
||||
let data = drive_count - parity;
|
||||
let mut metas = (1..=drive_count)
|
||||
.map(|index| {
|
||||
let mut info = FileInfo::new("bucket/object", data, parity);
|
||||
info.size = 1024;
|
||||
info.erasure.index = index;
|
||||
info
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut errs = vec![Some(DiskError::DiskNotFound); drive_count];
|
||||
errs[..data].fill(None);
|
||||
assert_eq!(
|
||||
SetDisks::object_quorum_from_meta(&metas, &errs, parity).expect("exact data quorum should resolve"),
|
||||
(data as i32, data as i32)
|
||||
);
|
||||
|
||||
errs[data - 1] = Some(DiskError::DiskNotFound);
|
||||
assert_eq!(
|
||||
SetDisks::object_quorum_from_meta(&metas, &errs, parity).expect_err("one fewer shard cannot resolve"),
|
||||
DiskError::ErasureReadQuorum,
|
||||
"layout {drive_count}/{parity} has consistent metadata but insufficient shards"
|
||||
);
|
||||
|
||||
metas[0].erasure.parity_blocks = usize::MAX;
|
||||
assert_eq!(
|
||||
SetDisks::object_quorum_from_meta(&metas, &errs, parity).expect_err("corrupt healthy replies must be rejected"),
|
||||
DiskError::FileCorrupt
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Too few healthy replies remains a genuine quorum condition where a
|
||||
/// retry may succeed once disks recover.
|
||||
#[test]
|
||||
|
||||
@@ -865,7 +865,6 @@ pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, ren
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
pub(crate) use ops::bucket::BucketInfoQuorum;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::hermetic_set_disks_isolated;
|
||||
@@ -3889,33 +3888,6 @@ pub struct SetDisks {
|
||||
>,
|
||||
}
|
||||
|
||||
/// Read every physical copy before selecting a version quorum. A minority
|
||||
/// legacy record is still evidence and must not disappear behind a majority
|
||||
/// not-found result. Only an explicit file/volume absence produces `None`;
|
||||
/// an unreadable disk cannot prove that no conflicting copy exists.
|
||||
pub(crate) async fn read_legacy_transition_state_metadata_copies(
|
||||
set: &SetDisks,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> std::result::Result<Vec<Option<Vec<u8>>>, DiskError> {
|
||||
let disk_object = rustfs_utils::path::encode_dir_object(object);
|
||||
let disks = set.get_disks_internal().await;
|
||||
if disks.is_empty() {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
|
||||
// Include inline bytes in the generation: a conditional repair preserves
|
||||
// the entire xl.meta, including payloads belonging to other versions.
|
||||
let (copies, errs) = SetDisks::read_all_raw_file_info(&disks, bucket, disk_object.as_str(), true).await;
|
||||
for err in errs.into_iter().flatten() {
|
||||
if !matches!(err, DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound) {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(copies.into_iter().map(|copy| copy.map(|copy| copy.buf)).collect())
|
||||
}
|
||||
|
||||
// DistributedLock sends the raw ObjectKey to its clients; LockRegistry clones
|
||||
// each endpoint's canonical Arc, so an exact Arc set identifies the lock domain.
|
||||
pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
|
||||
@@ -4264,7 +4236,7 @@ impl SetDisks {
|
||||
self.get_object_metadata_cache_generations[generation.index].load(Ordering::Acquire) == generation.value
|
||||
}
|
||||
|
||||
pub(crate) async fn invalidate_get_object_metadata_cache(&self, bucket: &str, object: &str) {
|
||||
async fn invalidate_get_object_metadata_cache(&self, bucket: &str, object: &str) {
|
||||
let hash = self.get_object_metadata_cache_hash(bucket, object);
|
||||
let hash_bytes = hash.to_le_bytes();
|
||||
let index = usize::from(u16::from_le_bytes([hash_bytes[0], hash_bytes[1]]) % GET_OBJECT_METADATA_CACHE_FENCE_SHARDS);
|
||||
|
||||
@@ -21,72 +21,12 @@
|
||||
|
||||
use super::super::{
|
||||
BUCKET_OP_IGNORED_ERRS, BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DiskError, Error, HashMap,
|
||||
MakeBucketOptions, Result, SetDisks, is_reserved_or_invalid_bucket, join_all, reduce_read_quorum_errs,
|
||||
reduce_write_quorum_errs,
|
||||
MakeBucketOptions, Result, SetDisks, is_reserved_or_invalid_bucket, join_all, reduce_write_quorum_errs,
|
||||
};
|
||||
use crate::api::bucket::metadata_sys;
|
||||
use crate::disk::DiskAPI;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) enum BucketInfoQuorum {
|
||||
Read,
|
||||
Write,
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(crate) async fn stat_bucket_with_quorum(&self, bucket: &str, quorum: BucketInfoQuorum) -> Result<BucketInfo> {
|
||||
let disks = self.disk_inventory().await;
|
||||
let disk_count = disks.len();
|
||||
let mut futures = Vec::with_capacity(disk_count);
|
||||
for disk in disks {
|
||||
let bucket = bucket.to_string();
|
||||
futures.push(async move {
|
||||
match disk {
|
||||
Some(disk) => disk.stat_volume(&bucket).await,
|
||||
None => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
let mut infos = Vec::with_capacity(results.len());
|
||||
let mut errs = Vec::with_capacity(results.len());
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(info) => {
|
||||
infos.push(Some(info));
|
||||
errs.push(None);
|
||||
}
|
||||
Err(err) => {
|
||||
infos.push(None);
|
||||
errs.push(Some(err));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let error = match quorum {
|
||||
// Bucket mutations use a majority regardless of object storage
|
||||
// class. A namespace read must intersect that majority; object
|
||||
// readers still enforce the persisted layout's data-shard quorum.
|
||||
BucketInfoQuorum::Read => reduce_read_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count.div_ceil(2).max(1)),
|
||||
BucketInfoQuorum::Write => reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count / 2 + 1),
|
||||
};
|
||||
if let Some(err) = error {
|
||||
return Err(err.into());
|
||||
}
|
||||
|
||||
infos
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.next()
|
||||
.map(|info| BucketInfo {
|
||||
name: info.name,
|
||||
created: info.created,
|
||||
..Default::default()
|
||||
})
|
||||
.ok_or(Error::VolumeNotFound)
|
||||
}
|
||||
|
||||
pub(crate) async fn list_bucket_for_scanner(&self, _opts: &BucketOptions) -> Result<(Vec<BucketInfo>, bool)> {
|
||||
let disks = self.disk_inventory().await;
|
||||
let write_quorum = (disks.len() / 2) + 1;
|
||||
@@ -191,12 +131,59 @@ impl BucketOperations for SetDisks {
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn get_bucket_info(&self, bucket: &str, _opts: &BucketOptions) -> Result<BucketInfo> {
|
||||
let mut info = self.stat_bucket_with_quorum(bucket, BucketInfoQuorum::Write).await?;
|
||||
if let Ok(sys) = metadata_sys::get(bucket).await {
|
||||
info.versioning = sys.versioning();
|
||||
info.object_locking = sys.object_locking();
|
||||
let disks = self.disk_inventory().await;
|
||||
let write_quorum = (disks.len() / 2) + 1;
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
for disk in disks {
|
||||
let bucket = bucket.to_string();
|
||||
futures.push(async move {
|
||||
match disk {
|
||||
Some(disk) => disk.stat_volume(&bucket).await,
|
||||
None => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
});
|
||||
}
|
||||
Ok(info)
|
||||
|
||||
let results = join_all(futures).await;
|
||||
let mut infos = Vec::with_capacity(results.len());
|
||||
let mut errs = Vec::with_capacity(results.len());
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(info) => {
|
||||
infos.push(Some(info));
|
||||
errs.push(None);
|
||||
}
|
||||
Err(err) => {
|
||||
infos.push(None);
|
||||
errs.push(Some(err));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, write_quorum) {
|
||||
return Err(err.into());
|
||||
}
|
||||
|
||||
let mut versioning = false;
|
||||
let mut object_locking = false;
|
||||
if let Ok(sys) = metadata_sys::get(bucket).await {
|
||||
versioning = sys.versioning();
|
||||
object_locking = sys.object_locking();
|
||||
}
|
||||
|
||||
infos
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.next()
|
||||
.map(|info| BucketInfo {
|
||||
name: info.name,
|
||||
created: info.created,
|
||||
versioning,
|
||||
object_locking,
|
||||
..Default::default()
|
||||
})
|
||||
.ok_or(Error::VolumeNotFound)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
|
||||
@@ -19,7 +19,7 @@ use crate::bucket::{
|
||||
};
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::set_disk::{BucketInfoQuorum, get_lock_acquire_timeout};
|
||||
use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use futures::stream::{self, StreamExt};
|
||||
@@ -770,57 +770,19 @@ impl ECStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Prove a live bucket generation before repairing missing expansion volumes.
|
||||
/// Unlike request validation, repair only needs one erasure set to confirm
|
||||
/// existence; an incomplete expansion set is precisely what repair fixes.
|
||||
/// Callers must hold the bucket namespace lock through the subsequent heal.
|
||||
pub(crate) async fn bucket_exists_for_heal(&self, bucket: &str) -> Result<bool> {
|
||||
let results = futures::future::join_all(
|
||||
self.bucket_sets()
|
||||
.map(|(_, _, set)| async move { set.get_bucket_info(bucket, &BucketOptions::default()).await }),
|
||||
)
|
||||
.await;
|
||||
let mut first_error = None;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(_) => return Ok(true),
|
||||
Err(err) if is_err_strict_volume_not_found(&err) => {}
|
||||
Err(err) if first_error.is_none() => first_error = Some(err),
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
match first_error {
|
||||
Some(err) => Err(err),
|
||||
None => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(crate) async fn get_bucket_info_from_sets(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
|
||||
self.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Write)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_bucket_info_from_sets_with_quorum(
|
||||
&self,
|
||||
bucket: &str,
|
||||
opts: &BucketOptions,
|
||||
quorum: BucketInfoQuorum,
|
||||
) -> Result<BucketInfo> {
|
||||
// One host may participate in several pools after expansion. Resolve the
|
||||
// namespace against each erasure set so disks from different pools can
|
||||
// never be combined into one bucket quorum.
|
||||
// Bucket validation is request-path IO. Keep the previous peer fanout's
|
||||
// latency shape by probing every set concurrently; scanner listings use
|
||||
// a separate bounded path below because they run continuously.
|
||||
let mut scoped_results = futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
|
||||
let result = match quorum {
|
||||
BucketInfoQuorum::Read => set.stat_bucket_with_quorum(bucket, quorum).await,
|
||||
BucketInfoQuorum::Write => set.get_bucket_info(bucket, opts).await,
|
||||
};
|
||||
(pool_index, set_index, result)
|
||||
}))
|
||||
.await;
|
||||
let mut scoped_results =
|
||||
futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
|
||||
(pool_index, set_index, set.get_bucket_info(bucket, opts).await)
|
||||
}))
|
||||
.await;
|
||||
scoped_results.sort_unstable_by_key(|(pool_index, set_index, _)| (*pool_index, *set_index));
|
||||
|
||||
let mut first_info = None;
|
||||
@@ -844,11 +806,7 @@ impl ECStore {
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
|
||||
let mut info = match self.get_bucket_info_from_sets(bucket, opts).await {
|
||||
Ok(info) => info,
|
||||
Err(Error::ErasureWriteQuorum) => return self.get_bucket_info_at_read_quorum(bucket, opts).await,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let mut info = self.get_bucket_info_from_sets(bucket, opts).await?;
|
||||
|
||||
if let Ok(sys) = metadata_sys::get_in(&self.ctx, bucket).await {
|
||||
if should_override_created_from_metadata(sys.created) {
|
||||
@@ -861,35 +819,6 @@ impl ECStore {
|
||||
Ok(info)
|
||||
}
|
||||
|
||||
async fn get_bucket_info_at_read_quorum(&self, bucket: &str, opts: &BucketOptions) -> Result<BucketInfo> {
|
||||
// Lock order: bucket lifecycle -> internal metadata object read locks.
|
||||
// Keep create/delete from changing the namespace while a read quorum
|
||||
// confirms both physical presence and persisted bucket metadata.
|
||||
let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
await_bucket_namespace_operation(Some(&guard), bucket, "bucket read quorum validation", async {
|
||||
let mut info = self
|
||||
.get_bucket_info_from_sets_with_quorum(bucket, opts, BucketInfoQuorum::Read)
|
||||
.await?;
|
||||
let (metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await?;
|
||||
if !persisted {
|
||||
// A minority of directories left by failed creation is not an
|
||||
// authoritative bucket. Never turn fabricated defaults into
|
||||
// permission to serve degraded reads.
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
if metadata.name != bucket {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
if should_override_created_from_metadata(metadata.created) {
|
||||
info.created = Some(metadata.created);
|
||||
}
|
||||
info.versioning = metadata.versioning();
|
||||
info.object_locking = metadata.object_locking();
|
||||
Ok(info)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
|
||||
// TODO(backlog): support cached bucket listing via opts.cached
|
||||
@@ -1120,7 +1049,7 @@ mod tests {
|
||||
run_physical_bucket_deletion, scan_metadata_less_residue, scan_metadata_less_residue_with_budget,
|
||||
should_override_created_from_metadata, validate_table_bucket_delete_allowed,
|
||||
};
|
||||
use crate::bucket::metadata::{BucketMetadata, table_bucket_catalog_metadata_prefix};
|
||||
use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
|
||||
use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE};
|
||||
@@ -1147,7 +1076,6 @@ mod tests {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::{Notify, OnceCell};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
@@ -1431,18 +1359,11 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn setup_multi_pool_bucket_test_env() -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
setup_bucket_quorum_test_env(&[4, 4], None).await
|
||||
}
|
||||
|
||||
async fn setup_bucket_quorum_test_env(
|
||||
drives_per_pool: &[usize],
|
||||
standard_parity: Option<usize>,
|
||||
) -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
let temp_dir = tempfile::tempdir().expect("multi-pool bucket test directory should be created");
|
||||
let mut pools = Vec::new();
|
||||
for (pool_index, &drive_count) in drives_per_pool.iter().enumerate() {
|
||||
for pool_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..drive_count {
|
||||
for disk_index in 0..4 {
|
||||
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
|
||||
tokio::fs::create_dir_all(&disk_path)
|
||||
.await
|
||||
@@ -1457,7 +1378,7 @@ mod tests {
|
||||
pools.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: drive_count,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("bucket-test-pool-{pool_index}"),
|
||||
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
|
||||
@@ -1478,12 +1399,9 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("multi-pool ECStore should initialize");
|
||||
let mut storage_class_kvs = rustfs_config::server_config::KVS::new();
|
||||
if let Some(parity) = standard_parity {
|
||||
storage_class_kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), format!("EC:{parity}"));
|
||||
}
|
||||
let storage_class = crate::config::storageclass::lookup_config_for_pools_without_env(&storage_class_kvs, drives_per_pool)
|
||||
.expect("storage class should match every test erasure set");
|
||||
let storage_class =
|
||||
crate::config::storageclass::lookup_config_for_pools_without_env(&rustfs_config::server_config::KVS::new(), &[4, 4])
|
||||
.expect("multi-pool storage class should match both four-disk pools");
|
||||
for pool in &ecstore.pools {
|
||||
for set in &pool.disk_set {
|
||||
set.set_test_storage_class_config(storage_class.clone());
|
||||
@@ -2132,98 +2050,6 @@ mod tests {
|
||||
.expect("metadata initialization should recreate the bucket volume in the new pool");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_metadata_init_repairs_half_created_expansion_pool() {
|
||||
// Check both pool orders: an incomplete set must not hide a later
|
||||
// complete set, and a complete set must not weaken request validation.
|
||||
for complete_pool in 0..2 {
|
||||
let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
|
||||
let bucket = format!("partial-expansion-{}", Uuid::new_v4().simple());
|
||||
for pool_index in 0..2 {
|
||||
let present_disks = if pool_index == complete_pool { 4 } else { 2 };
|
||||
for disk_index in 0..present_disks {
|
||||
tokio::fs::create_dir(
|
||||
temp_dir
|
||||
.path()
|
||||
.join(format!("pool{pool_index}-disk{disk_index}"))
|
||||
.join(&bucket),
|
||||
)
|
||||
.await
|
||||
.expect("fixture bucket volume should be created");
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
ecstore
|
||||
.get_bucket_info_from_sets(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect_err("request validation must reject a half-created expansion set"),
|
||||
StorageError::ErasureWriteQuorum
|
||||
);
|
||||
|
||||
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
|
||||
|
||||
for pool_index in 0..2 {
|
||||
for disk_index in 0..4 {
|
||||
assert!(
|
||||
temp_dir
|
||||
.path()
|
||||
.join(format!("pool{pool_index}-disk{disk_index}"))
|
||||
.join(&bucket)
|
||||
.is_dir(),
|
||||
"metadata initialization must heal every missing expansion volume"
|
||||
);
|
||||
}
|
||||
}
|
||||
ecstore
|
||||
.get_bucket_info_from_sets(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect("strict request validation should succeed after volume repair");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_metadata_init_does_not_combine_partial_set_evidence() {
|
||||
let (temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
|
||||
let bucket = format!("no-quorum-expansion-{}", Uuid::new_v4().simple());
|
||||
for pool_index in 0..2 {
|
||||
for disk_index in 0..2 {
|
||||
tokio::fs::create_dir(
|
||||
temp_dir
|
||||
.path()
|
||||
.join(format!("pool{pool_index}-disk{disk_index}"))
|
||||
.join(&bucket),
|
||||
)
|
||||
.await
|
||||
.expect("fixture bucket volume should be created");
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
ecstore
|
||||
.bucket_exists_for_heal(&bucket)
|
||||
.await
|
||||
.expect_err("repair must require a complete quorum within one set"),
|
||||
StorageError::ErasureWriteQuorum
|
||||
);
|
||||
|
||||
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), vec![bucket.clone()]).await;
|
||||
|
||||
for pool_index in 0..2 {
|
||||
for disk_index in 0..4 {
|
||||
assert_eq!(
|
||||
temp_dir
|
||||
.path()
|
||||
.join(format!("pool{pool_index}-disk{disk_index}"))
|
||||
.join(&bucket)
|
||||
.is_dir(),
|
||||
disk_index < 2,
|
||||
"unproven bucket generations must not recreate missing volumes"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_metadata_init_does_not_recreate_stale_bucket_name() {
|
||||
@@ -2241,218 +2067,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_info_read_quorum_tracks_erasure_layout() {
|
||||
for (drive_count, parity) in [(2, 1), (3, 1), (4, 2), (5, 2), (6, 3), (8, 4), (6, 2), (12, 6)] {
|
||||
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[drive_count], Some(parity)).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("read-quorum-{drive_count}-{parity}");
|
||||
let object = "uncached-object";
|
||||
let body = b"erasure read quorum must follow the persisted layout".repeat(32_768);
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("healthy namespace should accept bucket creation");
|
||||
store
|
||||
.put_object(&bucket, object, &mut PutObjReader::from_vec(body.clone()), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("healthy erasure set should accept the seed object");
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let lock = set
|
||||
.new_ns_lock(&bucket, object)
|
||||
.await
|
||||
.expect("seed namespace lock should resolve");
|
||||
drop(
|
||||
lock.get_write_lock(Duration::from_secs(30))
|
||||
.await
|
||||
.expect("seed physical fanout must finish before taking disks offline"),
|
||||
);
|
||||
if (drive_count, parity) == (6, 3) {
|
||||
let mut kvs = rustfs_config::server_config::KVS::new();
|
||||
kvs.insert(crate::config::storageclass::CLASS_STANDARD.to_string(), "EC:2".to_string());
|
||||
set.set_test_storage_class_config(
|
||||
crate::config::storageclass::lookup_config_for_pools_without_env(&kvs, &[drive_count])
|
||||
.expect("a later storage-class change must not raise old objects' read quorum"),
|
||||
);
|
||||
}
|
||||
|
||||
let offline_indexes = (0..parity).collect::<Vec<_>>();
|
||||
let offline = take_set_disks_offline(&store, set, &offline_indexes).await;
|
||||
let info = store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect("bucket validation must admit the object's exact read quorum");
|
||||
assert_eq!(info.name, bucket);
|
||||
|
||||
let mut reader = store
|
||||
.get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the persisted layout should remain readable at its exact data-shard quorum");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("quorum read should reconstruct the body");
|
||||
assert_eq!(restored, body, "layout {drive_count}/{parity} must retain exact object contents");
|
||||
drop(reader);
|
||||
|
||||
if drive_count - parity == drive_count / 2 {
|
||||
let error = store
|
||||
.get_bucket_info_from_sets(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect_err("bucket mutations must retain their majority namespace check");
|
||||
assert_eq!(error, StorageError::ErasureWriteQuorum);
|
||||
}
|
||||
|
||||
let below_quorum = take_set_disks_offline(&store, set, &[parity]).await;
|
||||
let read = store
|
||||
.get_object_reader(&bucket, object, None, Default::default(), &ObjectOptions::default())
|
||||
.await;
|
||||
match read {
|
||||
Ok(mut reader) => assert!(
|
||||
reader.stream.read_to_end(&mut Vec::new()).await.is_err(),
|
||||
"layout {drive_count}/{parity} must reject fewer than its data-shard quorum"
|
||||
),
|
||||
Err(error) => assert!(
|
||||
matches!(error, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _)),
|
||||
"a missing shard must report read quorum loss, got {error}"
|
||||
),
|
||||
}
|
||||
restore_set_disks(&store, set, below_quorum).await;
|
||||
restore_set_disks(&store, set, offline).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_info_read_quorum_is_scoped_to_each_erasure_set() {
|
||||
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4, 6], None).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "read-quorum-mixed-pools";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("healthy pools should accept bucket creation");
|
||||
|
||||
let first_set = &store.pools[0].disk_set[0];
|
||||
let second_set = &store.pools[1].disk_set[0];
|
||||
let first_offline = take_set_disks_offline(&store, first_set, &[0, 1]).await;
|
||||
let second_offline = take_set_disks_offline(&store, second_set, &[0, 1, 2]).await;
|
||||
store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect("each set independently satisfies its namespace read quorum");
|
||||
|
||||
for (set, extra_disk) in [(first_set, 2), (second_set, 3)] {
|
||||
let extra_offline = take_set_disks_offline(&store, set, &[extra_disk]).await;
|
||||
assert_eq!(
|
||||
store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect_err("another pool must not subsidize a set below its read quorum"),
|
||||
StorageError::ErasureReadQuorum
|
||||
);
|
||||
restore_set_disks(&store, set, extra_offline).await;
|
||||
}
|
||||
restore_set_disks(&store, first_set, first_offline).await;
|
||||
restore_set_disks(&store, second_set, second_offline).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_info_read_quorum_requires_authoritative_metadata() {
|
||||
for state in ["missing", "corrupt", "foreign", "incarnation"] {
|
||||
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("read-quorum-{state}-metadata");
|
||||
let mut metadata = if state == "missing" {
|
||||
store
|
||||
.make_bucket_on_sets(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("simulate directories left before bucket metadata is published");
|
||||
BucketMetadata::new(&bucket)
|
||||
} else {
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("healthy bucket should publish metadata");
|
||||
metadata_sys::get_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("seed metadata should be cached")
|
||||
.as_ref()
|
||||
.clone()
|
||||
};
|
||||
let path = metadata.save_file_path();
|
||||
match state {
|
||||
"corrupt" => crate::config::com::save_config(store.clone(), &path, b"corrupt".to_vec())
|
||||
.await
|
||||
.expect("persist corrupt metadata while the cached copy remains valid"),
|
||||
"foreign" => {
|
||||
metadata.name = "different-bucket".to_string();
|
||||
let mut encoded = vec![1, 0, 1, 0];
|
||||
encoded.extend(metadata.marshal_msg().expect("foreign metadata should encode"));
|
||||
crate::config::com::save_config(store.clone(), &path, encoded)
|
||||
.await
|
||||
.expect("persist metadata for a different bucket at the requested path");
|
||||
}
|
||||
"incarnation" => crate::bucket::metadata::save_bucket_incarnation(store.clone(), &bucket, Uuid::new_v4())
|
||||
.await
|
||||
.expect("persist a different bucket generation"),
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
|
||||
let error = store
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect_err("read admission must not trust residual directories or cached metadata");
|
||||
match state {
|
||||
"missing" => assert_eq!(error, StorageError::ErasureReadQuorum),
|
||||
"foreign" => assert_eq!(error, StorageError::FileCorrupt),
|
||||
"incarnation" => assert!(error.to_string().contains("sidecar does not match bucket metadata")),
|
||||
"corrupt" => assert!(error.to_string().contains("format invalid"), "unexpected corruption error: {error}"),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
restore_set_disks(&store, set, offline).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_info_read_quorum_accepts_persisted_legacy_metadata() {
|
||||
let (_temp_dir, store) = setup_bucket_quorum_test_env(&[4], None).await;
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "interop";
|
||||
store
|
||||
.make_bucket_on_sets(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("legacy bucket directories should exist");
|
||||
let hex = include_str!("../../tests/fixtures/minio/bucket_metadata.blob.hex")
|
||||
.split_whitespace()
|
||||
.collect::<String>();
|
||||
let body = (0..hex.len())
|
||||
.step_by(2)
|
||||
.map(|index| u8::from_str_radix(&hex[index..index + 2], 16).expect("pinned MinIO metadata fixture"))
|
||||
.collect();
|
||||
crate::config::com::save_config(store.clone(), &BucketMetadata::new(bucket).save_file_path(), body)
|
||||
.await
|
||||
.expect("legacy metadata should be persisted without an incarnation sidecar");
|
||||
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let offline = take_set_disks_offline(&store, set, &[0, 1]).await;
|
||||
let info = store
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect("persisted MinIO metadata should authorize reads at the namespace read quorum");
|
||||
assert_eq!(info.name, bucket);
|
||||
assert!(info.versioning);
|
||||
assert!(info.object_locking);
|
||||
restore_set_disks(&store, set, offline).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn bucket_namespace_reads_report_missing_when_every_set_is_absent() {
|
||||
@@ -2476,7 +2090,6 @@ mod tests {
|
||||
#[serial]
|
||||
async fn bucket_namespace_reads_fail_closed_when_any_set_loses_quorum() {
|
||||
let (_temp_dir, ecstore) = setup_multi_pool_bucket_test_env().await;
|
||||
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), Vec::new()).await;
|
||||
let bucket = format!("degraded-expansion-{}", Uuid::new_v4().simple());
|
||||
ecstore.pools[0].disk_set[0]
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
@@ -2484,7 +2097,6 @@ mod tests {
|
||||
.expect("bucket should be created in the original pool only");
|
||||
ecstore.pools[1].disk_set[0].disks.write().await[0] = None;
|
||||
ecstore.pools[1].disk_set[0].disks.write().await[1] = None;
|
||||
ecstore.pools[1].disk_set[0].disks.write().await[2] = None;
|
||||
|
||||
let list_err = ecstore
|
||||
.list_bucket(&BucketOptions::default())
|
||||
@@ -2496,7 +2108,7 @@ mod tests {
|
||||
.get_bucket_info(&bucket, &BucketOptions::default())
|
||||
.await
|
||||
.expect_err("bucket validation must fail when an expansion pool is unavailable");
|
||||
assert_eq!(info_err, StorageError::ErasureReadQuorum);
|
||||
assert_eq!(info_err, StorageError::ErasureWriteQuorum);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -95,6 +95,7 @@ fn preflight_startup_rpc_secret_with(
|
||||
}
|
||||
}
|
||||
|
||||
const LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(60 * 3);
|
||||
const LOCAL_DECOMMISSION_RESUME_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
const LOCAL_DECOMMISSION_WATCHDOG_INTERVAL: Duration = Duration::from_secs(30);
|
||||
const LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY: Duration = Duration::from_secs(60 * 5);
|
||||
@@ -279,26 +280,20 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
async fn reconcile_local_decommission_after_init(store: &Arc<ECStore>, rx: CancellationToken) -> Result<()> {
|
||||
store
|
||||
.ensure_pool_meta_side_effects_safe("decommission worker recovery blocked while pool metadata requires recovery")
|
||||
.await?;
|
||||
if store.has_active_local_decommission_worker().await {
|
||||
return Ok(());
|
||||
}
|
||||
store.refresh_pool_status_meta().await?;
|
||||
let resume_required = pool_meta_has_active_decommission(&*store.pool_meta.read().await);
|
||||
if resume_required {
|
||||
crate::core::pools::acquire_pool_activation_fleet_proof(&store.ctx).await?;
|
||||
}
|
||||
store.spawn_missing_local_decommission_routines_with_token(rx).await
|
||||
}
|
||||
|
||||
async fn supervise_local_decommission_after_init(store: Arc<ECStore>, rx: CancellationToken) {
|
||||
run_local_decommission_watchdog(rx.clone(), || {
|
||||
let store = store.clone();
|
||||
let worker_rx = rx.clone();
|
||||
async move { reconcile_local_decommission_after_init(&store, worker_rx).await }
|
||||
async move {
|
||||
store
|
||||
.ensure_pool_meta_side_effects_safe("decommission worker recovery blocked while pool metadata requires recovery")
|
||||
.await?;
|
||||
if store.has_active_local_decommission_worker().await {
|
||||
return Ok(());
|
||||
}
|
||||
store.refresh_pool_status_meta().await?;
|
||||
store.spawn_missing_local_decommission_routines_with_token(worker_rx).await
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -789,9 +784,14 @@ impl ECStore {
|
||||
);
|
||||
}
|
||||
if has_local_decommission_leadership {
|
||||
// The watchdog checks recovery safety and retries transient failures.
|
||||
// Resume persisted work without an unconditional cold-start delay.
|
||||
tokio::spawn(supervise_local_decommission_after_init(self.clone(), rx.clone()));
|
||||
let store = self.clone();
|
||||
let decommission_rx = rx.clone();
|
||||
tokio::spawn(async move {
|
||||
if !wait_for_local_decommission_resume_delay(&decommission_rx, LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY).await {
|
||||
return;
|
||||
}
|
||||
supervise_local_decommission_after_init(store, decommission_rx).await;
|
||||
});
|
||||
}
|
||||
|
||||
let recovery_store = self.clone();
|
||||
@@ -2104,44 +2104,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_local_decommission_watchdog_cancelled_start_does_not_reconcile() {
|
||||
let rx = CancellationToken::new();
|
||||
rx.cancel();
|
||||
run_local_decommission_watchdog(rx, || async {
|
||||
panic!("cancelled startup must not schedule persisted work");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn test_local_decommission_recovery_waits_for_live_fleet_proof_before_reserving_worker() {
|
||||
let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
|
||||
mark_test_pool_decommissioning(&store, 0).await;
|
||||
assert!(store.ctx.is_dist_erasure().await);
|
||||
let worker_rx = CancellationToken::new();
|
||||
|
||||
{
|
||||
let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
|
||||
let err = super::reconcile_local_decommission_after_init(&store, worker_rx.clone())
|
||||
.await
|
||||
.expect_err("cold distributed recovery must wait for live fleet proof");
|
||||
assert!(
|
||||
crate::core::pools::is_pool_activation_fleet_proof_error(&err),
|
||||
"recovery must reach the live fleet proof gate: {err:?}"
|
||||
);
|
||||
assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
|
||||
assert!(pool_meta_has_active_decommission(&*store.pool_meta.read().await));
|
||||
}
|
||||
|
||||
super::reconcile_local_decommission_after_init(&store, worker_rx.clone())
|
||||
.await
|
||||
.expect("restored fleet proof should admit the persisted worker");
|
||||
assert!(store.has_active_local_decommission_worker().await);
|
||||
worker_rx.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_local_decommission_watchdog_retries_general_failures_until_cancelled() {
|
||||
let rx = CancellationToken::new();
|
||||
@@ -2153,13 +2115,11 @@ mod tests {
|
||||
let attempts = attempts.clone();
|
||||
let rx = rx.clone();
|
||||
async move {
|
||||
match attempts.fetch_add(1, Ordering::SeqCst) {
|
||||
0 => Err(StorageError::other("pool activation requires a live fleet capability proof")),
|
||||
1 => Err(StorageError::SlowDown),
|
||||
_ => {
|
||||
rx.cancel();
|
||||
Ok(())
|
||||
}
|
||||
if attempts.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
Err(StorageError::SlowDown)
|
||||
} else {
|
||||
rx.cancel();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2168,11 +2128,8 @@ mod tests {
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
tokio::time::advance(LOCAL_DECOMMISSION_RESUME_RETRY_DELAY).await;
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
tokio::time::advance(local_decommission_watchdog_retry_delay(2)).await;
|
||||
task.await.expect("watchdog task should exit after cancellation");
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 3);
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
@@ -12803,422 +12760,6 @@ mod tests {
|
||||
body
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn legacy_transition_state_inspection_and_apply_keep_all_disk_copies_unchanged() {
|
||||
run_large_stack_async_test("legacy-state-reconcile-inspection", || {
|
||||
legacy_transition_state_inspection_and_apply_case(false)
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[cfg(not(windows))]
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn legacy_transition_state_backfill_retries_partial_commits_and_preserves_other_bytes() {
|
||||
run_large_stack_async_test("legacy-state-reconcile-backfill", || {
|
||||
legacy_transition_state_inspection_and_apply_case(true)
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn legacy_transition_state_inspection_and_apply_case(write_enabled: bool) {
|
||||
#[cfg(windows)]
|
||||
assert!(!write_enabled, "Windows supports inspection but cannot prove repair directory durability");
|
||||
use crate::bucket::lifecycle::legacy_transition_state_reconcile::{
|
||||
LegacyTransitionStateReconcileOutcome as Outcome, LegacyTransitionStateReconcileRequest,
|
||||
LegacyTransitionStateReconcileSelector,
|
||||
};
|
||||
for (remote_version, expected_state) in [
|
||||
("", rustfs_filemeta::TransitionVersionState::KnownDisabled),
|
||||
("null", rustfs_filemeta::TransitionVersionState::SuspendedNull),
|
||||
("opaque-version", rustfs_filemeta::TransitionVersionState::Exact),
|
||||
] {
|
||||
let temp_dir = tempfile::tempdir().expect("legacy reconcile store directory");
|
||||
let (ctx, store, _shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "legacy-state-reconcile-inspect", &[4]))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let tier_name = "LEGACY-RECONCILE";
|
||||
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
|
||||
backend.set_put_remote_version(Some(remote_version.to_string())).await;
|
||||
let bucket = "legacy-state-reconcile-bucket";
|
||||
let object = "archive.bin";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create legacy fixture bucket");
|
||||
let mut reader = PutObjReader::from_vec(b"legacy reconcile body".repeat(1024));
|
||||
let source = store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("write source");
|
||||
{
|
||||
// Create the fixture under the existing remote-version writer
|
||||
// gate. This does not authorize legacy metadata reconciliation.
|
||||
let _proof = crate::services::notification_sys::install_current_remote_version_state_fleet_proof_for_test();
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE, Some("true")),
|
||||
(rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED, Some("true")),
|
||||
],
|
||||
store.transition_object(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: source.etag.clone().expect("source ETag"),
|
||||
..Default::default()
|
||||
},
|
||||
mod_time: source.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("transition source");
|
||||
}
|
||||
assert!(
|
||||
crate::services::notification_sys::acquire_legacy_transition_state_reconcile_fleet_proof()
|
||||
.await
|
||||
.is_none(),
|
||||
"fixture setup must not grant the missing reconciliation write capability"
|
||||
);
|
||||
let selector = LegacyTransitionStateReconcileSelector {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: "null".to_string(),
|
||||
};
|
||||
if expected_state == rustfs_filemeta::TransitionVersionState::Exact {
|
||||
backend
|
||||
.set_transition_candidate_probe_override(Some(
|
||||
crate::services::tier::warm_backend::TransitionCandidateProbe::Ambiguous,
|
||||
))
|
||||
.await;
|
||||
}
|
||||
let converged = store
|
||||
.inspect_legacy_transition_state(selector.clone())
|
||||
.await
|
||||
.expect("inspect an already explicit transition");
|
||||
assert_eq!(converged.outcome, Outcome::Migrated, "{converged:?}");
|
||||
assert!(!converged.changed);
|
||||
backend.set_transition_candidate_probe_override(None).await;
|
||||
rewrite_transitioned_xlmeta_as_legacy_unknown(temp_dir.path(), 0, bucket, object, remote_version.is_empty()).await;
|
||||
let paths = (0..4)
|
||||
.map(|disk| {
|
||||
temp_dir
|
||||
.path()
|
||||
.join(format!("pool0/set0/disk{disk}/{bucket}/{object}/{STORAGE_FORMAT_FILE}"))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut original = Vec::new();
|
||||
for path in &paths {
|
||||
original.push(tokio::fs::read(path).await.expect("original xl.meta"));
|
||||
}
|
||||
backend.clear_op_log().await;
|
||||
let inspection = store.inspect_legacy_transition_state(selector.clone());
|
||||
assert!(
|
||||
std::mem::size_of_val(&inspection) <= 4 * 1024,
|
||||
"admin inspection future must remain stack-bounded"
|
||||
);
|
||||
let inspected = inspection.await.expect("inspect legacy state");
|
||||
assert_eq!(inspected.outcome, Outcome::ReadyToMigrate, "{inspected:?}");
|
||||
assert!(!inspected.readiness.post_ready, "current fleet cannot authorize conditional writes");
|
||||
let target = inspected.target.expect("live probe should establish one model");
|
||||
assert_eq!(target.state, expected_state);
|
||||
let request = LegacyTransitionStateReconcileRequest {
|
||||
confirm: true,
|
||||
selector,
|
||||
source: inspected.source.expect("immutable source"),
|
||||
original_sets: inspected.original_sets,
|
||||
target,
|
||||
reconciliation_digest: inspected.reconciliation_digest.expect("expected tuple digest"),
|
||||
};
|
||||
#[cfg(not(windows))]
|
||||
if write_enabled {
|
||||
crate::services::notification_sys::with_legacy_transition_state_fleet_proof_for_test(async {
|
||||
crate::disk::local::bucket_durability::set(bucket, Some(crate::disk::local::DurabilityMode::None));
|
||||
let unsynced = store.reconcile_legacy_transition_state(request.clone()).await;
|
||||
crate::disk::local::bucket_durability::set(bucket, None);
|
||||
let unsynced = unsynced.expect("repair without metadata durability");
|
||||
assert_eq!(unsynced.outcome, Outcome::BackendUnavailable);
|
||||
assert!(!unsynced.changed);
|
||||
let rollback = paths[0].parent().expect("object directory").join(Uuid::new_v4().to_string());
|
||||
tokio::fs::create_dir(&rollback).await.expect("pending rollback directory");
|
||||
tokio::fs::write(rollback.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP), &original[0])
|
||||
.await
|
||||
.expect("pending old metadata backup");
|
||||
let unsettled = store.reconcile_legacy_transition_state(request.clone()).await;
|
||||
tokio::fs::remove_dir_all(&rollback).await.expect("settle fixture rollback");
|
||||
let unsettled = unsettled.expect("repair must wait for rollback");
|
||||
assert_eq!(unsettled.outcome, Outcome::BackendUnavailable);
|
||||
assert!(!unsettled.changed);
|
||||
for (path, bytes) in paths.iter().zip(&original) {
|
||||
assert_eq!(tokio::fs::read(path).await.expect("blocked repair leaves original bytes"), *bytes);
|
||||
}
|
||||
// The first disk commits; the second stops after staging.
|
||||
// This models an interrupted cross-disk effect without rollback.
|
||||
let disks = store.all_set_disks()[0].disk_inventory().await;
|
||||
let first_disk = disks[0].as_ref().expect("first physical disk");
|
||||
let publication_path = first_disk
|
||||
.get_object_path_for_io_if_local(bucket, &format!("{object}/{STORAGE_FORMAT_FILE}"))
|
||||
.expect("local disk")
|
||||
.expect("publication path");
|
||||
let crash_key = format!("{object}/{STORAGE_FORMAT_FILE}");
|
||||
let _hook = crate::disk::os::prepared_publication_test_hooks::install_at(
|
||||
crate::disk::os::prepared_publication_test_hooks::Stage::Rename,
|
||||
&publication_path,
|
||||
move || {
|
||||
crate::crash_inject::arm(crate::crash_inject::CrashPoint::MetaWriteAfterTmpBeforeRename, &crash_key);
|
||||
},
|
||||
);
|
||||
let partial = store
|
||||
.reconcile_legacy_transition_state(request.clone())
|
||||
.await
|
||||
.expect("partial repair response");
|
||||
assert_eq!(partial.outcome, Outcome::BackendUnavailable, "{partial:?}");
|
||||
assert!(partial.changed, "first copy was committed: {partial:?}");
|
||||
assert!(partial.changes_indeterminate);
|
||||
assert_ne!(tokio::fs::read(&paths[0]).await.expect("first committed copy"), original[0]);
|
||||
for (path, bytes) in paths[1..].iter().zip(&original[1..]) {
|
||||
assert_eq!(tokio::fs::read(path).await.expect("uncommitted copy"), *bytes);
|
||||
}
|
||||
assert!(
|
||||
store.all_set_disks()[0]
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.is_err(),
|
||||
"cleanup cannot select a partial repair subset"
|
||||
);
|
||||
let repaired = store
|
||||
.reconcile_legacy_transition_state(request.clone())
|
||||
.await
|
||||
.expect("retry original snapshot");
|
||||
assert_eq!(repaired.outcome, Outcome::Migrated, "{repaired:?}");
|
||||
assert!(repaired.changed);
|
||||
assert!(!repaired.changes_indeterminate);
|
||||
let mut committed = Vec::new();
|
||||
for (path, original) in paths.iter().zip(&original) {
|
||||
let raw = tokio::fs::read(path).await.expect("repaired copy");
|
||||
let metadata = FileMeta::load(&raw).expect("decode repaired copy");
|
||||
let previous = FileMeta::load(original).expect("decode original copy");
|
||||
assert_eq!(
|
||||
metadata.transition_reconcile_generation(None).unwrap(),
|
||||
previous.transition_reconcile_generation(None).unwrap()
|
||||
);
|
||||
let (_, version) = metadata.find_version(None).expect("selected version");
|
||||
let info = version.into_fileinfo(bucket, object, true).expect("repaired FileInfo");
|
||||
assert_eq!(info.transition_version_state, expected_state);
|
||||
assert_eq!(info.transition_version, request.target.remote_version);
|
||||
committed.push(raw);
|
||||
}
|
||||
assert!(
|
||||
store.all_set_disks()[0]
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("converged cleanup snapshot")
|
||||
.is_some()
|
||||
);
|
||||
let replay = store
|
||||
.reconcile_legacy_transition_state(request.clone())
|
||||
.await
|
||||
.expect("idempotent original request replay");
|
||||
assert_eq!(replay.outcome, Outcome::Migrated, "{replay:?}");
|
||||
assert!(!replay.changed);
|
||||
for (path, expected) in paths.iter().zip(&committed) {
|
||||
assert_eq!(
|
||||
tokio::fs::read(path).await.expect("replayed copy"),
|
||||
*expected,
|
||||
"idempotence preserves raw encoding"
|
||||
);
|
||||
}
|
||||
for (path, bytes) in paths.iter().zip(&original) {
|
||||
tokio::fs::write(path, bytes)
|
||||
.await
|
||||
.expect("reset independent cancellation fixture");
|
||||
}
|
||||
let (entered_tx, entered) = tokio::sync::oneshot::channel();
|
||||
let (release, released) = std::sync::mpsc::channel::<()>();
|
||||
let _pause = crate::disk::os::prepared_publication_test_hooks::install_at(
|
||||
crate::disk::os::prepared_publication_test_hooks::Stage::Rename,
|
||||
&publication_path,
|
||||
move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = released.recv();
|
||||
},
|
||||
);
|
||||
let mut repair = Box::pin(store.reconcile_legacy_transition_state(request.clone()));
|
||||
tokio::select! {
|
||||
result = &mut repair => panic!("repair completed before publication pause: {result:?}"),
|
||||
result = entered => result.expect("publication executor entered"),
|
||||
}
|
||||
let update_options = crate::disk::UpdateMetadataOpts::default();
|
||||
let mut update = Box::pin(first_disk.update_metadata(
|
||||
bucket,
|
||||
object,
|
||||
FileInfo {
|
||||
metadata: HashMap::from([("x-amz-meta-concurrent".to_string(), "kept".to_string())]),
|
||||
..Default::default()
|
||||
},
|
||||
&update_options,
|
||||
));
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut())
|
||||
.await
|
||||
.is_err(),
|
||||
"another metadata RMW must wait for publication"
|
||||
);
|
||||
drop(repair);
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(25), update.as_mut())
|
||||
.await
|
||||
.is_err(),
|
||||
"cancelling the coordinator must not release an in-flight disk mutation"
|
||||
);
|
||||
release.send(()).expect("resume owned publication");
|
||||
update.await.expect("serialized metadata update");
|
||||
let raw = tokio::fs::read(&paths[0])
|
||||
.await
|
||||
.expect("cancelled repair and later metadata update");
|
||||
let (_, version) = FileMeta::load(&raw)
|
||||
.expect("metadata after cancellation")
|
||||
.find_version(None)
|
||||
.expect("selected version");
|
||||
let info = version
|
||||
.into_fileinfo(bucket, object, true)
|
||||
.expect("metadata after serialized update");
|
||||
assert_eq!(info.transition_version_state, expected_state);
|
||||
assert_eq!(info.metadata.get("x-amz-meta-concurrent").map(String::as_str), Some("kept"));
|
||||
let stale = store
|
||||
.reconcile_legacy_transition_state(request.clone())
|
||||
.await
|
||||
.expect("stale original request");
|
||||
assert_eq!(
|
||||
stale.outcome,
|
||||
Outcome::Corrupt,
|
||||
"unrelated metadata change invalidates the original generation: {stale:?}"
|
||||
);
|
||||
assert!(!stale.changed);
|
||||
assert_eq!(tokio::fs::read(&paths[0]).await.expect("stale write leaves bytes unchanged"), raw);
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
// A pinned version probe uses GET to verify that exact
|
||||
// candidate; every backend operation still targets it.
|
||||
let operations = backend.op_log().await;
|
||||
assert!(
|
||||
operations.iter().all(|operation| match operation {
|
||||
MockWarmOp::Probe { object } | MockWarmOp::Get { object } => object == &request.source.remote_object,
|
||||
_ => false,
|
||||
}),
|
||||
"unexpected backend effects: {operations:?}"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
continue;
|
||||
}
|
||||
let mut tampered = request.clone();
|
||||
tampered.source.remote_object.push_str("-other");
|
||||
let probes_before = backend.op_log().await.len();
|
||||
let rejected = store
|
||||
.reconcile_legacy_transition_state(tampered)
|
||||
.await
|
||||
.expect("reject tampered tuple");
|
||||
assert_eq!(rejected.outcome, Outcome::Corrupt);
|
||||
assert_eq!(backend.op_log().await.len(), probes_before, "invalid digest must not probe the backend");
|
||||
let applied = store
|
||||
.reconcile_legacy_transition_state(request)
|
||||
.await
|
||||
.expect("apply must report unavailable write authority");
|
||||
assert_eq!(applied.outcome, Outcome::BackendUnavailable, "{applied:?}");
|
||||
assert_eq!(applied.reason_code, "write_fence_unavailable");
|
||||
assert!(!applied.changed);
|
||||
for (path, expected) in paths.iter().zip(&original) {
|
||||
assert_eq!(tokio::fs::read(path).await.expect("xl.meta after inspection"), *expected);
|
||||
}
|
||||
assert_eq!(backend.remove_count().await, 0);
|
||||
assert!(
|
||||
backend
|
||||
.op_log()
|
||||
.await
|
||||
.iter()
|
||||
.all(|operation| matches!(operation, MockWarmOp::Probe { .. }))
|
||||
);
|
||||
|
||||
backend.set_unreachable(true).await;
|
||||
let unavailable = store
|
||||
.inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: "null".to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("unreachable tier is a diagnostic outcome");
|
||||
assert_eq!(unavailable.outcome, Outcome::BackendUnavailable);
|
||||
assert!(!unavailable.changed);
|
||||
backend.set_unreachable(false).await;
|
||||
for candidate in ["", "00000000-0000-0000-0000-000000000000", "bad\nversion"] {
|
||||
backend
|
||||
.set_transition_candidate_probe_override(Some(
|
||||
crate::services::tier::warm_backend::TransitionCandidateProbe::VersionedPresent(candidate.to_string()),
|
||||
))
|
||||
.await;
|
||||
let invalid_proof = store
|
||||
.inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: "null".to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("invalid backend proof is a diagnostic outcome");
|
||||
assert_eq!(invalid_proof.outcome, Outcome::BackendUnavailable, "{invalid_proof:?}");
|
||||
assert!(invalid_proof.target.is_none());
|
||||
}
|
||||
backend.set_transition_candidate_probe_override(None).await;
|
||||
|
||||
backend.clear_op_log().await;
|
||||
for path in &paths[1..] {
|
||||
tokio::fs::remove_file(path)
|
||||
.await
|
||||
.expect("hide majority metadata copies in fixture");
|
||||
}
|
||||
let minority = store
|
||||
.inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: "null".to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("inspect minority legacy record");
|
||||
assert_eq!(
|
||||
minority.outcome,
|
||||
Outcome::BackendUnavailable,
|
||||
"a minority owner must remain visible: {minority:?}"
|
||||
);
|
||||
assert!(
|
||||
backend.op_log().await.is_empty(),
|
||||
"unproven metadata quorum cannot initiate a remote probe"
|
||||
);
|
||||
for (path, bytes) in paths.iter().zip(&original) {
|
||||
tokio::fs::write(path, bytes).await.expect("restore fixture copies");
|
||||
}
|
||||
tokio::fs::write(&paths[0], b"corrupt-xl-meta")
|
||||
.await
|
||||
.expect("inject corrupt copy");
|
||||
let corrupt = store
|
||||
.inspect_legacy_transition_state(LegacyTransitionStateReconcileSelector {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: "null".to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("inspect corrupt legacy record");
|
||||
assert_eq!(corrupt.outcome, Outcome::Corrupt, "{corrupt:?}");
|
||||
assert!(backend.op_log().await.is_empty(), "corruption must fail before backend I/O");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
|
||||
@@ -164,16 +164,6 @@ pub fn max_keys_plus_one(max_keys: i32, add_one: bool) -> i32 {
|
||||
max_keys
|
||||
}
|
||||
|
||||
fn list_versions_scan_limit(max_keys: i32, has_version_marker: bool) -> i32 {
|
||||
if max_keys <= 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// The marker object's versions may all be filtered out after gathering.
|
||||
// Reserve its raw entry in addition to the next-page lookahead entry.
|
||||
max_keys_plus_one(max_keys, true) + i32::from(has_version_marker)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
|
||||
enum GatherResultsState {
|
||||
LimitReached,
|
||||
@@ -2149,19 +2139,15 @@ fn build_list_versions_next_marker(
|
||||
// here; advertise it as the literal `null` marker so a resumed listing
|
||||
// parses it back to `VersionMarker::Null` instead of a nil UUID that
|
||||
// `find_version_index` can never match (issue #6745).
|
||||
let version_marker = if last.is_dir && last.mod_time.is_none() {
|
||||
// A CommonPrefix has no version to resume; a version marker would
|
||||
// make the next page include this same prefix again.
|
||||
None
|
||||
} else {
|
||||
(
|
||||
Some(append_list_cache_id_to_marker(last.name.clone(), cache_id)),
|
||||
Some(
|
||||
last.version_id
|
||||
.filter(|v| !v.is_nil())
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_else(|| "null".to_string()),
|
||||
)
|
||||
};
|
||||
(Some(append_list_cache_id_to_marker(last.name.clone(), cache_id)), version_marker)
|
||||
),
|
||||
)
|
||||
} else if let Some(last_prefix) = prefixes.last() {
|
||||
(Some(append_list_cache_id_to_marker(last_prefix.clone(), cache_id)), None)
|
||||
} else {
|
||||
@@ -2880,20 +2866,6 @@ fn listing_entries_supplement_target(
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(directory) = entries.0.iter().flatten().find(|entry| entry.is_dir()) {
|
||||
let directory_copies = entries
|
||||
.0
|
||||
.iter()
|
||||
.flatten()
|
||||
.filter(|entry| entry.is_dir() && entry.name == directory.name)
|
||||
.count();
|
||||
// A committed child may have some of its directory copies only on
|
||||
// fallback disks, just like object metadata in a partial primary sample.
|
||||
if directory_copies < resolver.dir_quorum {
|
||||
return Some(directory.name.clone());
|
||||
}
|
||||
}
|
||||
|
||||
for (idx, entry) in entries.0.iter().enumerate() {
|
||||
let Some(entry) = entry.as_ref().filter(|entry| entry.is_object()) else {
|
||||
continue;
|
||||
@@ -4046,7 +4018,8 @@ impl ECStore {
|
||||
None
|
||||
};
|
||||
|
||||
let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker);
|
||||
let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
|
||||
// Always request max_keys + 1 to detect if there are more results
|
||||
let mut opts = ListPathOptions {
|
||||
bucket: bucket.to_owned(),
|
||||
prefix: prefix.to_owned(),
|
||||
@@ -5352,7 +5325,7 @@ impl Sets {
|
||||
None
|
||||
};
|
||||
|
||||
let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker);
|
||||
let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
|
||||
let mut opts = ListPathOptions {
|
||||
bucket: bucket.to_owned(),
|
||||
prefix: prefix.to_owned(),
|
||||
@@ -6061,7 +6034,7 @@ impl SetDisks {
|
||||
|
||||
let has_version_marker = version_marker.is_some();
|
||||
let version_marker = version_marker.map(parse_version_marker).transpose()?;
|
||||
let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker);
|
||||
let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
|
||||
let mut opts = ListPathOptions {
|
||||
bucket: bucket.to_owned(),
|
||||
prefix: prefix.to_owned(),
|
||||
@@ -6275,7 +6248,7 @@ impl SetDisks {
|
||||
None
|
||||
};
|
||||
|
||||
let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker);
|
||||
let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
|
||||
let mut opts = ListPathOptions {
|
||||
bucket: bucket.to_owned(),
|
||||
prefix: prefix.to_owned(),
|
||||
@@ -7468,153 +7441,6 @@ mod test {
|
||||
assert!(cancel.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_versions_pagination_scan_limit_boundaries() {
|
||||
for has_version_marker in [false, true] {
|
||||
assert_eq!(super::list_versions_scan_limit(-1, has_version_marker), 0);
|
||||
assert_eq!(super::list_versions_scan_limit(0, has_version_marker), 0);
|
||||
let marker_slot = i32::from(has_version_marker);
|
||||
assert_eq!(super::list_versions_scan_limit(1, has_version_marker), 2 + marker_slot);
|
||||
assert_eq!(super::list_versions_scan_limit(MAX_OBJECT_LIST, has_version_marker), 1001 + marker_slot);
|
||||
assert_eq!(super::list_versions_scan_limit(i32::MAX, has_version_marker), 1001 + marker_slot);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_versions_pagination_does_not_require_an_empty_final_page() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "version-pagination-bucket";
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("pagination bucket should be created");
|
||||
let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
|
||||
|
||||
for kind in ["objects", "deletes", "null", "mixed", "delimiter"] {
|
||||
let count = if kind == "mixed" { 5 } else { 10 };
|
||||
let mut expected = Vec::new();
|
||||
for index in 0..count {
|
||||
let name = if kind == "delimiter" && index % 2 == 1 {
|
||||
format!("{kind}/testobject-{index:02}/child")
|
||||
} else {
|
||||
format!("{kind}/testobject-{index:02}")
|
||||
};
|
||||
let entry = match kind {
|
||||
"deletes" => test_delete_marker_meta_entry(&name, mod_time),
|
||||
"null" => test_object_meta_entry(&name),
|
||||
"mixed" => test_object_with_delete_marker_meta_entry(&name, mod_time, mod_time + time::Duration::SECOND),
|
||||
_ => test_object_meta_entry_with_erasure_versions(&name, &[(mod_time, "etag", 2, 2)]),
|
||||
};
|
||||
for dir in &dirs {
|
||||
let object_dir = dir.path().join(bucket).join(&name);
|
||||
tokio::fs::create_dir_all(&object_dir)
|
||||
.await
|
||||
.expect("pagination object directory should be created");
|
||||
tokio::fs::write(object_dir.join(STORAGE_FORMAT_FILE), &entry.metadata)
|
||||
.await
|
||||
.expect("pagination metadata should be written");
|
||||
}
|
||||
if kind == "delimiter" && index % 2 == 1 {
|
||||
expected.push((name.trim_end_matches("child").to_owned(), None, false));
|
||||
} else {
|
||||
let versions = entry.file_info_versions(bucket).expect("fixture versions should decode");
|
||||
expected.extend(
|
||||
versions
|
||||
.versions
|
||||
.iter()
|
||||
.map(|version| (name.clone(), version.version_id, version.deleted)),
|
||||
);
|
||||
}
|
||||
}
|
||||
let prefix = format!("{kind}/");
|
||||
let delimiter = (kind == "delimiter").then(|| "/".to_owned());
|
||||
// Exercise each public/internal entry point with the reported page size.
|
||||
// The store entry point also covers exact and one-over limit boundaries.
|
||||
for (layer, max_keys) in [(0, 0), (0, 1), (0, 5), (0, 9), (0, 10), (0, 11), (1, 5), (2, 5), (3, 5)] {
|
||||
if layer == 3 && delimiter.is_some() {
|
||||
continue;
|
||||
}
|
||||
let mut marker = None;
|
||||
let mut version_marker = None;
|
||||
let expected_pages = if max_keys == 0 {
|
||||
1
|
||||
} else {
|
||||
10usize.div_ceil(usize::try_from(max_keys).expect("positive page size"))
|
||||
};
|
||||
let mut actual = Vec::new();
|
||||
for page in 0..expected_pages {
|
||||
let result = match layer {
|
||||
0 => {
|
||||
store
|
||||
.clone()
|
||||
.inner_list_object_versions(bucket, &prefix, marker, version_marker, delimiter.clone(), max_keys)
|
||||
.await
|
||||
}
|
||||
1 => {
|
||||
store.pools[0]
|
||||
.clone()
|
||||
.inner_list_object_versions(bucket, &prefix, marker, version_marker, delimiter.clone(), max_keys)
|
||||
.await
|
||||
}
|
||||
2 => {
|
||||
store.pools[0].disk_set[0]
|
||||
.clone()
|
||||
.inner_list_object_versions(bucket, &prefix, marker, version_marker, delimiter.clone(), max_keys)
|
||||
.await
|
||||
}
|
||||
_ => {
|
||||
store.pools[0].disk_set[0]
|
||||
.clone()
|
||||
.inner_list_object_versions_for_recursive_delete(
|
||||
bucket,
|
||||
&prefix,
|
||||
marker,
|
||||
version_marker,
|
||||
max_keys,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
.expect("version page should list successfully");
|
||||
let page_size = usize::try_from(max_keys).expect("nonnegative page size");
|
||||
assert_eq!(result.objects.len() + result.prefixes.len(), (10 - page * page_size).min(page_size));
|
||||
let has_more = page + 1 < expected_pages;
|
||||
assert_eq!(result.is_truncated, has_more, "{kind}, layer {layer}, max_keys {max_keys}, page {page}");
|
||||
assert_eq!(
|
||||
result.next_marker.is_some(),
|
||||
has_more,
|
||||
"key marker must exist only when another page exists"
|
||||
);
|
||||
if !has_more {
|
||||
assert!(
|
||||
result.next_version_idmarker.is_none(),
|
||||
"the final page must not advertise a version marker"
|
||||
);
|
||||
}
|
||||
actual.extend(
|
||||
result
|
||||
.objects
|
||||
.into_iter()
|
||||
.map(|object| (object.name, object.version_id, object.delete_marker)),
|
||||
);
|
||||
actual.extend(result.prefixes.into_iter().map(|prefix| (prefix, None, false)));
|
||||
marker = result.next_marker;
|
||||
version_marker = result.next_version_idmarker;
|
||||
}
|
||||
// Objects and CommonPrefixes are serialized separately; compare their
|
||||
// identities without relying on their relative position in the response.
|
||||
actual.sort();
|
||||
let mut expected = if max_keys == 0 { Vec::new() } else { expected.clone() };
|
||||
expected.sort();
|
||||
assert_eq!(actual, expected, "{kind}, layer {layer}, max_keys {max_keys}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_marker_is_applied_only_when_key_marker_entry_is_present() {
|
||||
let version_marker = Some(VersionMarker::Null);
|
||||
@@ -9622,71 +9448,6 @@ mod test {
|
||||
assert!(supplemented.is_latest_delete_marker());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn latest_listing_supplement_checks_fallback_disks_for_common_prefix_quorum() {
|
||||
let mut fallback_disks = Vec::new();
|
||||
let mut fallback_tempdirs = Vec::new();
|
||||
for index in 0..4 {
|
||||
let tempdir = tempfile::tempdir().expect("fallback tempdir should be created");
|
||||
let endpoint = Endpoint::try_from(tempdir.path().to_str().expect("fallback path should be utf8"))
|
||||
.expect("fallback endpoint should parse");
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("fallback disk should be created");
|
||||
disk.make_volume("bucket").await.expect("fallback bucket should be created");
|
||||
for copies in [3, 4] {
|
||||
if index < copies {
|
||||
let object = format!("quux-{copies}/thud");
|
||||
let entry = test_object_meta_entry(&object);
|
||||
disk.write_all("bucket", &format!("{object}/{STORAGE_FORMAT_FILE}"), bytes::Bytes::from(entry.metadata))
|
||||
.await
|
||||
.expect("fallback child metadata should be written");
|
||||
}
|
||||
}
|
||||
fallback_disks.push(disk);
|
||||
fallback_tempdirs.push(tempdir);
|
||||
}
|
||||
let supplement = ListingSupplement::new(
|
||||
ListingSupplementOptions {
|
||||
bucket: "bucket".to_owned(),
|
||||
path: String::new(),
|
||||
recursive: false,
|
||||
incl_deleted: false,
|
||||
skip_hidden_prefix_check: false,
|
||||
filter_prefix: None,
|
||||
forward_to: None,
|
||||
per_disk_limit: 100,
|
||||
skip_total_timeout: true,
|
||||
walkdir_timeout: None,
|
||||
walkdir_stall_timeout: None,
|
||||
},
|
||||
Arc::new(fallback_disks),
|
||||
FallbackClaimTracker::default(),
|
||||
);
|
||||
// A 16-drive EC:4 set asks 12 primary disks. A committed write may
|
||||
// exist on eight primary disks and all four remaining fallback disks.
|
||||
let resolver = list_metadata_resolution_params("bucket".to_owned(), 4, 12, false, 0);
|
||||
for fallback_copies in [3, 4] {
|
||||
let prefix = format!("quux-{fallback_copies}/");
|
||||
let mut primary = vec![Some(test_dir_meta_entry(&prefix)); 8];
|
||||
primary.extend([None, None, None, None]);
|
||||
let entry =
|
||||
resolve_listing_entries_with_supplement(MetaCacheEntries(primary), resolver.clone(), true, supplement.clone())
|
||||
.await;
|
||||
assert_eq!(
|
||||
entry.map(|entry| entry.name),
|
||||
(fallback_copies == 4).then_some(prefix),
|
||||
"the common prefix needs all twelve copies, including fallback disks"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn latest_listing_supplement_keeps_a_subquorum_delete_marker_hidden() {
|
||||
let object_mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
|
||||
|
||||
@@ -3079,7 +3079,12 @@ impl ECStore {
|
||||
let store = Arc::clone(self);
|
||||
let write = async move {
|
||||
let object = "buckets/.scanner-pause-backlog.json";
|
||||
let mut opts = ObjectOptions::default();
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(preconditions),
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
// Match migration: fixed object namespace -> durable pool metadata ->
|
||||
// actual replica namespace. The replica need not be the hash-routed set.
|
||||
let object_guard = if store.single_pool() {
|
||||
@@ -3105,14 +3110,9 @@ impl ECStore {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let result = crate::data_movement::scanner_backlog::persist_native_scanner_pause_backlog_replica(
|
||||
set,
|
||||
data,
|
||||
preconditions,
|
||||
opts,
|
||||
"publish",
|
||||
)
|
||||
.await;
|
||||
let result = set
|
||||
.put_object(RUSTFS_META_BUCKET, object, &mut PutObjReader::from_vec(data), &opts)
|
||||
.await;
|
||||
drop(capacity_guard);
|
||||
drop(object_guard);
|
||||
result
|
||||
@@ -3747,37 +3747,29 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
no_lock: bool,
|
||||
) -> Result<usize> {
|
||||
let capacity_owner = DecommissionCapacityOwner::from_options(opts);
|
||||
match self
|
||||
.get_pool_info_existing_with_opts(bucket, object, &data_movement_pool_lookup_opts(opts, no_lock))
|
||||
.await
|
||||
{
|
||||
Ok((pinfo, _)) => {
|
||||
if let Some(owner) = capacity_owner {
|
||||
if self.is_decommission_capacity_target_reserved(owner, pinfo.index).await? {
|
||||
return Ok(pinfo.index);
|
||||
}
|
||||
} else {
|
||||
return Ok(pinfo.index);
|
||||
}
|
||||
}
|
||||
Ok((pinfo, _)) => Ok(pinfo.index),
|
||||
Err(err) => {
|
||||
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
|
||||
let expected_data_bytes = opts
|
||||
.capacity_expected_data_bytes()
|
||||
.or_else(|| usize::try_from(size).ok())
|
||||
.unwrap_or_default();
|
||||
return self
|
||||
.select_decommission_capacity_target_pool(owner, expected_data_bytes)
|
||||
.await;
|
||||
}
|
||||
|
||||
self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
|
||||
}
|
||||
}
|
||||
if let Some(owner) = capacity_owner {
|
||||
let expected_data_bytes = opts
|
||||
.capacity_expected_data_bytes()
|
||||
.or_else(|| usize::try_from(size).ok())
|
||||
.unwrap_or_default();
|
||||
return self
|
||||
.select_decommission_capacity_target_pool(owner, expected_data_bytes)
|
||||
.await;
|
||||
}
|
||||
|
||||
self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
|
||||
}
|
||||
|
||||
async fn find_data_movement_target_info(
|
||||
@@ -4259,10 +4251,8 @@ impl ECStore {
|
||||
}
|
||||
|
||||
/// Return metadata for DELETE preflight, including an explicitly addressed
|
||||
/// delete marker. GET/HEAD may also use this metadata-only lookup to enrich
|
||||
/// an already failed read with marker headers, never to serve marker data.
|
||||
/// Normal reads must keep using `get_object_info`; authorization and Object
|
||||
/// Lock enforcement still belong to the caller and locked delete.
|
||||
/// delete marker. Read APIs must keep using `get_object_info`; authorization
|
||||
/// and Object Lock enforcement still belong to the caller and locked delete.
|
||||
#[instrument(level = "trace", skip_all)]
|
||||
pub async fn get_object_info_for_delete(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
|
||||
self.get_object_info_snapshot(bucket, object, opts, true).await
|
||||
|
||||
@@ -50,9 +50,6 @@ use tracing::{error, warn};
|
||||
use uuid::Uuid;
|
||||
use xxhash_rust::xxh64;
|
||||
|
||||
mod transition_reconcile;
|
||||
pub use transition_reconcile::TransitionStateReconcileTarget;
|
||||
|
||||
// XL header specifies the format
|
||||
pub static XL_FILE_HEADER: [u8; 4] = *b"XL2 ";
|
||||
// pub static XL_FILE_VERSION_CURRENT: [u8; 4] = [0; 4];
|
||||
@@ -394,16 +391,6 @@ impl FileMeta {
|
||||
|
||||
if ver_vid == fi_vid {
|
||||
let mut ver = FileMetaVersion::try_from(version.meta.as_slice())?;
|
||||
let previous = ver
|
||||
.object
|
||||
.as_ref()
|
||||
.is_some_and(|object| {
|
||||
rustfs_utils::http::contains_key_bytes(
|
||||
&object.meta_sys,
|
||||
rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
)
|
||||
})
|
||||
.then(|| ver.clone());
|
||||
|
||||
if let Some(ref mut obj) = ver.object {
|
||||
if replace_user_metadata {
|
||||
@@ -460,9 +447,6 @@ impl FileMeta {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(previous) = previous {
|
||||
transition_reconcile::preserve_reconciled_transition(&previous, &mut ver)?;
|
||||
}
|
||||
// Update
|
||||
version.header = ver.header();
|
||||
version.meta = ver.marshal_msg()?;
|
||||
@@ -508,7 +492,7 @@ impl FileMeta {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn add_version_filemata(&mut self, mut version: FileMetaVersion) -> Result<()> {
|
||||
pub fn add_version_filemata(&mut self, version: FileMetaVersion) -> Result<()> {
|
||||
if !version.valid() {
|
||||
return Err(Error::other("file meta version invalid"));
|
||||
}
|
||||
@@ -528,7 +512,6 @@ impl FileMeta {
|
||||
if existing.free_version() != version.free_version() {
|
||||
return Err(Error::other("cannot replace a free version with a non-free version"));
|
||||
}
|
||||
transition_reconcile::preserve_reconciled_transition(&existing, &mut version)?;
|
||||
return self.set_idx(fidx, version);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,384 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::{FileMeta, FileMetaVersion};
|
||||
use crate::{Error, Result, TRANSITION_COMPLETE, TransitionVersionState};
|
||||
use rustfs_utils::http::metadata_compat::{
|
||||
SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_bytes,
|
||||
get_consistent_bytes, insert_bytes, remove_bytes,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
const RECONCILE_SUFFIXES: [&str; 3] = [
|
||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
SUFFIX_TRANSITIONED_VERSION_ID,
|
||||
SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
];
|
||||
|
||||
/// The only fields a legacy transition repair is allowed to persist.
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TransitionStateReconcileTarget {
|
||||
pub state: TransitionVersionState,
|
||||
pub remote_version: Option<String>,
|
||||
pub destination_id: String,
|
||||
}
|
||||
|
||||
impl TransitionStateReconcileTarget {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
let valid_version = match self.state {
|
||||
TransitionVersionState::KnownDisabled => self.remote_version.is_none(),
|
||||
TransitionVersionState::SuspendedNull => self.remote_version.as_deref() == Some("null"),
|
||||
TransitionVersionState::Exact => self.remote_version.as_deref().is_some_and(|value| {
|
||||
!value.is_empty()
|
||||
&& value.len() <= 1024
|
||||
&& value != "null"
|
||||
&& !value.chars().any(char::is_control)
|
||||
&& !Uuid::parse_str(value).is_ok_and(|id| id.is_nil())
|
||||
}),
|
||||
TransitionVersionState::Unknown => false,
|
||||
};
|
||||
if !valid_version
|
||||
|| self.destination_id.len() != 64
|
||||
|| !self
|
||||
.destination_id
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl FileMeta {
|
||||
/// Canonical identity of every version and inline byte, excluding only the
|
||||
/// three repairable suffixes on the selected version. It survives a repair
|
||||
/// and encoding-order changes, while detecting unrelated metadata changes.
|
||||
pub fn transition_reconcile_generation(&self, version_id: Option<Uuid>) -> Result<Vec<u8>> {
|
||||
if self
|
||||
.versions
|
||||
.iter()
|
||||
.filter(|version| version.header.version_id.unwrap_or_default() == version_id.unwrap_or_default())
|
||||
.count()
|
||||
!= 1
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let (selected, _) = self.find_version(version_id)?;
|
||||
let mut versions = Vec::with_capacity(self.versions.len());
|
||||
for index in 0..self.versions.len() {
|
||||
let mut version = self.get_idx(index)?;
|
||||
if index == selected {
|
||||
let object = version.object.as_mut().ok_or(Error::FileCorrupt)?;
|
||||
for suffix in RECONCILE_SUFFIXES {
|
||||
remove_bytes(&mut object.meta_sys, suffix);
|
||||
}
|
||||
}
|
||||
versions.push(version);
|
||||
}
|
||||
versions.sort_by_key(|version| version.get_version_id().unwrap_or_default());
|
||||
let mut value = serde_json::to_value((&versions, &self.data)).map_err(|_| Error::FileCorrupt)?;
|
||||
value.sort_all_objects();
|
||||
serde_json::to_vec(&value).map_err(|_| Error::FileCorrupt)
|
||||
}
|
||||
|
||||
/// Returns false for an already converged record. Callers must serialize
|
||||
/// the read/check/commit and compare the observed metadata generation.
|
||||
pub fn reconcile_transition_state(
|
||||
&mut self,
|
||||
version_id: Option<Uuid>,
|
||||
target: &TransitionStateReconcileTarget,
|
||||
) -> Result<bool> {
|
||||
target.validate()?;
|
||||
let (index, mut version) = self.find_version(version_id)?;
|
||||
let info = version.into_fileinfo("", "", true)?;
|
||||
info.validate_for_metadata_read()?;
|
||||
if info.transition_status != TRANSITION_COMPLETE
|
||||
|| info.transition_tier.is_empty()
|
||||
|| info.transitioned_objname.is_empty()
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let object = version.object.as_mut().ok_or(Error::FileCorrupt)?;
|
||||
let destination = get_consistent_bytes(&object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID);
|
||||
if contains_key_bytes(&object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
|
||||
&& destination != Some(target.destination_id.as_bytes())
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
if contains_key_bytes(&object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) {
|
||||
if info.transition_version_state == target.state
|
||||
&& info.transition_version == target.remote_version
|
||||
&& destination == Some(target.destination_id.as_bytes())
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
if info.transition_version_state != TransitionVersionState::Unknown
|
||||
|| info
|
||||
.transition_version
|
||||
.as_deref()
|
||||
.filter(|value| !value.is_empty())
|
||||
.is_some_and(|value| Some(value) != target.remote_version.as_deref())
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
insert_bytes(
|
||||
&mut object.meta_sys,
|
||||
SUFFIX_TRANSITIONED_VERSION_STATE,
|
||||
target.state.as_str().as_bytes().to_vec(),
|
||||
);
|
||||
remove_bytes(&mut object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID);
|
||||
if let Some(version) = &target.remote_version {
|
||||
insert_bytes(&mut object.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, version.as_bytes().to_vec());
|
||||
}
|
||||
insert_bytes(
|
||||
&mut object.meta_sys,
|
||||
SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
target.destination_id.as_bytes().to_vec(),
|
||||
);
|
||||
version.into_fileinfo("", "", true)?.validate_for_metadata_read()?;
|
||||
self.set_idx(index, version)?;
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
/// A stale healer or metadata writer may carry the original absent fields.
|
||||
/// Preserve a proven binding for the same immutable transition, or reject an
|
||||
/// attempted change of meaning. A new payload/version or a delete is separate.
|
||||
pub(super) fn preserve_reconciled_transition(previous: &FileMetaVersion, next: &mut FileMetaVersion) -> Result<()> {
|
||||
let (Some(previous_object), Some(next_object)) = (&previous.object, &mut next.object) else {
|
||||
return Ok(());
|
||||
};
|
||||
if !contains_key_bytes(&previous_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
|
||||
|| !contains_key_bytes(&previous_object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE)
|
||||
|| previous_object.data_dir != next_object.data_dir
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let previous_info = previous.into_fileinfo("", "", true)?;
|
||||
if previous_info.transition_version_state == TransitionVersionState::Unknown
|
||||
|| previous_info.transition_status != TRANSITION_COMPLETE
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let next_info = next.into_fileinfo("", "", true)?;
|
||||
if previous_info.transition_tier != next_info.transition_tier
|
||||
|| previous_info.transitioned_objname != next_info.transitioned_objname
|
||||
|| previous_info.transition_status != next_info.transition_status
|
||||
|| previous_info.size != next_info.size
|
||||
|| previous_info.metadata.get("etag") != next_info.metadata.get("etag")
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
let destination =
|
||||
get_consistent_bytes(&previous_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID).ok_or(Error::FileCorrupt)?;
|
||||
previous_info.validate_for_metadata_read()?;
|
||||
TransitionStateReconcileTarget {
|
||||
state: previous_info.transition_version_state,
|
||||
remote_version: previous_info.transition_version.clone(),
|
||||
destination_id: std::str::from_utf8(destination).map_err(|_| Error::FileCorrupt)?.to_string(),
|
||||
}
|
||||
.validate()?;
|
||||
let next_object = next.object.as_mut().ok_or(Error::FileCorrupt)?;
|
||||
if next_info
|
||||
.transition_version
|
||||
.as_ref()
|
||||
.is_some_and(|version| Some(version) != previous_info.transition_version.as_ref())
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
if contains_key_bytes(&next_object.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) {
|
||||
if previous_info.transition_version_state != next_info.transition_version_state
|
||||
|| previous_info.transition_version != next_info.transition_version
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
} else if next_info.transition_version_state != TransitionVersionState::Unknown {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
if contains_key_bytes(&next_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
|
||||
&& get_consistent_bytes(&next_object.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID) != Some(destination)
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
for suffix in RECONCILE_SUFFIXES {
|
||||
remove_bytes(&mut next_object.meta_sys, suffix);
|
||||
if let Some(value) = get_consistent_bytes(&previous_object.meta_sys, suffix) {
|
||||
insert_bytes(&mut next_object.meta_sys, suffix, value.to_vec());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{ErasureInfo, FileInfo, ObjectPartInfo};
|
||||
|
||||
fn legacy() -> (FileMeta, FileInfo) {
|
||||
let info = FileInfo {
|
||||
version_id: Some(Uuid::from_u128(1)),
|
||||
data_dir: Some(Uuid::from_u128(2)),
|
||||
mod_time: Some(time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("fixture time")),
|
||||
size: 7,
|
||||
parts: vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 7,
|
||||
actual_size: 7,
|
||||
..Default::default()
|
||||
}],
|
||||
erasure: ErasureInfo {
|
||||
algorithm: "ReedSolomon".to_string(),
|
||||
data_blocks: 2,
|
||||
parity_blocks: 2,
|
||||
block_size: 1024 * 1024,
|
||||
index: 1,
|
||||
distribution: vec![1, 2, 3, 4],
|
||||
..Default::default()
|
||||
},
|
||||
transition_status: TRANSITION_COMPLETE.to_string(),
|
||||
transition_tier: "WARM".to_string(),
|
||||
transitioned_objname: "remote-object".to_string(),
|
||||
metadata: std::collections::HashMap::from([("etag".to_string(), "source-etag".to_string())]),
|
||||
data: Some(bytes::Bytes::from_static(b"payload")),
|
||||
..Default::default()
|
||||
};
|
||||
let mut metadata = FileMeta::new();
|
||||
metadata.add_version(info.clone()).expect("legacy fixture");
|
||||
(metadata, info)
|
||||
}
|
||||
|
||||
fn target(state: TransitionVersionState) -> TransitionStateReconcileTarget {
|
||||
TransitionStateReconcileTarget {
|
||||
state,
|
||||
remote_version: match state {
|
||||
TransitionVersionState::Exact => Some("opaque-version".to_string()),
|
||||
TransitionVersionState::SuspendedNull => Some("null".to_string()),
|
||||
_ => None,
|
||||
},
|
||||
destination_id: "ab".repeat(32),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_reconcile_preserves_payload_and_generation_and_is_idempotent() {
|
||||
for state in [
|
||||
TransitionVersionState::KnownDisabled,
|
||||
TransitionVersionState::SuspendedNull,
|
||||
TransitionVersionState::Exact,
|
||||
] {
|
||||
let (mut metadata, info) = legacy();
|
||||
let mut other = info.clone();
|
||||
other.version_id = Some(Uuid::from_u128(3));
|
||||
other.data_dir = Some(Uuid::from_u128(4));
|
||||
other.transition_status.clear();
|
||||
other.transition_tier.clear();
|
||||
other.transitioned_objname.clear();
|
||||
other.data = Some(bytes::Bytes::from_static(b"other!!"));
|
||||
metadata.add_version(other.clone()).expect("unrelated inline version");
|
||||
let other_before = metadata.find_version(other.version_id).expect("unrelated version").1;
|
||||
let original_data = metadata.data.clone();
|
||||
let generation = metadata
|
||||
.transition_reconcile_generation(info.version_id)
|
||||
.expect("initial generation");
|
||||
let target = target(state);
|
||||
assert!(metadata.reconcile_transition_state(info.version_id, &target).expect("repair"));
|
||||
let bytes = metadata.marshal_msg().expect("encode repair");
|
||||
let mut reloaded = FileMeta::load(&bytes).expect("reload repair");
|
||||
assert_eq!(reloaded.data, original_data);
|
||||
assert_eq!(
|
||||
reloaded
|
||||
.find_version(other.version_id)
|
||||
.expect("preserved unrelated version")
|
||||
.1,
|
||||
other_before
|
||||
);
|
||||
assert_eq!(
|
||||
reloaded
|
||||
.transition_reconcile_generation(info.version_id)
|
||||
.expect("repaired generation"),
|
||||
generation
|
||||
);
|
||||
assert!(!reloaded.reconcile_transition_state(info.version_id, &target).expect("retry"));
|
||||
let (_, version) = reloaded.find_version(info.version_id).expect("selected version");
|
||||
let repaired = version.into_fileinfo("", "", true).expect("decode explicit state");
|
||||
assert_eq!(repaired.transition_version_state, state);
|
||||
assert_eq!(repaired.transition_version, target.remote_version);
|
||||
assert_eq!(repaired.parts, info.parts);
|
||||
for prefix in [
|
||||
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
|
||||
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
|
||||
] {
|
||||
assert_eq!(
|
||||
version
|
||||
.object
|
||||
.as_ref()
|
||||
.expect("object")
|
||||
.meta_sys
|
||||
.get(&format!("{prefix}{SUFFIX_TRANSITIONED_VERSION_STATE}")),
|
||||
Some(&state.as_str().as_bytes().to_vec())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_reconcile_generation_detects_unrelated_metadata_and_inline_changes() {
|
||||
let (mut metadata, info) = legacy();
|
||||
let original = metadata.transition_reconcile_generation(info.version_id).expect("generation");
|
||||
let mut updated = info.clone();
|
||||
updated.metadata.insert("user-tag".to_string(), "changed".to_string());
|
||||
metadata.update_object_version(updated).expect("update unrelated field");
|
||||
assert_ne!(metadata.transition_reconcile_generation(info.version_id).expect("generation"), original);
|
||||
let (mut metadata, mut info) = legacy();
|
||||
info.data = Some(bytes::Bytes::from_static(b"changed"));
|
||||
metadata.add_version(info.clone()).expect("change inline bytes");
|
||||
assert_ne!(metadata.transition_reconcile_generation(info.version_id).expect("generation"), original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_reconcile_binding_survives_stale_heal_and_metadata_writes() {
|
||||
let (mut metadata, mut stale) = legacy();
|
||||
let target = target(TransitionVersionState::Exact);
|
||||
metadata
|
||||
.reconcile_transition_state(stale.version_id, &target)
|
||||
.expect("repair");
|
||||
metadata
|
||||
.add_version(stale.clone())
|
||||
.expect("stale heal must preserve the binding");
|
||||
stale.metadata.insert("user-tag".to_string(), "updated".to_string());
|
||||
metadata
|
||||
.update_object_version(stale.clone())
|
||||
.expect("ordinary metadata update");
|
||||
assert!(
|
||||
!metadata
|
||||
.reconcile_transition_state(stale.version_id, &target)
|
||||
.expect("binding remains exact")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_reconcile_rejects_explicit_unknown_and_conflicting_binding() {
|
||||
let (mut metadata, mut info) = legacy();
|
||||
rustfs_utils::http::insert_str(&mut info.metadata, SUFFIX_TRANSITIONED_VERSION_STATE, "unknown".to_string());
|
||||
metadata.add_version(info.clone()).expect("explicit unknown fixture");
|
||||
assert!(
|
||||
metadata
|
||||
.reconcile_transition_state(info.version_id, &target(TransitionVersionState::Exact))
|
||||
.is_err()
|
||||
);
|
||||
let (mut metadata, mut stale) = legacy();
|
||||
metadata
|
||||
.reconcile_transition_state(stale.version_id, &target(TransitionVersionState::Exact))
|
||||
.expect("repair");
|
||||
stale.transition_version_state = TransitionVersionState::KnownDisabled;
|
||||
assert!(
|
||||
metadata.add_version(stale).is_err(),
|
||||
"an explicit state cannot be replaced with a different model"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -31,7 +31,6 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
test-util = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
@@ -105,7 +104,6 @@ walkdir = { workspace = true }
|
||||
http = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
chrono = { workspace = true }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -847,7 +847,7 @@ mod tests {
|
||||
|
||||
fn create_test_heal_manager() -> Arc<HealManager> {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
Arc::new(HealManager::new_without_root_recovery_for_test(storage, None))
|
||||
Arc::new(HealManager::new(storage, None))
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1481,7 +1481,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn test_process_start_request_returns_admission_result() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(
|
||||
let manager = Arc::new(HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
@@ -1826,7 +1826,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_process_query_request_reports_displaced_terminal_detail() {
|
||||
let heal_manager = Arc::new(HealManager::new_without_root_recovery_for_test(
|
||||
let heal_manager = Arc::new(HealManager::new(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
|
||||
+41
-331
@@ -536,14 +536,14 @@ fn completed_status_is_retrying(status: &HealTaskStatus) -> bool {
|
||||
matches!(status, HealTaskStatus::Retrying { .. })
|
||||
}
|
||||
|
||||
fn retry_budget_for_result(task: &HealTask, result: &Result<()>, retryable_batch_failure: bool) -> Option<(Duration, String)> {
|
||||
fn retry_budget_for_result(task: &HealTask, result: &Result<()>) -> Option<(Duration, String)> {
|
||||
let Err(err) = result else {
|
||||
return None;
|
||||
};
|
||||
if task.retry_attempts >= MAX_RECOVERABLE_HEAL_RETRIES {
|
||||
return None;
|
||||
}
|
||||
if task.has_batch_failure() && !retryable_batch_failure {
|
||||
if task.has_batch_failure() {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -559,13 +559,12 @@ fn retry_budget_for_result(task: &HealTask, result: &Result<()>, retryable_batch
|
||||
|
||||
#[cfg(test)]
|
||||
fn retry_request_for_result(task: &HealTask, result: &Result<()>) -> Option<(HealRequest, Duration, String)> {
|
||||
let (delay, error) = retry_budget_for_result(task, result, false)?;
|
||||
let (delay, error) = retry_budget_for_result(task, result)?;
|
||||
Some((task.retry_request(), delay, error))
|
||||
}
|
||||
|
||||
async fn retry_request_for_result_with_budget(task: &HealTask, result: &Result<()>) -> Option<(HealRequest, Duration, String)> {
|
||||
let retryable_batch_failure = task.batch_failure_is_retryable().await;
|
||||
let (delay, error) = retry_budget_for_result(task, result, retryable_batch_failure)?;
|
||||
let (delay, error) = retry_budget_for_result(task, result)?;
|
||||
let request = match task.retry_request_with_remaining_timeout().await {
|
||||
Ok(request) => request,
|
||||
Err(err) => {
|
||||
@@ -842,10 +841,6 @@ pub struct HealManager {
|
||||
replacement_recovery_anchors: Arc<std::sync::Mutex<HashMap<String, String>>>,
|
||||
/// Set IDs whose durable replacement metadata is corrupt or conflicting.
|
||||
replacement_recovery_blocked_sets: Arc<std::sync::Mutex<HashSet<String>>>,
|
||||
/// Durable handoff of interrupted administrator root traversals.
|
||||
root_recovery: Arc<root_recovery::RootHealRecovery>,
|
||||
/// Keep forceStart's cancellation side effects inside the shutdown fence.
|
||||
force_start_shutdown: Mutex<()>,
|
||||
/// Storage layer interface
|
||||
storage: Arc<dyn HealStorageAPI>,
|
||||
/// Cancel token
|
||||
@@ -881,7 +876,6 @@ struct HealQueueContext<'a> {
|
||||
retrying_heals: &'a Arc<Mutex<HashMap<String, RetryingHeal>>>,
|
||||
mrf_repair_notice_targets: &'a Arc<StdMutex<HashMap<String, Vec<MrfRepairNoticeTarget>>>>,
|
||||
replacement_recovery_anchors: &'a Arc<std::sync::Mutex<HashMap<String, String>>>,
|
||||
root_recovery: &'a Arc<root_recovery::RootHealRecovery>,
|
||||
config: &'a Arc<RwLock<HealConfig>>,
|
||||
statistics: &'a Arc<RwLock<HealStatistics>>,
|
||||
storage: &'a Arc<dyn HealStorageAPI>,
|
||||
@@ -945,10 +939,6 @@ impl HealManager {
|
||||
matches!(request.source, HealRequestSource::Admin | HealRequestSource::Internal)
|
||||
}
|
||||
|
||||
fn queued_request_can_be_displaced(request: &HealRequest) -> bool {
|
||||
!root_recovery::is_admin_heal_recovery(&request.heal_type, request.source)
|
||||
}
|
||||
|
||||
fn request_bypasses_mainline_throttle(request: &HealRequest) -> bool {
|
||||
request.force_start
|
||||
|| matches!(request.source, HealRequestSource::Admin | HealRequestSource::Internal)
|
||||
@@ -1085,15 +1075,11 @@ impl HealManager {
|
||||
let per_object_request = request.heal_type.is_per_object();
|
||||
|
||||
if queue_len >= queue_capacity && !request.force_start {
|
||||
if Self::can_displace_queued_work(&request)
|
||||
&& queue.can_displace_lower_priority_where(request.priority, Self::queued_request_can_be_displaced)
|
||||
{
|
||||
if Self::can_displace_queued_work(&request) && queue.can_displace_lower_priority(request.priority) {
|
||||
let request_id = request.id.clone();
|
||||
let priority = request.priority;
|
||||
let source = request.source;
|
||||
if let Some(displaced) =
|
||||
queue.push_displacing_lower_priority_where(request, Self::queued_request_can_be_displaced)
|
||||
{
|
||||
if let Some(displaced) = queue.push_displacing_lower_priority(request) {
|
||||
publish_heal_queue_length(queue);
|
||||
Self::record_admission_metric(source, HealAdmissionResult::Accepted, context);
|
||||
demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
|
||||
@@ -1367,51 +1353,6 @@ impl HealManager {
|
||||
});
|
||||
}
|
||||
|
||||
async fn start_root_recovery_terminal_gc(&self) {
|
||||
let cancel = self.cancel_token.clone();
|
||||
let root_recovery = self.root_recovery.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut ticker = interval(RESUME_GC_INTERVAL);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = cancel.cancelled() => break,
|
||||
_ = ticker.tick() => {
|
||||
match root_recovery.gc_terminal_receipts_once(SystemTime::now()).await {
|
||||
Ok(report) => {
|
||||
if report.pending_removed > 0 || report.terminals_removed > 0 || report.budget_exhausted {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_RESUME_GC,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
state = "root_terminal_gc",
|
||||
scanned = report.scanned,
|
||||
retained = report.retained,
|
||||
pending_removed = report.pending_removed,
|
||||
terminals_removed = report.terminals_removed,
|
||||
budget_exhausted = report.budget_exhausted,
|
||||
"Root heal terminal receipt GC inspected durable state"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_RESUME_GC,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
state = "root_terminal_gc_failed",
|
||||
error = %error,
|
||||
"Root heal terminal receipt GC failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Create new HealManager
|
||||
pub fn new(storage: Arc<dyn HealStorageAPI>, config: Option<HealConfig>) -> Self {
|
||||
Self::new_with_workload_provider(storage, config, None)
|
||||
@@ -1422,15 +1363,6 @@ impl HealManager {
|
||||
storage: Arc<dyn HealStorageAPI>,
|
||||
config: Option<HealConfig>,
|
||||
workload_provider: Option<WorkloadSnapshotProviderRef>,
|
||||
) -> Self {
|
||||
Self::new_with_root_recovery(storage, config, workload_provider, Arc::new(root_recovery::RootHealRecovery::default()))
|
||||
}
|
||||
|
||||
fn new_with_root_recovery(
|
||||
storage: Arc<dyn HealStorageAPI>,
|
||||
config: Option<HealConfig>,
|
||||
workload_provider: Option<WorkloadSnapshotProviderRef>,
|
||||
root_recovery: Arc<root_recovery::RootHealRecovery>,
|
||||
) -> Self {
|
||||
let config = config.unwrap_or_default();
|
||||
Self {
|
||||
@@ -1445,8 +1377,6 @@ impl HealManager {
|
||||
mrf_repair_notice_targets: Arc::new(StdMutex::new(HashMap::new())),
|
||||
replacement_recovery_anchors: Arc::new(std::sync::Mutex::new(HashMap::new())),
|
||||
replacement_recovery_blocked_sets: Arc::new(std::sync::Mutex::new(HashSet::new())),
|
||||
root_recovery,
|
||||
force_start_shutdown: Mutex::new(()),
|
||||
storage,
|
||||
cancel_token: CancellationToken::new(),
|
||||
statistics: Arc::new(RwLock::new(HealStatistics::new())),
|
||||
@@ -1456,27 +1386,6 @@ impl HealManager {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[doc(hidden)]
|
||||
pub fn new_without_root_recovery_for_test(storage: Arc<dyn HealStorageAPI>, config: Option<HealConfig>) -> Self {
|
||||
Self::new_with_root_recovery(storage, config, None, Arc::new(root_recovery::RootHealRecovery::disabled_for_tests()))
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[doc(hidden)]
|
||||
pub fn new_with_workload_provider_without_root_recovery_for_test(
|
||||
storage: Arc<dyn HealStorageAPI>,
|
||||
config: Option<HealConfig>,
|
||||
workload_provider: Option<WorkloadSnapshotProviderRef>,
|
||||
) -> Self {
|
||||
Self::new_with_root_recovery(
|
||||
storage,
|
||||
config,
|
||||
workload_provider,
|
||||
Arc::new(root_recovery::RootHealRecovery::disabled_for_tests()),
|
||||
)
|
||||
}
|
||||
|
||||
/// Start HealManager
|
||||
pub async fn start(&self) -> Result<()> {
|
||||
let mut state = self.state.write().await;
|
||||
@@ -1503,23 +1412,6 @@ impl HealManager {
|
||||
"Heal manager starting"
|
||||
);
|
||||
|
||||
// Restore graceful-shutdown root responsibilities before automatic
|
||||
// repair can admit overlapping work.
|
||||
if let Err(error) = self.replay_root_heals().await {
|
||||
// A missing owner or invalid root record must not block existing
|
||||
// replacement recovery. Keep its file for a later restart after
|
||||
// the owner is readable or the record has been repaired.
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_MANAGER_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
state = "root_recovery_deferred",
|
||||
error = %error,
|
||||
"Root heal restart recovery deferred"
|
||||
);
|
||||
}
|
||||
|
||||
// start scheduler
|
||||
self.start_scheduler().await?;
|
||||
|
||||
@@ -1529,7 +1421,6 @@ impl HealManager {
|
||||
|
||||
// Inspect resume artifacts in a bounded, fail-closed background task.
|
||||
self.start_resume_gc().await;
|
||||
self.start_root_recovery_terminal_gc().await;
|
||||
|
||||
// start auto disk scanner to heal unformatted disks
|
||||
if self.config.read().await.enable_auto_heal {
|
||||
@@ -1558,7 +1449,6 @@ impl HealManager {
|
||||
|
||||
/// Stop HealManager
|
||||
pub async fn stop(&self) -> Result<()> {
|
||||
let _force_start_guard = self.force_start_shutdown.lock().await;
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_MANAGER_STATE,
|
||||
@@ -1568,39 +1458,11 @@ impl HealManager {
|
||||
"Heal manager stopping"
|
||||
);
|
||||
|
||||
// Keep scheduler, cancellation, and retry ownership stable until every
|
||||
// unfinished admin control-plane heal has a durable successor. A failed
|
||||
// write must leave the manager running and the shutdown marker unclean.
|
||||
let mut active_heals = self.active_heals.lock().await;
|
||||
let queue = self.heal_queue.lock().await;
|
||||
let retrying = self.retrying_heals.lock().await;
|
||||
for task in active_heals.values() {
|
||||
if root_recovery::is_admin_heal_recovery(&task.heal_type, task.source) {
|
||||
if task.get_status().await == HealTaskStatus::Completed {
|
||||
self.root_recovery.remove(&task.id, &task.heal_type, task.source).await?;
|
||||
} else {
|
||||
let mut request = match task.retry_request_with_remaining_timeout().await {
|
||||
Ok(request) => request,
|
||||
Err(Error::TaskTimeout) => {
|
||||
let mut request = task.retry_request();
|
||||
request.options.timeout = Some(Duration::ZERO);
|
||||
request
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
request.retry_attempts = task.retry_attempts;
|
||||
self.root_recovery.persist(&request).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
for request in queue.requests().chain(retrying.values().map(|retrying| &retrying.request)) {
|
||||
self.root_recovery.persist(request).await?;
|
||||
}
|
||||
// cancel all tasks
|
||||
self.cancel_token.cancel();
|
||||
drop(retrying);
|
||||
drop(queue);
|
||||
|
||||
// cancel active workers after the durable handoff
|
||||
// wait for all tasks to complete
|
||||
let mut active_heals = self.active_heals.lock().await;
|
||||
for task in active_heals.values() {
|
||||
if let Err(e) = task.cancel().await {
|
||||
warn!(
|
||||
@@ -1727,17 +1589,6 @@ impl HealManager {
|
||||
let admission_start = Instant::now();
|
||||
let source = request.source;
|
||||
let force_start = request.force_start;
|
||||
// A forceStart must not retire an old durable owner if shutdown will
|
||||
// reject its replacement. Hold the same gate through final admission.
|
||||
let _force_start_guard = if source == HealRequestSource::Admin && force_start {
|
||||
let guard = self.force_start_shutdown.lock().await;
|
||||
if self.cancel_token.is_cancelled() {
|
||||
return Err(Error::Other("Heal manager is stopping".to_string()));
|
||||
}
|
||||
Some(guard)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// HS-06 forceStart semantics (admin only): MinIO stops the old task
|
||||
// first and then starts the new one. Cancel any active admin task
|
||||
// overlapping this request's path before entering admission, so the
|
||||
@@ -1745,9 +1596,7 @@ impl HealManager {
|
||||
if request.source == HealRequestSource::Admin && request.force_start {
|
||||
let overlapping: Vec<String> = {
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
let queue = self.heal_queue.lock().await;
|
||||
let retrying = self.retrying_heals.lock().await;
|
||||
let mut ids = active_heals
|
||||
active_heals
|
||||
.iter()
|
||||
.filter(|(task_id, task)| {
|
||||
task.source == HealRequestSource::Admin
|
||||
@@ -1755,19 +1604,7 @@ impl HealManager {
|
||||
&& *task_id != &request.id
|
||||
})
|
||||
.map(|(task_id, _)| task_id.clone())
|
||||
.collect::<Vec<_>>();
|
||||
ids.extend(
|
||||
queue
|
||||
.requests()
|
||||
.chain(retrying.values().map(|retrying| &retrying.request))
|
||||
.filter(|pending| {
|
||||
root_recovery::is_admin_heal_recovery(&pending.heal_type, pending.source)
|
||||
&& heal_types_overlap(&request.heal_type, &pending.heal_type) != OverlapVerdict::Disjoint
|
||||
&& pending.id != request.id
|
||||
})
|
||||
.map(|pending| pending.id.clone()),
|
||||
);
|
||||
ids
|
||||
.collect()
|
||||
};
|
||||
for task_id in overlapping {
|
||||
match self.cancel_task(&task_id).await {
|
||||
@@ -1781,16 +1618,17 @@ impl HealManager {
|
||||
result = "force_start_cancelled_overlap",
|
||||
"Admin forceStart cancelled an overlapping heal task"
|
||||
),
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
// A failed or timed-out replay may have only its durable owner
|
||||
// left. Cancel only records that overlap this forced start.
|
||||
for pending in self.root_recovery.pending().await? {
|
||||
if pending.id != request.id
|
||||
&& heal_types_overlap(&request.heal_type, &pending.heal_type) != OverlapVerdict::Disjoint
|
||||
{
|
||||
self.cancel_task(&pending.id).await?;
|
||||
Err(err) => warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %request.id,
|
||||
cancelled_task_id = %task_id,
|
||||
error = %err,
|
||||
result = "force_start_cancel_failed",
|
||||
"Admin forceStart failed to cancel an overlapping heal task"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1803,9 +1641,6 @@ impl HealManager {
|
||||
// active -> retrying transitions can slip between duplicate checks.
|
||||
let lock_phase_start = Instant::now();
|
||||
let active_heals = self.active_heals.lock().await;
|
||||
if self.cancel_token.is_cancelled() {
|
||||
return Err(Error::Other("Heal manager is stopping".to_string()));
|
||||
}
|
||||
#[cfg(test)]
|
||||
pause_duplicate_admission_after_active_lock(&request.id).await;
|
||||
let mut queue = self.heal_queue.lock().await;
|
||||
@@ -2019,29 +1854,9 @@ impl HealManager {
|
||||
}
|
||||
}
|
||||
|
||||
let durable_handoff = root_recovery::is_admin_heal_recovery(&request.heal_type, request.source);
|
||||
let durable_handoff_required = durable_handoff
|
||||
&& (request.force_start
|
||||
|| queue.len() < config.queue_size
|
||||
|| (Self::can_displace_queued_work(&request)
|
||||
&& queue.can_displace_lower_priority_where(request.priority, Self::queued_request_can_be_displaced)));
|
||||
// An admin receipt is a control-plane responsibility. Persist it before
|
||||
// queue publication so a crash after admission can replay it.
|
||||
if durable_handoff_required {
|
||||
self.root_recovery.persist(&request).await?;
|
||||
}
|
||||
|
||||
let mut task_id = request.id.clone();
|
||||
let request_id = request.id.clone();
|
||||
let request_heal_type = request.heal_type.clone();
|
||||
let request_source = request.source;
|
||||
let admission_decision = Self::admit_request_to_queue(&mut queue, request, &config, "submit");
|
||||
let admission = admission_decision.result;
|
||||
if durable_handoff_required && !admission.is_admitted() {
|
||||
self.root_recovery
|
||||
.remove(&request_id, &request_heal_type, request_source)
|
||||
.await?;
|
||||
}
|
||||
if admission == HealAdmissionResult::Merged
|
||||
&& let Some(queued_id) = queue.queued_request_id_for_dedup_key(&dedup_key)
|
||||
{
|
||||
@@ -2112,7 +1927,7 @@ impl HealManager {
|
||||
/// Retrying, never Pending — then the queue, and finally a terminal
|
||||
/// completed entry. `heal_path` additionally constrains the map matches
|
||||
/// the way the `*_for_path` variants always have.
|
||||
async fn lookup_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> Result<TaskStateLookup> {
|
||||
async fn lookup_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> TaskStateLookup {
|
||||
let matches_path = |heal_type: &HealType| heal_path.is_none_or(|path| heal_type_matches_path(heal_type, path));
|
||||
|
||||
{
|
||||
@@ -2121,7 +1936,7 @@ impl HealManager {
|
||||
.get(canonical_task_id)
|
||||
.filter(|task| matches_path(&task.heal_type))
|
||||
{
|
||||
return Ok(TaskStateLookup::Active(Arc::clone(task)));
|
||||
return TaskStateLookup::Active(Arc::clone(task));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2131,7 +1946,7 @@ impl HealManager {
|
||||
.get(canonical_task_id)
|
||||
.filter(|retrying| matches_path(&retrying.request.heal_type))
|
||||
{
|
||||
return Ok(TaskStateLookup::Retrying(retrying.status()));
|
||||
return TaskStateLookup::Retrying(retrying.status());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2144,7 +1959,7 @@ impl HealManager {
|
||||
prune_completed_heal_statuses(&mut completed_heals);
|
||||
if let Some(completed) = completed_heals.get(canonical_task_id).filter(|c| matches_path(&c.heal_type)) {
|
||||
if completed_status_is_retrying(&completed.status) {
|
||||
return Ok(TaskStateLookup::Completed(Arc::clone(completed)));
|
||||
return TaskStateLookup::Completed(Arc::clone(completed));
|
||||
}
|
||||
terminal_completed = Some(Arc::clone(completed));
|
||||
}
|
||||
@@ -2157,7 +1972,7 @@ impl HealManager {
|
||||
None => queue.contains_request_id(canonical_task_id),
|
||||
};
|
||||
if queued {
|
||||
return Ok(TaskStateLookup::Queued);
|
||||
return TaskStateLookup::Queued;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2170,55 +1985,15 @@ impl HealManager {
|
||||
.cloned();
|
||||
}
|
||||
|
||||
if terminal_completed.is_none()
|
||||
&& let Some(completed) = self.root_recovery.completed(canonical_task_id).await?
|
||||
&& matches_path(&completed.heal_type)
|
||||
{
|
||||
terminal_completed = Some(Arc::new(completed));
|
||||
}
|
||||
|
||||
Ok(match terminal_completed {
|
||||
match terminal_completed {
|
||||
Some(completed) => TaskStateLookup::Completed(completed),
|
||||
None => TaskStateLookup::NotFound,
|
||||
})
|
||||
}
|
||||
|
||||
async fn publish_admin_terminal(
|
||||
&self,
|
||||
task_id: &str,
|
||||
heal_type: &HealType,
|
||||
source: HealRequestSource,
|
||||
completed: &CompletedHealStatus,
|
||||
) -> Result<bool> {
|
||||
self.root_recovery
|
||||
.persist_terminal(task_id, heal_type, source, completed)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn publish_admin_cancelled_terminal(
|
||||
&self,
|
||||
task_id: &str,
|
||||
heal_type: &HealType,
|
||||
source: HealRequestSource,
|
||||
) -> Result<bool> {
|
||||
let completed = CompletedHealStatus {
|
||||
outcome: None,
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: heal_type.clone(),
|
||||
status: HealTaskStatus::Cancelled,
|
||||
result_items_truncated: false,
|
||||
completed_at: SystemTime::now(),
|
||||
seqed_items: Vec::new(),
|
||||
next_seq: 0,
|
||||
min_seq: 0,
|
||||
};
|
||||
self.publish_admin_terminal(task_id, heal_type, source, &completed).await
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_task_status(&self, task_id: &str) -> Result<HealTaskStatus> {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
match self.lookup_task_state(&canonical_task_id, None).await? {
|
||||
match self.lookup_task_state(&canonical_task_id, None).await {
|
||||
TaskStateLookup::Active(task) => Ok(task.get_status().await),
|
||||
TaskStateLookup::Retrying(status) => Ok(status),
|
||||
TaskStateLookup::Completed(completed) => Ok(completed.status.clone()),
|
||||
@@ -2238,7 +2013,7 @@ impl HealManager {
|
||||
/// full-snapshot semantics.
|
||||
pub async fn get_task_report_since(&self, task_id: &str, since: Option<u64>) -> Result<HealTaskReport> {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
match self.lookup_task_state(&canonical_task_id, None).await? {
|
||||
match self.lookup_task_state(&canonical_task_id, None).await {
|
||||
TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await),
|
||||
TaskStateLookup::Retrying(status) => Ok(empty_task_report(status)),
|
||||
TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)),
|
||||
@@ -2261,7 +2036,7 @@ impl HealManager {
|
||||
since: Option<u64>,
|
||||
) -> Result<HealTaskReport> {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? {
|
||||
match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await {
|
||||
TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await),
|
||||
TaskStateLookup::Retrying(status) => Ok(empty_task_report(status)),
|
||||
TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)),
|
||||
@@ -2284,7 +2059,7 @@ impl HealManager {
|
||||
/// recently completed task, a different token is invalid for that path.
|
||||
pub async fn get_task_status_for_path(&self, heal_path: &str, task_id: &str) -> Result<HealTaskStatus> {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? {
|
||||
match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await {
|
||||
TaskStateLookup::Active(task) => Ok(task.get_status().await),
|
||||
TaskStateLookup::Retrying(status) => Ok(status),
|
||||
TaskStateLookup::Completed(completed) => Ok(completed.status.clone()),
|
||||
@@ -2338,18 +2113,11 @@ impl HealManager {
|
||||
}
|
||||
drop(completed_heals);
|
||||
|
||||
{
|
||||
let mut displaced_terminals = lock_displaced_terminals(&self.displaced_terminals);
|
||||
prune_completed_heal_statuses(&mut displaced_terminals);
|
||||
if displaced_terminals
|
||||
.values()
|
||||
.any(|terminal| heal_type_matches_path(&terminal.heal_type, heal_path))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
self.root_recovery.completed_matches_path(heal_path).await.unwrap_or(false)
|
||||
let mut displaced_terminals = lock_displaced_terminals(&self.displaced_terminals);
|
||||
prune_completed_heal_statuses(&mut displaced_terminals);
|
||||
displaced_terminals
|
||||
.values()
|
||||
.any(|terminal| heal_type_matches_path(&terminal.heal_type, heal_path))
|
||||
}
|
||||
|
||||
/// Get task progress
|
||||
@@ -2361,7 +2129,7 @@ impl HealManager {
|
||||
|
||||
pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
let progress = match self.lookup_task_state(&canonical_task_id, None).await? {
|
||||
let progress = match self.lookup_task_state(&canonical_task_id, None).await {
|
||||
TaskStateLookup::Active(task) => Some(task.get_progress().await),
|
||||
TaskStateLookup::Completed(completed) => completed.progress.clone(),
|
||||
_ => None,
|
||||
@@ -2377,11 +2145,8 @@ impl HealManager {
|
||||
{
|
||||
let mut active_heals = self.active_heals.lock().await;
|
||||
if let Some(task) = active_heals.get(&canonical_task_id) {
|
||||
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
|
||||
self.publish_admin_terminal(&canonical_task_id, &task.heal_type, task.source, &completed)
|
||||
.await?;
|
||||
self.root_recovery.remove(&task.id, &task.heal_type, task.source).await?;
|
||||
task.cancel().await?;
|
||||
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
|
||||
publish_completed_heal(&self.completed_heals, &self.task_aliases, &canonical_task_id, completed, true).await;
|
||||
active_heals.remove(&canonical_task_id);
|
||||
publish_active_heal_count(&active_heals);
|
||||
@@ -2403,13 +2168,6 @@ impl HealManager {
|
||||
|
||||
{
|
||||
let mut retrying_heals = self.retrying_heals.lock().await;
|
||||
if let Some(retrying) = retrying_heals.get(&canonical_task_id) {
|
||||
self.publish_admin_cancelled_terminal(&canonical_task_id, &retrying.request.heal_type, retrying.request.source)
|
||||
.await?;
|
||||
self.root_recovery
|
||||
.remove(&canonical_task_id, &retrying.request.heal_type, retrying.request.source)
|
||||
.await?;
|
||||
}
|
||||
if let Some(retrying) = retrying_heals.remove(&canonical_task_id) {
|
||||
retrying.cancel_token.cancel();
|
||||
drop(retrying_heals);
|
||||
@@ -2430,13 +2188,6 @@ impl HealManager {
|
||||
}
|
||||
|
||||
let mut queue = self.heal_queue.lock().await;
|
||||
if let Some(request) = queue.requests().find(|request| request.id == canonical_task_id) {
|
||||
self.publish_admin_cancelled_terminal(&canonical_task_id, &request.heal_type, request.source)
|
||||
.await?;
|
||||
self.root_recovery
|
||||
.remove(&request.id, &request.heal_type, request.source)
|
||||
.await?;
|
||||
}
|
||||
if queue.remove_request_id(&canonical_task_id).is_some() {
|
||||
publish_heal_queue_length(&queue);
|
||||
info!(
|
||||
@@ -2454,10 +2205,6 @@ impl HealManager {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
drop(queue);
|
||||
if self.root_recovery.cancel_pending(&canonical_task_id).await? {
|
||||
return Ok(());
|
||||
}
|
||||
Err(Error::TaskNotFound {
|
||||
task_id: task_id.to_string(),
|
||||
})
|
||||
@@ -2476,11 +2223,8 @@ impl HealManager {
|
||||
|
||||
for task_id in &task_ids {
|
||||
if let Some(task) = active_heals.get(task_id) {
|
||||
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
|
||||
self.publish_admin_terminal(task_id, &task.heal_type, task.source, &completed)
|
||||
.await?;
|
||||
self.root_recovery.remove(&task.id, &task.heal_type, task.source).await?;
|
||||
task.cancel().await?;
|
||||
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
|
||||
publish_completed_heal(&self.completed_heals, &self.task_aliases, task_id, completed, true).await;
|
||||
}
|
||||
active_heals.remove(task_id);
|
||||
@@ -2507,13 +2251,6 @@ impl HealManager {
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
for task_id in &task_ids {
|
||||
if let Some(retrying) = retrying_heals.get(task_id) {
|
||||
self.publish_admin_cancelled_terminal(task_id, &retrying.request.heal_type, retrying.request.source)
|
||||
.await?;
|
||||
self.root_recovery
|
||||
.remove(task_id, &retrying.request.heal_type, retrying.request.source)
|
||||
.await?;
|
||||
}
|
||||
if let Some(retrying) = retrying_heals.remove(task_id) {
|
||||
retrying.cancel_token.cancel();
|
||||
cancelled += 1;
|
||||
@@ -2535,21 +2272,6 @@ impl HealManager {
|
||||
}
|
||||
}
|
||||
|
||||
let queued_matches = {
|
||||
let queue = self.heal_queue.lock().await;
|
||||
queue
|
||||
.requests()
|
||||
.filter(|request| heal_type_matches_path(&request.heal_type, heal_path))
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
for request in &queued_matches {
|
||||
self.publish_admin_cancelled_terminal(&request.id, &request.heal_type, request.source)
|
||||
.await?;
|
||||
self.root_recovery
|
||||
.remove(&request.id, &request.heal_type, request.source)
|
||||
.await?;
|
||||
}
|
||||
let mut queue = self.heal_queue.lock().await;
|
||||
let queued_cancelled = queue.remove_matching(|request| heal_type_matches_path(&request.heal_type, heal_path));
|
||||
if !queued_cancelled.is_empty() {
|
||||
@@ -2562,17 +2284,6 @@ impl HealManager {
|
||||
self.remove_mrf_repair_notice_targets_for_task(&request.id);
|
||||
}
|
||||
|
||||
for pending in self
|
||||
.root_recovery
|
||||
.pending()
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|pending| heal_type_matches_path(&pending.heal_type, heal_path))
|
||||
{
|
||||
if self.root_recovery.cancel_pending(&pending.id).await? {
|
||||
cancelled += 1;
|
||||
}
|
||||
}
|
||||
if cancelled == 0 {
|
||||
return Err(Error::TaskNotFound {
|
||||
task_id: heal_path.to_string(),
|
||||
@@ -2684,7 +2395,6 @@ impl std::fmt::Debug for HealManager {
|
||||
|
||||
mod auto_scan;
|
||||
mod queue;
|
||||
mod root_recovery;
|
||||
mod scheduler;
|
||||
mod unclean_shutdown;
|
||||
|
||||
|
||||
@@ -288,29 +288,16 @@ impl PriorityHealQueue {
|
||||
QueuePushOutcome::Accepted
|
||||
}
|
||||
|
||||
pub(super) fn can_displace_lower_priority_where<F>(&self, priority: HealPriority, can_displace: F) -> bool
|
||||
where
|
||||
F: Fn(&HealRequest) -> bool,
|
||||
{
|
||||
self.heap
|
||||
.iter()
|
||||
.any(|item| item.priority < priority && can_displace(&item.request))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn can_displace_lower_priority(&self, priority: HealPriority) -> bool {
|
||||
self.can_displace_lower_priority_where(priority, |_| true)
|
||||
self.heap.iter().any(|item| item.priority < priority)
|
||||
}
|
||||
|
||||
pub(super) fn push_displacing_lower_priority_where<F>(&mut self, request: HealRequest, can_displace: F) -> Option<HealRequest>
|
||||
where
|
||||
F: Fn(&HealRequest) -> bool,
|
||||
{
|
||||
pub(super) fn push_displacing_lower_priority(&mut self, request: HealRequest) -> Option<HealRequest> {
|
||||
let mut retained = BinaryHeap::new();
|
||||
let mut displaced: Option<PriorityQueueItem> = None;
|
||||
|
||||
while let Some(item) = self.heap.pop() {
|
||||
if item.priority < request.priority && can_displace(&item.request) {
|
||||
if item.priority < request.priority {
|
||||
let should_displace = displaced
|
||||
.as_ref()
|
||||
.map(|current| {
|
||||
@@ -350,11 +337,6 @@ impl PriorityHealQueue {
|
||||
displaced
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn push_displacing_lower_priority(&mut self, request: HealRequest) -> Option<HealRequest> {
|
||||
self.push_displacing_lower_priority_where(request, |_| true)
|
||||
}
|
||||
|
||||
/// Get statistics about queue contents by priority
|
||||
pub(super) fn get_priority_stats(&self) -> HashMap<HealPriority, usize> {
|
||||
let mut stats = HashMap::new();
|
||||
|
||||
@@ -1,938 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Graceful-shutdown handoff for administrator heals. This namespace is
|
||||
//! separate from erasure-set checkpoints and replacement generations, which
|
||||
//! cannot represent an admitted admin control-plane request. One coordinator
|
||||
//! disk owns each record; never create a fallback copy after an uncertain write
|
||||
//! or deletion.
|
||||
|
||||
use super::*;
|
||||
use crate::heal::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes};
|
||||
use crate::heal::{DiskStore, RUSTFS_META_BUCKET};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
// The metadata bucket already exists and its parent is durable. Creating a
|
||||
// nested journal directory here would also require syncing every ancestor.
|
||||
const ROOT_RECOVERY_PREFIX: &str = "root-heal-";
|
||||
const ROOT_TERMINAL_PREFIX: &str = "terminal-root-heal-";
|
||||
const LEGACY_ROOT_RECOVERY_SCHEMA: u32 = 1;
|
||||
const ROOT_RECOVERY_SCHEMA: u32 = 2;
|
||||
const ROOT_TERMINAL_SCHEMA: u32 = 1;
|
||||
const ROOT_TERMINAL_GC_SCAN_BUDGET: usize = 1024;
|
||||
const ROOT_TERMINAL_GC_DELETE_BUDGET: usize = 64;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
enum RecoveryHealType {
|
||||
Cluster,
|
||||
Bucket {
|
||||
bucket: String,
|
||||
},
|
||||
Object {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<String>,
|
||||
},
|
||||
Prefix {
|
||||
bucket: String,
|
||||
prefix: String,
|
||||
},
|
||||
ErasureSet {
|
||||
buckets: Vec<String>,
|
||||
set_disk_id: String,
|
||||
},
|
||||
Metadata {
|
||||
bucket: String,
|
||||
object: String,
|
||||
},
|
||||
EcDecode {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
impl RecoveryHealType {
|
||||
fn validate(&self) -> Result<()> {
|
||||
match self {
|
||||
Self::Cluster => {}
|
||||
Self::Bucket { bucket } => validate_recovery_component("bucket", bucket)?,
|
||||
Self::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
}
|
||||
| Self::EcDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
validate_recovery_component("bucket", bucket)?;
|
||||
validate_recovery_component("object", object)?;
|
||||
if let Some(version_id) = version_id {
|
||||
validate_recovery_component("version id", version_id)?;
|
||||
}
|
||||
}
|
||||
Self::Prefix { bucket, prefix } => {
|
||||
validate_recovery_component("bucket", bucket)?;
|
||||
validate_recovery_component("prefix", prefix)?;
|
||||
}
|
||||
Self::ErasureSet { buckets, set_disk_id } => {
|
||||
validate_recovery_component("set disk id", set_disk_id)?;
|
||||
if buckets.is_empty() {
|
||||
return Err(Error::Other("Admin heal recovery erasure set must name buckets".to_string()));
|
||||
}
|
||||
for bucket in buckets {
|
||||
validate_recovery_component("bucket", bucket)?;
|
||||
}
|
||||
}
|
||||
Self::Metadata { bucket, object } => {
|
||||
validate_recovery_component("bucket", bucket)?;
|
||||
validate_recovery_component("object", object)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&HealType> for RecoveryHealType {
|
||||
fn from(heal_type: &HealType) -> Self {
|
||||
match heal_type {
|
||||
HealType::Cluster => Self::Cluster,
|
||||
HealType::Bucket { bucket } => Self::Bucket { bucket: bucket.clone() },
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => Self::Object {
|
||||
bucket: bucket.clone(),
|
||||
object: object.clone(),
|
||||
version_id: version_id.clone(),
|
||||
},
|
||||
HealType::Prefix { bucket, prefix } => Self::Prefix {
|
||||
bucket: bucket.clone(),
|
||||
prefix: prefix.clone(),
|
||||
},
|
||||
HealType::ErasureSet { buckets, set_disk_id } => Self::ErasureSet {
|
||||
buckets: buckets.clone(),
|
||||
set_disk_id: set_disk_id.clone(),
|
||||
},
|
||||
HealType::Metadata { bucket, object } => Self::Metadata {
|
||||
bucket: bucket.clone(),
|
||||
object: object.clone(),
|
||||
},
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => Self::EcDecode {
|
||||
bucket: bucket.clone(),
|
||||
object: object.clone(),
|
||||
version_id: version_id.clone(),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<RecoveryHealType> for HealType {
|
||||
fn from(heal_type: RecoveryHealType) -> Self {
|
||||
match heal_type {
|
||||
RecoveryHealType::Cluster => Self::Cluster,
|
||||
RecoveryHealType::Bucket { bucket } => Self::Bucket { bucket },
|
||||
RecoveryHealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => Self::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
},
|
||||
RecoveryHealType::Prefix { bucket, prefix } => Self::Prefix { bucket, prefix },
|
||||
RecoveryHealType::ErasureSet { buckets, set_disk_id } => Self::ErasureSet { buckets, set_disk_id },
|
||||
RecoveryHealType::Metadata { bucket, object } => Self::Metadata { bucket, object },
|
||||
RecoveryHealType::EcDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => Self::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_recovery_component(label: &str, value: &str) -> Result<()> {
|
||||
if value.is_empty() || value.contains('\0') {
|
||||
return Err(Error::Other(format!("Invalid admin heal recovery {label}")));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn default_recovery_heal_type() -> RecoveryHealType {
|
||||
RecoveryHealType::Cluster
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct RootHealIntent {
|
||||
schema: u32,
|
||||
task_id: String,
|
||||
#[serde(default = "default_recovery_heal_type")]
|
||||
heal_type: RecoveryHealType,
|
||||
#[serde(deserialize_with = "decode_options")]
|
||||
options: HealOptions,
|
||||
priority: HealPriority,
|
||||
retry_attempts: u32,
|
||||
created_at: SystemTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct RootHealTerminal {
|
||||
schema: u32,
|
||||
task_id: String,
|
||||
heal_type: RecoveryHealType,
|
||||
status: HealTaskStatus,
|
||||
progress: Option<HealProgress>,
|
||||
completed_at: SystemTime,
|
||||
}
|
||||
|
||||
impl RootHealTerminal {
|
||||
fn from_completed(task_id: &str, completed: &CompletedHealStatus) -> Self {
|
||||
Self {
|
||||
schema: ROOT_TERMINAL_SCHEMA,
|
||||
task_id: task_id.to_owned(),
|
||||
heal_type: RecoveryHealType::from(&completed.heal_type),
|
||||
status: completed.status.clone(),
|
||||
progress: completed.progress.clone(),
|
||||
completed_at: completed.completed_at,
|
||||
}
|
||||
}
|
||||
|
||||
fn cancelled(task_id: &str, heal_type: &HealType) -> Self {
|
||||
Self {
|
||||
schema: ROOT_TERMINAL_SCHEMA,
|
||||
task_id: task_id.to_owned(),
|
||||
heal_type: RecoveryHealType::from(heal_type),
|
||||
status: HealTaskStatus::Cancelled,
|
||||
progress: None,
|
||||
completed_at: SystemTime::now(),
|
||||
}
|
||||
}
|
||||
|
||||
fn into_completed(self) -> CompletedHealStatus {
|
||||
CompletedHealStatus {
|
||||
outcome: None,
|
||||
progress: self.progress,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: self.heal_type.into(),
|
||||
status: self.status,
|
||||
result_items_truncated: false,
|
||||
completed_at: self.completed_at,
|
||||
seqed_items: Vec::new(),
|
||||
next_seq: 0,
|
||||
min_seq: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn retained_at(&self, now: SystemTime) -> bool {
|
||||
now.duration_since(self.completed_at)
|
||||
.map(|age| age <= KEEP_HEAL_TASK_STATUS_DURATION)
|
||||
.unwrap_or(true)
|
||||
}
|
||||
}
|
||||
|
||||
impl RootHealIntent {
|
||||
fn from_request(request: &HealRequest) -> Self {
|
||||
Self {
|
||||
schema: ROOT_RECOVERY_SCHEMA,
|
||||
task_id: request.id.clone(),
|
||||
heal_type: RecoveryHealType::from(&request.heal_type),
|
||||
options: request.options.clone(),
|
||||
priority: request.priority,
|
||||
retry_attempts: request.retry_attempts,
|
||||
created_at: request.created_at,
|
||||
}
|
||||
}
|
||||
|
||||
fn into_request(self) -> HealRequest {
|
||||
let mut request = HealRequest::new(self.heal_type.into(), self.options, self.priority);
|
||||
request.id = self.task_id;
|
||||
request.source = HealRequestSource::Admin;
|
||||
request.retry_attempts = self.retry_attempts;
|
||||
request.created_at = self.created_at;
|
||||
request
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct RootHealRecovery {
|
||||
mutation: Mutex<()>,
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
disabled_for_tests: bool,
|
||||
#[cfg(test)]
|
||||
disks: Option<Vec<DiskStore>>,
|
||||
}
|
||||
|
||||
pub(super) fn is_admin_heal_recovery(heal_type: &HealType, source: HealRequestSource) -> bool {
|
||||
source == HealRequestSource::Admin
|
||||
&& matches!(
|
||||
heal_type,
|
||||
HealType::Cluster
|
||||
| HealType::Bucket { .. }
|
||||
| HealType::Object { .. }
|
||||
| HealType::Prefix { .. }
|
||||
| HealType::ErasureSet { .. }
|
||||
| HealType::Metadata { .. }
|
||||
| HealType::ECDecode { .. }
|
||||
)
|
||||
}
|
||||
|
||||
fn decode_options<'de, D: serde::Deserializer<'de>>(deserializer: D) -> std::result::Result<HealOptions, D::Error> {
|
||||
let value = serde_json::Value::deserialize(deserializer)?;
|
||||
let object = value
|
||||
.as_object()
|
||||
.ok_or_else(|| serde::de::Error::custom("root heal options must be an object"))?;
|
||||
const FIELDS: &[&str] = &[
|
||||
"scan_mode",
|
||||
"remove_corrupted",
|
||||
"recreate_missing",
|
||||
"update_parity",
|
||||
"recursive",
|
||||
"dry_run",
|
||||
"no_lock",
|
||||
"timeout",
|
||||
"pool_index",
|
||||
"set_index",
|
||||
];
|
||||
if object.keys().any(|key| !FIELDS.contains(&key.as_str())) {
|
||||
return Err(serde::de::Error::custom("unknown root heal recovery option"));
|
||||
}
|
||||
let options: HealOptions = serde_json::from_value(value).map_err(serde::de::Error::custom)?;
|
||||
if options.no_lock {
|
||||
return Err(serde::de::Error::custom("administrator root heal cannot skip namespace locking"));
|
||||
}
|
||||
Ok(options)
|
||||
}
|
||||
|
||||
fn intent_path(task_id: &str) -> Result<String> {
|
||||
let parsed = uuid::Uuid::parse_str(task_id).map_err(|_| Error::Other("Invalid root heal recovery task id".to_string()))?;
|
||||
if parsed.to_string() != task_id {
|
||||
return Err(Error::Other("Noncanonical root heal recovery task id".to_string()));
|
||||
}
|
||||
Ok(format!("{ROOT_RECOVERY_PREFIX}{task_id}.json"))
|
||||
}
|
||||
|
||||
fn terminal_path(task_id: &str) -> Result<String> {
|
||||
let parsed = uuid::Uuid::parse_str(task_id).map_err(|_| Error::Other("Invalid root heal terminal task id".to_string()))?;
|
||||
if parsed.to_string() != task_id {
|
||||
return Err(Error::Other("Noncanonical root heal terminal task id".to_string()));
|
||||
}
|
||||
Ok(format!("{ROOT_TERMINAL_PREFIX}{task_id}.json"))
|
||||
}
|
||||
|
||||
fn decode_intent(task_id: &str, bytes: &[u8]) -> Result<RootHealIntent> {
|
||||
let _ = intent_path(task_id)?;
|
||||
let intent: RootHealIntent = serde_json::from_slice(bytes)
|
||||
.map_err(|error| Error::Other(format!("Invalid root heal recovery record {task_id}: {error}")))?;
|
||||
if intent.task_id != task_id {
|
||||
return Err(Error::Other(format!("Unsupported or mismatched root heal recovery record {task_id}")));
|
||||
}
|
||||
match intent.schema {
|
||||
LEGACY_ROOT_RECOVERY_SCHEMA if intent.heal_type == RecoveryHealType::Cluster => {}
|
||||
ROOT_RECOVERY_SCHEMA => {}
|
||||
_ => return Err(Error::Other(format!("Unsupported or mismatched root heal recovery record {task_id}"))),
|
||||
}
|
||||
intent.heal_type.validate()?;
|
||||
Ok(intent)
|
||||
}
|
||||
|
||||
fn decode_terminal(task_id: &str, bytes: &[u8]) -> Result<RootHealTerminal> {
|
||||
let _ = terminal_path(task_id)?;
|
||||
let terminal: RootHealTerminal = serde_json::from_slice(bytes)
|
||||
.map_err(|error| Error::Other(format!("Invalid root heal terminal record {task_id}: {error}")))?;
|
||||
if terminal.schema != ROOT_TERMINAL_SCHEMA || terminal.task_id != task_id {
|
||||
return Err(Error::Other(format!("Unsupported or mismatched root heal terminal record {task_id}")));
|
||||
}
|
||||
terminal.heal_type.validate()?;
|
||||
if !matches!(
|
||||
terminal.status,
|
||||
HealTaskStatus::Completed | HealTaskStatus::Cancelled | HealTaskStatus::Failed { .. }
|
||||
) {
|
||||
return Err(Error::Other(format!("Non-terminal root heal receipt {task_id}")));
|
||||
}
|
||||
Ok(terminal)
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) struct RootTerminalGcReport {
|
||||
pub(super) scanned: usize,
|
||||
pub(super) retained: usize,
|
||||
pub(super) pending_removed: usize,
|
||||
pub(super) terminals_removed: usize,
|
||||
pub(super) budget_exhausted: bool,
|
||||
}
|
||||
|
||||
impl RootHealRecovery {
|
||||
#[cfg(test)]
|
||||
pub(super) fn with_disks(disks: Vec<DiskStore>) -> Self {
|
||||
Self {
|
||||
mutation: Mutex::new(()),
|
||||
disabled_for_tests: false,
|
||||
disks: Some(disks),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(super) fn disabled_for_tests() -> Self {
|
||||
Self {
|
||||
mutation: Mutex::new(()),
|
||||
disabled_for_tests: true,
|
||||
#[cfg(test)]
|
||||
disks: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn disks(&self) -> Result<Vec<DiskStore>> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if self.disabled_for_tests {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
#[cfg(test)]
|
||||
if let Some(disks) = &self.disks {
|
||||
return Ok(disks.clone());
|
||||
}
|
||||
let map = local_disk_map_read().await;
|
||||
if map.values().any(Option::is_none) {
|
||||
return Err(Error::Other("Root heal recovery owner may be on an unavailable local disk".to_string()));
|
||||
}
|
||||
let mut disks = map.values().flatten().cloned().collect::<Vec<_>>();
|
||||
disks.sort_by_key(|disk| EcstoreDiskAPI::endpoint(disk.as_ref()).to_string());
|
||||
Ok(disks)
|
||||
}
|
||||
|
||||
async fn find(disks: &[DiskStore], task_id: &str) -> Result<Option<(DiskStore, EcstoreDiskBytes)>> {
|
||||
let path = intent_path(task_id)?;
|
||||
let mut found = None;
|
||||
for disk in disks {
|
||||
// read_all reports FileNotFound even when the whole metadata
|
||||
// volume is absent; that is an unknown owner, not empty state.
|
||||
EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?;
|
||||
match EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, &path).await {
|
||||
Ok(bytes) => {
|
||||
decode_intent(task_id, &bytes)?;
|
||||
if found.is_some() {
|
||||
return Err(Error::Other(format!("Multiple root heal recovery owners for {task_id}")));
|
||||
}
|
||||
found = Some((disk.clone(), bytes));
|
||||
}
|
||||
Err(DiskError::FileNotFound) => {}
|
||||
Err(error) => return Err(Error::Disk(error)),
|
||||
}
|
||||
}
|
||||
Ok(found)
|
||||
}
|
||||
|
||||
async fn find_terminal(disks: &[DiskStore], task_id: &str) -> Result<Option<(DiskStore, EcstoreDiskBytes)>> {
|
||||
let path = terminal_path(task_id)?;
|
||||
let mut found = None;
|
||||
for disk in disks {
|
||||
EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?;
|
||||
match EcstoreDiskAPI::read_all(disk.as_ref(), RUSTFS_META_BUCKET, &path).await {
|
||||
Ok(bytes) => {
|
||||
decode_terminal(task_id, &bytes)?;
|
||||
if found.is_some() {
|
||||
return Err(Error::Other(format!("Multiple root heal terminal owners for {task_id}")));
|
||||
}
|
||||
found = Some((disk.clone(), bytes));
|
||||
}
|
||||
Err(DiskError::FileNotFound) => {}
|
||||
Err(error) => return Err(Error::Disk(error)),
|
||||
}
|
||||
}
|
||||
Ok(found)
|
||||
}
|
||||
|
||||
async fn find_retained_terminal(
|
||||
disks: &[DiskStore],
|
||||
task_id: &str,
|
||||
now: SystemTime,
|
||||
) -> Result<Option<(DiskStore, EcstoreDiskBytes)>> {
|
||||
let Some((disk, bytes)) = Self::find_terminal(disks, task_id).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
if decode_terminal(task_id, &bytes)?.retained_at(now) {
|
||||
return Ok(Some((disk, bytes)));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn persist_terminal_locked(
|
||||
disks: &[DiskStore],
|
||||
task_id: &str,
|
||||
terminal: RootHealTerminal,
|
||||
) -> Result<Option<(DiskStore, EcstoreDiskBytes)>> {
|
||||
let path = terminal_path(task_id)?;
|
||||
if let Some((_, bytes)) = Self::find_terminal(disks, task_id).await? {
|
||||
let current = decode_terminal(task_id, &bytes)?;
|
||||
if current == terminal {
|
||||
return Self::find(disks, task_id).await;
|
||||
}
|
||||
return Err(Error::Other(format!("Root heal terminal record changed for {task_id}")));
|
||||
}
|
||||
let pending = Self::find(disks, task_id).await?;
|
||||
let disk = pending
|
||||
.as_ref()
|
||||
.map(|(disk, _)| disk.clone())
|
||||
.or_else(|| disks.first().cloned())
|
||||
.ok_or_else(|| Error::Other("No local disk available for root heal terminal receipt".to_string()))?;
|
||||
let bytes = serde_json::to_vec(&terminal)
|
||||
.map_err(|error| Error::Other(format!("Serialize root heal terminal receipt: {error}")))?;
|
||||
match EcstoreDiskAPI::compare_and_update_file(disk.as_ref(), RUSTFS_META_BUCKET, &path, None, Some(bytes.into())).await? {
|
||||
EcstoreConditionalFileUpdate::Updated => Ok(pending),
|
||||
_ => Err(Error::Other(format!("Root heal terminal record changed for {task_id}"))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn persist(&self, request: &HealRequest) -> Result<()> {
|
||||
if !is_admin_heal_recovery(&request.heal_type, request.source) {
|
||||
return Ok(());
|
||||
}
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if self.disabled_for_tests {
|
||||
return Ok(());
|
||||
}
|
||||
let _guard = self.mutation.lock().await;
|
||||
let disks = self.disks().await?;
|
||||
let existing = Self::find(&disks, &request.id).await?;
|
||||
let (disk, expected) = match existing {
|
||||
Some((disk, bytes)) => (disk, Some(bytes)),
|
||||
None => {
|
||||
let disk = disks
|
||||
.first()
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::Other("No local disk available for root heal shutdown recovery".to_string()))?;
|
||||
(disk, None)
|
||||
}
|
||||
};
|
||||
if request.options.no_lock {
|
||||
return Err(Error::Other("Administrator root heal cannot skip namespace locking".to_string()));
|
||||
}
|
||||
let bytes = serde_json::to_vec(&RootHealIntent::from_request(request))
|
||||
.map_err(|error| Error::Other(format!("Serialize root heal recovery record: {error}")))?;
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&intent_path(&request.id)?,
|
||||
expected,
|
||||
Some(bytes.into()),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
EcstoreConditionalFileUpdate::Updated => Ok(()),
|
||||
_ => Err(Error::Other(format!("Root heal recovery record changed for {}", request.id))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn remove(&self, task_id: &str, heal_type: &HealType, source: HealRequestSource) -> Result<bool> {
|
||||
if !is_admin_heal_recovery(heal_type, source) {
|
||||
return Ok(false);
|
||||
}
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if self.disabled_for_tests {
|
||||
return Ok(false);
|
||||
}
|
||||
self.remove_pending_by_id(task_id).await
|
||||
}
|
||||
|
||||
async fn remove_pending_by_id(&self, task_id: &str) -> Result<bool> {
|
||||
let _guard = self.mutation.lock().await;
|
||||
let Some((disk, bytes)) = Self::find(&self.disks().await?, task_id).await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&intent_path(task_id)?,
|
||||
Some(bytes),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
EcstoreConditionalFileUpdate::Updated => Ok(true),
|
||||
_ => Err(Error::Other(format!("Root heal recovery record changed while retiring {task_id}"))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn checkpoint_failed_execution(&self, task: &HealTask) -> Result<()> {
|
||||
if !is_admin_heal_recovery(&task.heal_type, task.source) {
|
||||
return Ok(());
|
||||
}
|
||||
let remaining = match task.retry_request_with_remaining_timeout().await {
|
||||
Ok(request) => request.options.timeout,
|
||||
Err(Error::TaskTimeout) => Some(Duration::ZERO),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let _guard = self.mutation.lock().await;
|
||||
let Some((disk, expected)) = Self::find(&self.disks().await?, &task.id).await? else {
|
||||
// A first execution that failed has no restart handoff to update.
|
||||
return Ok(());
|
||||
};
|
||||
let mut intent = decode_intent(&task.id, &expected)?;
|
||||
if HealType::from(intent.heal_type.clone()) != task.heal_type {
|
||||
return Err(Error::Other(format!("Root heal recovery owner changed for {}", task.id)));
|
||||
}
|
||||
let mut expected_options = intent.options.clone();
|
||||
expected_options.timeout = task.options.timeout;
|
||||
if intent.created_at != task.created_at || intent.priority != task.priority || expected_options != task.options {
|
||||
return Err(Error::Other(format!("Root heal recovery owner changed for {}", task.id)));
|
||||
}
|
||||
// A terminal timeout leaves no runtime owner for stop() to snapshot.
|
||||
// Checkpoint its consumed budget before publishing terminal status;
|
||||
// never refund time if an earlier checkpoint is already stricter.
|
||||
intent.options.timeout = match (intent.options.timeout, remaining) {
|
||||
(Some(previous), Some(remaining)) => Some(previous.min(remaining)),
|
||||
(previous, remaining) => previous.or(remaining),
|
||||
};
|
||||
intent.retry_attempts = intent.retry_attempts.max(task.retry_attempts);
|
||||
let bytes = serde_json::to_vec(&intent)
|
||||
.map_err(|error| Error::Other(format!("Serialize root heal recovery checkpoint: {error}")))?;
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&intent_path(&task.id)?,
|
||||
Some(expected),
|
||||
Some(bytes.into()),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
EcstoreConditionalFileUpdate::Updated => Ok(()),
|
||||
_ => Err(Error::Other(format!("Root heal recovery record changed while checkpointing {}", task.id))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn cancel_pending(&self, task_id: &str) -> Result<bool> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if self.disabled_for_tests {
|
||||
return Ok(false);
|
||||
}
|
||||
if intent_path(task_id).is_err() {
|
||||
return Ok(false);
|
||||
}
|
||||
let _guard = self.mutation.lock().await;
|
||||
let disks = self.disks().await?;
|
||||
if Self::find_terminal(&disks, task_id).await?.is_some() {
|
||||
if let Some((disk, bytes)) = Self::find(&disks, task_id).await? {
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&intent_path(task_id)?,
|
||||
Some(bytes),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
EcstoreConditionalFileUpdate::Updated => {}
|
||||
_ => return Err(Error::Other(format!("Root heal recovery record changed while cancelling {task_id}"))),
|
||||
}
|
||||
}
|
||||
return Ok(true);
|
||||
}
|
||||
let Some((disk, bytes)) = Self::find(&disks, task_id).await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
let pending = decode_intent(task_id, &bytes)?;
|
||||
let heal_type = HealType::from(pending.heal_type);
|
||||
let terminal = RootHealTerminal::cancelled(task_id, &heal_type);
|
||||
let _ = Self::persist_terminal_locked(&disks, task_id, terminal).await?;
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&intent_path(task_id)?,
|
||||
Some(bytes),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
EcstoreConditionalFileUpdate::Updated => Ok(true),
|
||||
_ => Err(Error::Other(format!("Root heal recovery record changed while cancelling {task_id}"))),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn persist_terminal(
|
||||
&self,
|
||||
task_id: &str,
|
||||
heal_type: &HealType,
|
||||
source: HealRequestSource,
|
||||
completed: &CompletedHealStatus,
|
||||
) -> Result<bool> {
|
||||
if !is_admin_heal_recovery(heal_type, source) || completed.heal_type != *heal_type {
|
||||
return Ok(false);
|
||||
}
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if self.disabled_for_tests {
|
||||
return Ok(false);
|
||||
}
|
||||
let _guard = self.mutation.lock().await;
|
||||
let disks = self.disks().await?;
|
||||
let pending =
|
||||
Self::persist_terminal_locked(&disks, task_id, RootHealTerminal::from_completed(task_id, completed)).await?;
|
||||
if let Some((disk, bytes)) = pending {
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&intent_path(task_id)?,
|
||||
Some(bytes),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
EcstoreConditionalFileUpdate::Updated => {}
|
||||
_ => {
|
||||
return Err(Error::Other(format!(
|
||||
"Root heal recovery record changed while publishing terminal {task_id}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(super) async fn completed(&self, task_id: &str) -> Result<Option<CompletedHealStatus>> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if self.disabled_for_tests {
|
||||
return Ok(None);
|
||||
}
|
||||
if terminal_path(task_id).is_err() {
|
||||
return Ok(None);
|
||||
}
|
||||
let _guard = self.mutation.lock().await;
|
||||
let disks = self.disks().await?;
|
||||
let Some((_, bytes)) = Self::find_retained_terminal(&disks, task_id, SystemTime::now()).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(decode_terminal(task_id, &bytes)?.into_completed()))
|
||||
}
|
||||
|
||||
pub(super) async fn completed_matches_path(&self, heal_path: &str) -> Result<bool> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if self.disabled_for_tests {
|
||||
return Ok(false);
|
||||
}
|
||||
let _guard = self.mutation.lock().await;
|
||||
let disks = self.disks().await?;
|
||||
for disk in &disks {
|
||||
EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?;
|
||||
let entries = match EcstoreDiskAPI::list_dir(disk.as_ref(), "", RUSTFS_META_BUCKET, "", -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound) => continue,
|
||||
Err(error) => return Err(Error::Disk(error)),
|
||||
};
|
||||
for entry in entries {
|
||||
let Some(task_id) = entry
|
||||
.strip_prefix(ROOT_TERMINAL_PREFIX)
|
||||
.and_then(|entry| entry.strip_suffix(".json"))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let Some((_, bytes)) = Self::find_retained_terminal(&disks, task_id, SystemTime::now()).await? else {
|
||||
continue;
|
||||
};
|
||||
let heal_type = HealType::from(decode_terminal(task_id, &bytes)?.heal_type);
|
||||
if heal_type_matches_path(&heal_type, heal_path) {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
pub(super) async fn gc_terminal_receipts_once(&self, now: SystemTime) -> Result<RootTerminalGcReport> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if self.disabled_for_tests {
|
||||
return Ok(RootTerminalGcReport::default());
|
||||
}
|
||||
let _guard = self.mutation.lock().await;
|
||||
let disks = self.disks().await?;
|
||||
let mut report = RootTerminalGcReport::default();
|
||||
let mut ids = HashSet::new();
|
||||
for disk in &disks {
|
||||
if report.scanned >= ROOT_TERMINAL_GC_SCAN_BUDGET {
|
||||
report.budget_exhausted = true;
|
||||
break;
|
||||
}
|
||||
EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?;
|
||||
let remaining = ROOT_TERMINAL_GC_SCAN_BUDGET.saturating_sub(report.scanned);
|
||||
let count = i32::try_from(remaining).unwrap_or(i32::MAX);
|
||||
let mut entries = match EcstoreDiskAPI::list_dir(disk.as_ref(), "", RUSTFS_META_BUCKET, "", count).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound) => continue,
|
||||
Err(error) => return Err(Error::Disk(error)),
|
||||
};
|
||||
entries.sort_unstable();
|
||||
for entry in entries {
|
||||
if report.scanned >= ROOT_TERMINAL_GC_SCAN_BUDGET {
|
||||
report.budget_exhausted = true;
|
||||
break;
|
||||
}
|
||||
report.scanned += 1;
|
||||
let Some(task_id) = entry
|
||||
.strip_prefix(ROOT_TERMINAL_PREFIX)
|
||||
.and_then(|entry| entry.strip_suffix(".json"))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let _ = terminal_path(task_id)?;
|
||||
ids.insert(task_id.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let mut ids = ids.into_iter().collect::<Vec<_>>();
|
||||
ids.sort();
|
||||
let mut deletes = 0usize;
|
||||
for task_id in ids {
|
||||
if deletes >= ROOT_TERMINAL_GC_DELETE_BUDGET {
|
||||
report.budget_exhausted = true;
|
||||
break;
|
||||
}
|
||||
let Some((terminal_disk, terminal_bytes)) = Self::find_terminal(&disks, &task_id).await? else {
|
||||
continue;
|
||||
};
|
||||
let terminal = decode_terminal(&task_id, &terminal_bytes)?;
|
||||
if terminal.retained_at(now) {
|
||||
report.retained += 1;
|
||||
continue;
|
||||
}
|
||||
if let Some((pending_disk, pending_bytes)) = Self::find(&disks, &task_id).await? {
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
pending_disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&intent_path(&task_id)?,
|
||||
Some(pending_bytes),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
EcstoreConditionalFileUpdate::Updated => {
|
||||
deletes += 1;
|
||||
report.pending_removed += 1;
|
||||
report.retained += 1;
|
||||
continue;
|
||||
}
|
||||
_ => {
|
||||
return Err(Error::Other(format!(
|
||||
"Root heal recovery record changed while pruning terminal receipt {task_id}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
terminal_disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&terminal_path(&task_id)?,
|
||||
Some(terminal_bytes),
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
EcstoreConditionalFileUpdate::Updated => {
|
||||
deletes += 1;
|
||||
report.terminals_removed += 1;
|
||||
}
|
||||
_ => return Err(Error::Other(format!("Root heal terminal record changed while pruning {task_id}"))),
|
||||
}
|
||||
}
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
pub(super) async fn pending(&self) -> Result<Vec<HealRequest>> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if self.disabled_for_tests {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let _guard = self.mutation.lock().await;
|
||||
let disks = self.disks().await?;
|
||||
let mut ids = HashSet::new();
|
||||
for disk in &disks {
|
||||
EcstoreDiskAPI::stat_volume(disk.as_ref(), RUSTFS_META_BUCKET).await?;
|
||||
let entries = match EcstoreDiskAPI::list_dir(disk.as_ref(), "", RUSTFS_META_BUCKET, "", -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound) => continue,
|
||||
Err(error) => return Err(Error::Disk(error)),
|
||||
};
|
||||
for entry in entries {
|
||||
let Some(task_id) = entry
|
||||
.strip_prefix(ROOT_RECOVERY_PREFIX)
|
||||
.and_then(|entry| entry.strip_suffix(".json"))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let _ = intent_path(task_id)?;
|
||||
ids.insert(task_id.to_string());
|
||||
}
|
||||
}
|
||||
let mut requests = Vec::new();
|
||||
for task_id in ids {
|
||||
if Self::find_terminal(&disks, &task_id).await?.is_some() {
|
||||
continue;
|
||||
}
|
||||
if let Some((_, bytes)) = Self::find(&disks, &task_id).await? {
|
||||
requests.push(decode_intent(&task_id, &bytes)?.into_request());
|
||||
}
|
||||
}
|
||||
requests.sort_by(|left, right| left.created_at.cmp(&right.created_at).then_with(|| left.id.cmp(&right.id)));
|
||||
Ok(requests)
|
||||
}
|
||||
}
|
||||
|
||||
impl HealManager {
|
||||
pub(super) async fn replay_root_heals(&self) -> Result<()> {
|
||||
// Decode every record before admitting anything. These are already
|
||||
// accepted responsibilities, so restore distinct IDs even when their
|
||||
// paths overlap or the configured admission capacity has changed.
|
||||
let requests = self.root_recovery.pending().await?;
|
||||
let active = self.active_heals.lock().await;
|
||||
let mut queue = self.heal_queue.lock().await;
|
||||
let retrying = self.retrying_heals.lock().await;
|
||||
for mut request in requests {
|
||||
request.force_start = true;
|
||||
let existing = active
|
||||
.get(&request.id)
|
||||
.map(|task| request_matches_task(&request, task))
|
||||
.or_else(|| {
|
||||
queue
|
||||
.requests()
|
||||
.find(|queued| queued.id == request.id)
|
||||
.map(|queued| request_matches_request(&request, queued))
|
||||
})
|
||||
.or_else(|| {
|
||||
retrying
|
||||
.get(&request.id)
|
||||
.map(|retrying| request_matches_request(&request, &retrying.request))
|
||||
});
|
||||
match existing {
|
||||
Some(true) => continue,
|
||||
Some(false) => return Err(Error::Other(format!("Conflicting root heal recovery task {}", request.id))),
|
||||
None => {}
|
||||
}
|
||||
queue.push(request);
|
||||
}
|
||||
publish_heal_queue_length(&queue);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -26,7 +26,6 @@ impl HealManager {
|
||||
let retrying_heals = self.retrying_heals.clone();
|
||||
let mrf_repair_notice_targets = self.mrf_repair_notice_targets.clone();
|
||||
let replacement_recovery_anchors = self.replacement_recovery_anchors.clone();
|
||||
let root_recovery = self.root_recovery.clone();
|
||||
let cancel_token = self.cancel_token.clone();
|
||||
let statistics = self.statistics.clone();
|
||||
let storage = self.storage.clone();
|
||||
@@ -60,7 +59,6 @@ impl HealManager {
|
||||
retrying_heals: &retrying_heals,
|
||||
mrf_repair_notice_targets: &mrf_repair_notice_targets,
|
||||
replacement_recovery_anchors: &replacement_recovery_anchors,
|
||||
root_recovery: &root_recovery,
|
||||
config: &config,
|
||||
statistics: &statistics,
|
||||
storage: &storage,
|
||||
@@ -80,7 +78,6 @@ impl HealManager {
|
||||
retrying_heals: &retrying_heals,
|
||||
mrf_repair_notice_targets: &mrf_repair_notice_targets,
|
||||
replacement_recovery_anchors: &replacement_recovery_anchors,
|
||||
root_recovery: &root_recovery,
|
||||
config: &config,
|
||||
statistics: &statistics,
|
||||
storage: &storage,
|
||||
@@ -109,7 +106,6 @@ impl HealManager {
|
||||
retrying_heals,
|
||||
mrf_repair_notice_targets,
|
||||
replacement_recovery_anchors,
|
||||
root_recovery,
|
||||
config,
|
||||
statistics,
|
||||
storage,
|
||||
@@ -121,9 +117,6 @@ impl HealManager {
|
||||
let config = config.read().await;
|
||||
let mainline_pressure = Self::mainline_throttle_active(&config, workload_provider);
|
||||
let mut active_heals_guard = active_heals.lock().await;
|
||||
if cancel_token.is_cancelled() {
|
||||
return;
|
||||
}
|
||||
publish_active_heal_count(&active_heals_guard);
|
||||
|
||||
// Check if new heal tasks can be started
|
||||
@@ -213,7 +206,6 @@ impl HealManager {
|
||||
let replacement_recovery_anchors_clone = replacement_recovery_anchors.clone();
|
||||
let statistics_clone = statistics.clone();
|
||||
let notify_clone = notify.clone();
|
||||
let root_recovery_clone = root_recovery.clone();
|
||||
let manager_cancel_token = cancel_token.clone();
|
||||
let task_type_label_for_spawn = task_type_label.clone();
|
||||
let task_set_label_for_spawn = task_set_label.clone();
|
||||
@@ -320,47 +312,7 @@ impl HealManager {
|
||||
completed_status_entry.status = HealTaskStatus::Cancelled;
|
||||
completed_status_entry.outcome = Some(Arc::new(task.get_outcome().await));
|
||||
}
|
||||
let terminal_completion = matches!(
|
||||
completed_status,
|
||||
HealTaskStatus::Completed | HealTaskStatus::Cancelled | HealTaskStatus::Failed { .. }
|
||||
);
|
||||
if owns_completion
|
||||
&& terminal_completion
|
||||
&& root_recovery::is_admin_heal_recovery(&task.heal_type, task.source)
|
||||
&& let Err(error) = root_recovery_clone
|
||||
.persist_terminal(&task_id, &task.heal_type, task.source, &completed_status_entry)
|
||||
.await
|
||||
{
|
||||
// Keep the durable responsibility if terminal
|
||||
// publication fails. Replaying the task is preferable
|
||||
// to losing the final receipt across restart.
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_SCHEDULER_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
task_id,
|
||||
state = "root_recovery_terminal_publish_failed",
|
||||
error = %error,
|
||||
"Failed to publish heal terminal receipt"
|
||||
);
|
||||
}
|
||||
if owns_completion
|
||||
&& !terminal_completion
|
||||
&& result.is_err()
|
||||
&& let Err(error) = root_recovery_clone.checkpoint_failed_execution(&task).await
|
||||
{
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_SCHEDULER_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
task_id,
|
||||
state = "root_recovery_checkpoint_failed",
|
||||
error = %error,
|
||||
"Failed to checkpoint root heal recovery execution budget"
|
||||
);
|
||||
}
|
||||
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
|
||||
let completed_status_for_verified_events = completed_status_entry.clone();
|
||||
// Keep retry ownership continuous: status snapshots acquire
|
||||
// these locks in the same active -> retrying order.
|
||||
|
||||
@@ -26,7 +26,6 @@ use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use std::sync::Mutex as StdMutex;
|
||||
use tempfile::TempDir;
|
||||
|
||||
mod root_recovery;
|
||||
mod running_mainline;
|
||||
|
||||
use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo};
|
||||
@@ -95,7 +94,6 @@ async fn process_manager_queue_once(manager: &HealManager) {
|
||||
retrying_heals: &manager.retrying_heals,
|
||||
mrf_repair_notice_targets: &manager.mrf_repair_notice_targets,
|
||||
replacement_recovery_anchors: &manager.replacement_recovery_anchors,
|
||||
root_recovery: &manager.root_recovery,
|
||||
config: &manager.config,
|
||||
statistics: &manager.statistics,
|
||||
storage: &manager.storage,
|
||||
@@ -151,7 +149,7 @@ fn completed_retention_cursor_boundaries_preserve_progress() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_retention_displaced_alias_does_not_resurrect_evicted_snapshot() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let request = HealRequest::bucket("bucket".to_string());
|
||||
manager.insert_task_alias("alias", &request.id).await;
|
||||
let terminal = record_displaced_terminal(&manager.displaced_terminals, &request);
|
||||
@@ -213,7 +211,7 @@ async fn completed_retention_clock_rollback_preserves_terminal_alias_queries() {
|
||||
},
|
||||
HealTaskStatus::Cancelled,
|
||||
] {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let mut snapshot = completed_retention_fixture(completed_at);
|
||||
snapshot.status = status.clone();
|
||||
let expected_progress = snapshot.progress.clone();
|
||||
@@ -367,7 +365,7 @@ async fn canonical_outcome_cancel_wins_before_worker_finalizes_success() {
|
||||
use crate::heal::outcome::{HealAbortReason, HealExecutionOutcome};
|
||||
use crate::heal::task::{OUTCOME_FINISH_TEST_HOOK, OutcomeFinishTestHook};
|
||||
let bucket = "canonical-outcome-cancel-before-finish";
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
|
||||
let task_id = request.id.clone();
|
||||
let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
|
||||
@@ -418,7 +416,7 @@ async fn canonical_outcome_cancel_wins_before_worker_finalizes_success() {
|
||||
#[tokio::test]
|
||||
async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
|
||||
let bucket = "completed-retention-retry-cancel";
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
|
||||
let task_id = request.id.clone();
|
||||
let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
|
||||
@@ -473,7 +471,7 @@ async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_han
|
||||
for outcome in ["success", "failed", "cancelled"] {
|
||||
let bucket = format!("completed-retention-{outcome}");
|
||||
let hook = Arc::new(CompletedRetentionHook::default());
|
||||
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None));
|
||||
let manager = Arc::new(HealManager::new(Arc::new(MockStorage), None));
|
||||
let request = HealRequest::object(bucket.clone(), "object".to_string(), None);
|
||||
let task_id = request.id.clone();
|
||||
let duplicate = HealRequest::object(bucket.clone(), "object".to_string(), None);
|
||||
@@ -713,7 +711,7 @@ impl HealStorageAPI for MockStorage {
|
||||
async fn assert_heal_start_retry_control_preserves_real_executor_progress(cancel: bool) {
|
||||
for phase in ["listing", "object"] {
|
||||
let bucket = format!("heal-start-retry-deadline-{phase}-{cancel}");
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Prefix {
|
||||
bucket: bucket.clone(),
|
||||
@@ -824,7 +822,7 @@ async fn heal_start_retry_cancellation_preserves_real_executor_progress() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_start_retry_scheduler_carries_explicit_budget_and_identity() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
let manager = HealManager::new(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
task_timeout: Duration::ZERO,
|
||||
@@ -865,88 +863,6 @@ async fn heal_start_retry_scheduler_carries_explicit_budget_and_identity() {
|
||||
COMPLETED_RETENTION_HOOKS.lock().await.remove(&task_id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_start_retry_scheduler_carries_explicit_budget_across_retries() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
task_timeout: Duration::ZERO,
|
||||
event_driven_scheduler_enable: false,
|
||||
..Default::default()
|
||||
}),
|
||||
);
|
||||
let mut request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
|
||||
request.source = HealRequestSource::Admin;
|
||||
request.options.timeout = Some(Duration::from_secs(60));
|
||||
let task_id = request.id.clone();
|
||||
let created_at = request.created_at;
|
||||
manager
|
||||
.submit_heal_request(request)
|
||||
.await
|
||||
.expect("admit explicit-budget task");
|
||||
|
||||
let mut previous_remaining = Duration::from_secs(60);
|
||||
for expected_attempt in 1..=3 {
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
if manager.retrying_heals.lock().await.contains_key(&task_id) {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("recoverable executor failure should enter retry backoff");
|
||||
|
||||
let retry = manager.retrying_heals.lock().await[&task_id].request.clone();
|
||||
assert_eq!(retry.id, task_id);
|
||||
assert_eq!(retry.created_at, created_at);
|
||||
assert_eq!(retry.source, HealRequestSource::Admin);
|
||||
assert_eq!(retry.retry_attempts, expected_attempt);
|
||||
let remaining = retry.options.timeout.expect("retry retains explicit budget");
|
||||
assert!(
|
||||
remaining > Duration::ZERO && remaining < previous_remaining,
|
||||
"retry attempt {expected_attempt} should carry only the unused explicit budget"
|
||||
);
|
||||
previous_remaining = remaining;
|
||||
assert_eq!(manager.operations_snapshot().await.queue_length, 0);
|
||||
assert_eq!(manager.operations_snapshot().await.retrying_tasks, 1);
|
||||
|
||||
let retry_delay = recoverable_heal_retry_delay(expected_attempt);
|
||||
tokio::time::timeout(retry_delay + Duration::from_secs(1), async {
|
||||
loop {
|
||||
if !manager.retrying_heals.lock().await.contains_key(&task_id) {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("retry backoff should requeue the same task id");
|
||||
assert_eq!(manager.operations_snapshot().await.queue_length, 1);
|
||||
}
|
||||
|
||||
process_manager_queue_once(&manager).await;
|
||||
let final_error = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if let Ok(HealTaskStatus::Failed { error }) = manager.get_task_status(&task_id).await {
|
||||
break error;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("retry limit should finish as a terminal failure");
|
||||
let final_error_lower = final_error.to_ascii_lowercase();
|
||||
assert!(
|
||||
final_error_lower.contains("insufficient") && final_error_lower.contains("read"),
|
||||
"terminal failure should retain the recoverable storage error: {final_error}"
|
||||
);
|
||||
assert_eq!(manager.operations_snapshot().await.queue_length, 0);
|
||||
assert_eq!(manager.operations_snapshot().await.retrying_tasks, 0);
|
||||
}
|
||||
|
||||
struct ManagerRecoveryTestHook {
|
||||
replacement_resume_disk: DiskStore,
|
||||
listed: StdMutex<bool>,
|
||||
@@ -1046,7 +962,7 @@ fn scoped_object_request(bucket: &str, object: &str, pool_index: usize, set_inde
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduler_bulkhead_starts_other_sets_and_retains_same_set_tail() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
{
|
||||
let mut config = manager.config.write().await;
|
||||
config.max_concurrent_heals = 2;
|
||||
@@ -1951,7 +1867,7 @@ fn test_heal_request_and_task_metric_labels_match() {
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_returns_merged_for_duplicate() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let request = HealRequest::new(
|
||||
HealType::Object {
|
||||
@@ -1982,7 +1898,7 @@ async fn test_submit_heal_request_returns_merged_for_duplicate() {
|
||||
#[tokio::test]
|
||||
async fn test_admin_duplicate_receipt_returns_canonical_task_without_alias() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
let mut original = HealRequest::object("bucket".to_string(), "object".to_string(), None);
|
||||
original.source = HealRequestSource::Admin;
|
||||
let original_id = original.id.clone();
|
||||
@@ -2009,7 +1925,7 @@ async fn test_admin_duplicate_receipt_returns_canonical_task_without_alias() {
|
||||
#[tokio::test]
|
||||
async fn test_task_alias_is_removed_after_terminal_completion() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
let original = HealRequest::object("bucket".to_string(), "object".to_string(), None);
|
||||
let original_id = original.id.clone();
|
||||
let duplicate = HealRequest::object("bucket".to_string(), "object".to_string(), None);
|
||||
@@ -2049,7 +1965,7 @@ async fn test_task_alias_is_removed_after_terminal_completion() {
|
||||
#[tokio::test]
|
||||
async fn test_duplicate_admission_is_atomic_with_queue_to_active_transition() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage.clone(), None));
|
||||
let manager = Arc::new(HealManager::new(storage.clone(), None));
|
||||
let mut original = HealRequest::object("bucket".to_string(), "object".to_string(), None);
|
||||
original.source = HealRequestSource::Admin;
|
||||
let original_id = original.id.clone();
|
||||
@@ -2119,7 +2035,7 @@ async fn test_duplicate_admission_is_atomic_with_queue_to_active_transition() {
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_returns_merged_for_active_duplicate() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None);
|
||||
let manager = HealManager::new(storage.clone(), None);
|
||||
let active_request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
|
||||
let active_task = Arc::new(HealTask::from_request(active_request, storage));
|
||||
manager.active_heals.lock().await.insert(active_task.id.clone(), active_task);
|
||||
@@ -2139,7 +2055,7 @@ async fn test_submit_heal_request_returns_merged_for_active_duplicate() {
|
||||
#[tokio::test]
|
||||
async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None);
|
||||
let manager = HealManager::new(storage.clone(), None);
|
||||
let active_request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
|
||||
let active_task = Arc::new(HealTask::from_request(active_request, storage));
|
||||
let active_task_id = active_task.id.clone();
|
||||
@@ -2181,7 +2097,7 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
|
||||
#[tokio::test]
|
||||
async fn test_queued_duplicate_token_can_query_and_cancel_original_request() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
let original_request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
|
||||
let original_task_id = original_request.id.clone();
|
||||
let duplicate_request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
|
||||
@@ -2372,7 +2288,7 @@ fn durable_replacement_recovery_re_admits_only_the_matching_generation() {
|
||||
|
||||
#[test]
|
||||
fn replacement_recovery_blocker_is_set_scoped() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
|
||||
manager.block_replacement_recovery_set("pool_0_set_0");
|
||||
|
||||
@@ -2478,7 +2394,7 @@ async fn scheduler_completes_cleanup_pending_recovery_from_manager_anchor() {
|
||||
|
||||
let (hook, _hook_guard) = ManagerRecoveryTestHook::install(anchor.clone());
|
||||
let storage = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None);
|
||||
let manager = HealManager::new(storage.clone(), None);
|
||||
let mut request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec!["bucket-a".to_string()],
|
||||
@@ -2613,7 +2529,7 @@ fn test_retry_request_for_recoverable_error_stops_at_limit() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_retry_request_rescans_batch_when_all_exhausted_objects_are_retryable() {
|
||||
async fn test_retry_request_does_not_rescan_batch_after_object_retries_exhausted() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage);
|
||||
let result = Err(task
|
||||
@@ -2627,32 +2543,7 @@ async fn test_retry_request_rescans_batch_when_all_exhausted_objects_are_retryab
|
||||
})
|
||||
.await);
|
||||
|
||||
let (retry_request, retry_delay, error) = retry_request_for_result_with_budget(&task, &result)
|
||||
.await
|
||||
.expect("all-retryable batch failure should rescan within the manager retry budget");
|
||||
|
||||
assert_eq!(retry_request.id, task.id);
|
||||
assert_eq!(retry_request.retry_attempts, 1);
|
||||
assert!(retry_delay > Duration::ZERO);
|
||||
assert!(error.contains("Lock acquisition timeout"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_retry_request_does_not_rescan_batch_with_permanent_failures() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), storage);
|
||||
let result = Err(task
|
||||
.record_batch_failure(BatchHealFailure {
|
||||
scope: "bucket:bucket".to_string(),
|
||||
failed: 2,
|
||||
retryable: 1,
|
||||
permanent: 1,
|
||||
first_object: "object-a".to_string(),
|
||||
first_error: "Lock acquisition timeout".to_string(),
|
||||
})
|
||||
.await);
|
||||
|
||||
assert!(retry_request_for_result_with_budget(&task, &result).await.is_none());
|
||||
assert!(retry_request_for_result(&task, &result).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2714,7 +2605,7 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
|
||||
#[tokio::test]
|
||||
async fn test_cancel_task_cancels_retrying_backoff() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
let mut request = HealRequest::bucket("bucket".to_string());
|
||||
request.retry_attempts = 1;
|
||||
let task_id = request.id.clone();
|
||||
@@ -2741,7 +2632,7 @@ async fn test_cancel_task_cancels_retrying_backoff() {
|
||||
#[tokio::test]
|
||||
async fn test_cancel_tasks_for_path_cancels_retrying_backoff() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
let mut request = HealRequest::bucket("bucket".to_string());
|
||||
request.retry_attempts = 1;
|
||||
let task_id = request.id.clone();
|
||||
@@ -2763,7 +2654,7 @@ async fn test_cancel_tasks_for_path_cancels_retrying_backoff() {
|
||||
#[tokio::test]
|
||||
async fn test_cancel_tasks_for_empty_path_cancels_queued_cluster_only() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High);
|
||||
let cluster_request_id = cluster_request.id.clone();
|
||||
@@ -2802,7 +2693,7 @@ async fn test_cancel_tasks_for_empty_path_cancels_queued_cluster_only() {
|
||||
#[tokio::test]
|
||||
async fn test_cancel_tasks_for_empty_path_cancels_active_cluster_only() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None);
|
||||
let manager = HealManager::new(storage.clone(), None);
|
||||
|
||||
let cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High);
|
||||
let cluster_request_id = cluster_request.id.clone();
|
||||
@@ -2833,7 +2724,7 @@ async fn test_cancel_tasks_for_empty_path_cancels_active_cluster_only() {
|
||||
#[tokio::test]
|
||||
async fn test_cancel_tasks_for_empty_path_cancels_retrying_cluster_only() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let mut cluster_request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High);
|
||||
cluster_request.retry_attempts = 1;
|
||||
@@ -2872,7 +2763,7 @@ fn test_heal_type_matches_path_accepts_legacy_root() {
|
||||
#[tokio::test]
|
||||
async fn test_retrying_duplicate_token_can_query_and_cancel_original_retry() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
let mut original_request = HealRequest::bucket("bucket".to_string());
|
||||
original_request.retry_attempts = 1;
|
||||
let original_task_id = original_request.id.clone();
|
||||
@@ -2908,7 +2799,7 @@ async fn test_retrying_duplicate_token_can_query_and_cancel_original_retry() {
|
||||
#[tokio::test]
|
||||
async fn test_get_task_status_reports_pending_for_queued_request() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let request = HealRequest::bucket("bucket".to_string());
|
||||
let request_id = request.id.clone();
|
||||
@@ -2932,7 +2823,7 @@ async fn test_get_task_status_reports_pending_for_queued_request() {
|
||||
#[tokio::test]
|
||||
async fn test_operations_snapshot_counts_queue_by_source_and_priority() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let mut scanner_request = HealRequest::new(
|
||||
HealType::Object {
|
||||
@@ -2989,7 +2880,7 @@ async fn test_operations_snapshot_counts_queue_by_source_and_priority() {
|
||||
// HS-06 (backlog#1870): overlap policy + forceStart semantics.
|
||||
fn manager_with_policy(policy: HealOverlapPolicy) -> HealManager {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
HealManager::new_without_root_recovery_for_test(
|
||||
HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
overlap_policy: policy,
|
||||
@@ -3011,12 +2902,6 @@ fn admin_prefix_request(bucket: &str, prefix: &str) -> HealRequest {
|
||||
request
|
||||
}
|
||||
|
||||
fn internal_prefix_request(bucket: &str, prefix: &str) -> HealRequest {
|
||||
let mut request = admin_prefix_request(bucket, prefix);
|
||||
request.source = HealRequestSource::Internal;
|
||||
request
|
||||
}
|
||||
|
||||
async fn insert_active_task(manager: &HealManager, request: HealRequest) -> String {
|
||||
let task = Arc::new(HealTask::from_request(request, manager.storage.clone()));
|
||||
let task_id = task.id.clone();
|
||||
@@ -3125,7 +3010,7 @@ async fn admin_force_start_cancels_overlapping_active_task_first() {
|
||||
#[tokio::test]
|
||||
async fn admission_snapshot_tracks_start_duplicate_force_start_and_displacement() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(
|
||||
let manager = Arc::new(HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
@@ -3133,7 +3018,7 @@ async fn admission_snapshot_tracks_start_duplicate_force_start_and_displacement(
|
||||
}),
|
||||
));
|
||||
|
||||
let mut paused = internal_prefix_request("bucket-a", "logs/");
|
||||
let mut paused = admin_prefix_request("bucket-a", "logs/");
|
||||
paused.priority = HealPriority::Low;
|
||||
let hook = Arc::new(DuplicateAdmissionTestHook {
|
||||
request_id: paused.id.clone(),
|
||||
@@ -3163,7 +3048,7 @@ async fn admission_snapshot_tracks_start_duplicate_force_start_and_displacement(
|
||||
);
|
||||
*DUPLICATE_ADMISSION_TEST_HOOK.lock().await = None;
|
||||
|
||||
let duplicate = internal_prefix_request("bucket-a", "logs/");
|
||||
let duplicate = admin_prefix_request("bucket-a", "logs/");
|
||||
let duplicate_receipt = manager
|
||||
.submit_heal_request_with_receipt(duplicate)
|
||||
.await
|
||||
@@ -3207,7 +3092,7 @@ async fn admission_snapshot_tracks_start_duplicate_force_start_and_displacement(
|
||||
#[tokio::test]
|
||||
async fn test_operations_snapshot_counts_active_by_source_and_priority() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let mut request = HealRequest::bucket("bucket-a".to_string());
|
||||
request.priority = HealPriority::High;
|
||||
@@ -3229,7 +3114,7 @@ async fn test_operations_snapshot_counts_active_by_source_and_priority() {
|
||||
#[tokio::test]
|
||||
async fn test_operations_snapshot_counts_retry_backoff_as_owned_work() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
let mut request = HealRequest::bucket("bucket-retry".to_string());
|
||||
request.priority = HealPriority::Urgent;
|
||||
request.source = HealRequestSource::Admin;
|
||||
@@ -3255,7 +3140,7 @@ async fn test_operations_snapshot_counts_retry_backoff_as_owned_work() {
|
||||
#[tokio::test]
|
||||
async fn test_scheduler_retry_transitions_keep_continuous_single_ownership() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage, None));
|
||||
let manager = Arc::new(HealManager::new(storage, None));
|
||||
{
|
||||
let mut config = manager.config.write().await;
|
||||
config.enable_auto_heal = false;
|
||||
@@ -3328,7 +3213,7 @@ async fn test_scheduler_retry_transitions_keep_continuous_single_ownership() {
|
||||
#[tokio::test]
|
||||
async fn test_active_progress_snapshot_sums_active_task_progress() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let first = Arc::new(HealTask::from_request(
|
||||
HealRequest::bucket("bucket-a".to_string()),
|
||||
@@ -3373,7 +3258,7 @@ async fn test_active_progress_snapshot_sums_active_task_progress() {
|
||||
#[tokio::test]
|
||||
async fn test_get_task_status_for_path_rejects_wrong_token_when_path_is_active() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
manager
|
||||
.submit_heal_request(HealRequest::bucket("bucket".to_string()))
|
||||
@@ -3389,7 +3274,7 @@ async fn test_get_task_status_for_path_rejects_wrong_token_when_path_is_active()
|
||||
#[tokio::test]
|
||||
async fn test_get_task_status_for_path_rejects_token_from_other_active_path() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let bucket_request = HealRequest::bucket("bucket".to_string());
|
||||
let other_request = HealRequest::bucket("other".to_string());
|
||||
@@ -3413,7 +3298,7 @@ async fn test_get_task_status_for_path_rejects_token_from_other_active_path() {
|
||||
#[tokio::test]
|
||||
async fn test_get_task_status_for_path_does_not_accept_token_from_inactive_path() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let request = HealRequest::bucket("bucket".to_string());
|
||||
let request_id = request.id.clone();
|
||||
@@ -3432,7 +3317,7 @@ async fn test_get_task_status_for_path_does_not_accept_token_from_inactive_path(
|
||||
#[tokio::test]
|
||||
async fn test_get_task_status_for_path_returns_not_found_when_path_is_inactive() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
assert!(matches!(
|
||||
manager.get_task_status_for_path("bucket", "old-token").await,
|
||||
@@ -3443,7 +3328,7 @@ async fn test_get_task_status_for_path_returns_not_found_when_path_is_inactive()
|
||||
#[tokio::test]
|
||||
async fn test_get_task_status_for_empty_path_does_not_match_unrelated_tasks() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let request = HealRequest::bucket("bucket".to_string());
|
||||
let request_id = request.id.clone();
|
||||
@@ -3466,7 +3351,7 @@ async fn test_get_task_status_for_empty_path_does_not_match_unrelated_tasks() {
|
||||
#[tokio::test]
|
||||
async fn test_get_task_report_queries_queued_task_by_token_without_path() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
@@ -3495,7 +3380,7 @@ async fn test_get_task_report_queries_queued_task_by_token_without_path() {
|
||||
#[tokio::test]
|
||||
async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
// A completed entry recorded in a Retrying state for a task whose
|
||||
// request is also (still) queued under the same id: the retrying
|
||||
@@ -3539,7 +3424,7 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
|
||||
#[tokio::test]
|
||||
async fn test_get_task_status_reads_recent_completed_status() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
manager.completed_heals.lock().await.insert(
|
||||
"completed-token".to_string(),
|
||||
@@ -3571,7 +3456,7 @@ async fn test_get_task_status_reads_recent_completed_status() {
|
||||
#[tokio::test]
|
||||
async fn test_get_task_report_for_path_reads_completed_items() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
manager.completed_heals.lock().await.insert(
|
||||
"completed-token".to_string(),
|
||||
@@ -3619,7 +3504,7 @@ async fn test_get_task_report_for_path_reads_completed_items() {
|
||||
#[tokio::test]
|
||||
async fn test_get_task_report_for_empty_path_does_not_match_unrelated_tasks() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
manager
|
||||
.submit_heal_request(HealRequest::bucket("bucket".to_string()))
|
||||
@@ -3635,7 +3520,7 @@ async fn test_get_task_report_for_empty_path_does_not_match_unrelated_tasks() {
|
||||
#[tokio::test]
|
||||
async fn test_cancel_task_removes_queued_request() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let request = HealRequest::bucket("bucket".to_string());
|
||||
let request_id = request.id.clone();
|
||||
@@ -3659,7 +3544,7 @@ async fn mrf_ownership_unverified_completion_does_not_emit_repaired() {
|
||||
let version_id = Some([9u8; 16]);
|
||||
let _ = rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket);
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let mut request = HealRequest::object(bucket.to_string(), object.to_string(), None);
|
||||
request.source = HealRequestSource::Mrf;
|
||||
@@ -3706,7 +3591,7 @@ async fn mrf_ownership_dry_run_and_empty_window_do_not_emit_repaired() {
|
||||
} else {
|
||||
"mrf-dry-run-outcome"
|
||||
};
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let request = HealRequest::new(
|
||||
if empty_window {
|
||||
HealType::Cluster
|
||||
@@ -3762,7 +3647,7 @@ async fn mrf_ownership_queued_cancel_does_not_emit_repaired() {
|
||||
let object = "object";
|
||||
let _ = rustfs_common::mrf_channel::take_mrf_repaired_events_for(bucket);
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let mut request = HealRequest::object(bucket.to_string(), object.to_string(), None);
|
||||
request.source = HealRequestSource::Mrf;
|
||||
@@ -3793,7 +3678,7 @@ async fn mrf_ownership_queued_cancel_does_not_emit_repaired() {
|
||||
#[tokio::test]
|
||||
async fn test_cancel_tasks_for_path_removes_matching_queued_requests() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let bucket_request = HealRequest::bucket("bucket".to_string());
|
||||
let bucket_request_id = bucket_request.id.clone();
|
||||
@@ -3832,7 +3717,7 @@ async fn test_cancel_tasks_for_path_removes_matching_queued_requests() {
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
let manager = HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
@@ -3869,7 +3754,7 @@ async fn test_submit_heal_request_returns_merged_before_full_for_duplicate() {
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_returns_dropped_for_low_priority_when_full() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
let manager = HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
@@ -3912,7 +3797,7 @@ async fn test_submit_heal_request_returns_dropped_for_low_priority_when_full() {
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_returns_full_for_normal_priority_when_full() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
let manager = HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
@@ -3954,7 +3839,7 @@ async fn test_submit_heal_request_returns_full_for_normal_priority_when_full() {
|
||||
#[tokio::test]
|
||||
async fn test_high_priority_request_displaces_lower_priority_when_queue_full() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
let manager = HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
@@ -4009,7 +3894,7 @@ async fn test_high_priority_request_displaces_lower_priority_when_queue_full() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn displaced_task_remains_queryable() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
let manager = HealManager::new(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
@@ -4051,7 +3936,7 @@ async fn displaced_task_remains_queryable() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn displaced_archive_failure_keeps_queryable_terminal() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "archive-failure".to_string(),
|
||||
@@ -4074,7 +3959,7 @@ async fn displaced_archive_failure_keeps_queryable_terminal() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduler_retry_displacement_keeps_evicted_task_queryable() {
|
||||
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(
|
||||
let manager = Arc::new(HealManager::new(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
@@ -4139,7 +4024,7 @@ async fn scheduler_retry_displacement_keeps_evicted_task_queryable() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_displacers_produce_one_terminal_generation() {
|
||||
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(
|
||||
let manager = Arc::new(HealManager::new(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
@@ -4191,7 +4076,7 @@ async fn concurrent_displacers_produce_one_terminal_generation() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn successor_chain_is_bounded_and_authorized() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
let manager = HealManager::new(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
@@ -4244,7 +4129,7 @@ async fn successor_chain_is_bounded_and_authorized() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn displaced_terminal_expires_after_bounded_ttl() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "expires".to_string(),
|
||||
@@ -4267,7 +4152,7 @@ async fn displaced_terminal_expires_after_bounded_ttl() {
|
||||
#[tokio::test]
|
||||
async fn test_displacing_registered_mrf_task_drops_notice_ownership() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
let manager = HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
@@ -4315,7 +4200,7 @@ async fn test_displacing_registered_mrf_task_drops_notice_ownership() {
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_drops_read_repair_under_pressure() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
let manager = HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
@@ -4349,7 +4234,7 @@ async fn test_submit_heal_request_drops_read_repair_under_pressure() {
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_drops_low_scanner_under_pressure() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
let manager = HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
@@ -4383,7 +4268,7 @@ async fn test_submit_heal_request_drops_low_scanner_under_pressure() {
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_accepts_admin_high_under_pressure() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
let manager = HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 10,
|
||||
@@ -4423,7 +4308,7 @@ async fn test_mainline_throttle_delays_background_heal_start() {
|
||||
limit: 10,
|
||||
state: AdmissionState::Open,
|
||||
});
|
||||
let manager = HealManager::new_with_workload_provider_without_root_recovery_for_test(
|
||||
let manager = HealManager::new_with_workload_provider(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
max_concurrent_heals: 1,
|
||||
@@ -4456,7 +4341,7 @@ async fn test_mainline_throttle_delays_background_heal_start_under_write_pressur
|
||||
limit: 10,
|
||||
state: AdmissionState::Open,
|
||||
});
|
||||
let manager = HealManager::new_with_workload_provider_without_root_recovery_for_test(
|
||||
let manager = HealManager::new_with_workload_provider(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
max_concurrent_heals: 1,
|
||||
@@ -4489,7 +4374,7 @@ async fn test_mainline_throttle_allows_admin_high_start() {
|
||||
limit: 10,
|
||||
state: AdmissionState::Saturated,
|
||||
});
|
||||
let manager = HealManager::new_with_workload_provider_without_root_recovery_for_test(
|
||||
let manager = HealManager::new_with_workload_provider(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
max_concurrent_heals: 1,
|
||||
@@ -4515,7 +4400,7 @@ async fn test_mainline_throttle_allows_admin_high_start() {
|
||||
#[tokio::test]
|
||||
async fn configured_task_timeout_applies_only_when_request_timeout_is_absent() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
let manager = HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
max_concurrent_heals: 1,
|
||||
@@ -4569,7 +4454,7 @@ async fn configured_task_timeout_applies_only_when_request_timeout_is_absent() {
|
||||
#[tokio::test]
|
||||
async fn test_force_start_bypasses_duplicate_and_full_admission() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
let manager = HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
@@ -4628,7 +4513,7 @@ async fn test_force_start_bypasses_duplicate_and_full_admission() {
|
||||
#[tokio::test]
|
||||
async fn test_force_start_marks_dedup_key_for_future_duplicates() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
let manager = HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
@@ -4685,7 +4570,7 @@ async fn test_force_start_marks_dedup_key_for_future_duplicates() {
|
||||
#[tokio::test]
|
||||
async fn same_request_id_replay_reuses_existing_task_without_force_start_duplication() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage, None);
|
||||
let manager = HealManager::new(storage, None);
|
||||
|
||||
let mut original = admin_prefix_request("bucket", "logs/");
|
||||
original.force_start = true;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -141,7 +141,7 @@ async fn start_fixture(
|
||||
started: AtomicUsize::new(0),
|
||||
committed: AtomicUsize::new(0),
|
||||
});
|
||||
let manager = HealManager::new_with_workload_provider_without_root_recovery_for_test(
|
||||
let manager = HealManager::new_with_workload_provider(
|
||||
storage.clone(),
|
||||
Some(HealConfig {
|
||||
mainline_throttle_enable: pacing_enabled,
|
||||
|
||||
@@ -1137,28 +1137,9 @@ fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool, retain_replay_j
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::heal::manager::HealConfig;
|
||||
use crate::heal::storage::{ECStoreHealStorage, HealStorageAPI};
|
||||
use crate::heal::{DiskError, RUSTFS_META_BUCKET};
|
||||
use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfVerifiedRepairDisposition, MrfVerifiedRepairEvent};
|
||||
use serde_json::{Map, Value, json};
|
||||
use serial_test::serial;
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::io::Write as _;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc as StdArc;
|
||||
use std::time::{Duration as StdDuration, Instant};
|
||||
|
||||
const W13_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_EVIDENCE_DIR";
|
||||
const W13_SOURCE_REVISION_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_SOURCE_REVISION";
|
||||
const W13_SELECTION_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_SELECTION";
|
||||
const W13_SOAK_SECONDS_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_SOAK_SECONDS";
|
||||
const W13_ALLOW_SHORT_SOAK_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_ALLOW_SHORT_SOAK";
|
||||
const W13_RUN_ID_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_RUN_ID";
|
||||
const W13_WINDOW_ID_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_WINDOW_ID";
|
||||
const W13_ENOSPC_ROOT_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_ENOSPC_ROOT";
|
||||
const W13_ENOSPC_FILL_LIMIT_ENV: &str = "RUSTFS_SCANNER_HEAL_W13_ENOSPC_FILL_LIMIT_BYTES";
|
||||
|
||||
fn intent(bucket: &str, object: &str, attempts: u8) -> MrfIntent {
|
||||
MrfIntent {
|
||||
@@ -1179,748 +1160,6 @@ mod tests {
|
||||
payload
|
||||
}
|
||||
|
||||
fn w13_timestamp() -> String {
|
||||
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
|
||||
}
|
||||
|
||||
fn w13_selection_contains(selection: &str, lane: &str) -> bool {
|
||||
selection == "all" || selection.split(',').any(|item| item.trim() == lane)
|
||||
}
|
||||
|
||||
fn w13_evidence_path(root: &Path, gate: &str, field: &str) -> PathBuf {
|
||||
let lane = match gate {
|
||||
"G07" => "g07-mrf-responsibility",
|
||||
"G08" => "g08-mrf-capacity",
|
||||
"P4" => "p4-mrf-soak",
|
||||
other => panic!("unsupported W13 evidence gate: {other}"),
|
||||
};
|
||||
root.join(lane).join(format!("{gate}-{field}.json"))
|
||||
}
|
||||
|
||||
struct W13Evidence<'a> {
|
||||
source_revision: &'a str,
|
||||
run_id: &'a str,
|
||||
window_id: &'a str,
|
||||
started_at: &'a str,
|
||||
finished_at: &'a str,
|
||||
gate: &'a str,
|
||||
field: &'a str,
|
||||
artifact_kind: &'a str,
|
||||
extra: Map<String, Value>,
|
||||
}
|
||||
|
||||
fn write_w13_evidence(root: &Path, evidence: W13Evidence<'_>) {
|
||||
let path = w13_evidence_path(root, evidence.gate, evidence.field);
|
||||
fs::create_dir_all(path.parent().expect("W13 evidence artifact parent")).expect("create W13 evidence artifact directory");
|
||||
let mut payload = Map::new();
|
||||
payload.insert("schema".to_string(), json!(1));
|
||||
payload.insert("evidence_type".to_string(), json!("measured"));
|
||||
payload.insert("artifact_kind".to_string(), json!(evidence.artifact_kind));
|
||||
payload.insert("source_revision".to_string(), json!(evidence.source_revision));
|
||||
payload.insert("run_id".to_string(), json!(evidence.run_id));
|
||||
payload.insert("measurement_window_id".to_string(), json!(evidence.window_id));
|
||||
payload.insert("started_at".to_string(), json!(evidence.started_at));
|
||||
payload.insert("finished_at".to_string(), json!(evidence.finished_at));
|
||||
payload.insert("gate".to_string(), json!(evidence.gate));
|
||||
payload.insert("field".to_string(), json!(evidence.field));
|
||||
payload.insert(
|
||||
"command".to_string(),
|
||||
json!([
|
||||
"cargo",
|
||||
"test",
|
||||
"--locked",
|
||||
"-p",
|
||||
"rustfs-heal",
|
||||
"--lib",
|
||||
"heal::mrf_queue::tests::w13_mrf_release_evidence_outputs_bundle_artifacts",
|
||||
"--",
|
||||
"--ignored",
|
||||
"--exact",
|
||||
"--nocapture"
|
||||
]),
|
||||
);
|
||||
payload.insert(
|
||||
"summary".to_string(),
|
||||
json!(format!("Measured W13 MRF evidence for {}.{}", evidence.gate, evidence.field)),
|
||||
);
|
||||
payload.extend(evidence.extra);
|
||||
let bytes = serde_json::to_vec_pretty(&Value::Object(payload)).expect("serialize W13 evidence payload");
|
||||
fs::write(&path, [bytes.as_slice(), b"\n"].concat()).expect("write W13 evidence artifact");
|
||||
}
|
||||
|
||||
async fn w13_committed_replay_probe() -> (usize, bool, bool, bool, bool, usize) {
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.prefix("rustfs_mrf_w13_replay_evidence")
|
||||
.build()
|
||||
.await;
|
||||
let bucket = "w13-replay-bucket";
|
||||
let object = "w13-replay-object";
|
||||
env.make_bucket(bucket, false).await;
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
|
||||
let manager = Arc::new(HealManager::new(
|
||||
storage.clone(),
|
||||
Some(HealConfig {
|
||||
queue_size: 2,
|
||||
heal_interval: Duration::from_secs(3600),
|
||||
enable_auto_heal: false,
|
||||
..Default::default()
|
||||
}),
|
||||
));
|
||||
let disks = journal_disks().await;
|
||||
assert!(!disks.is_empty(), "W13 evidence requires real local MRF disks");
|
||||
|
||||
let config = MrfConsumerConfig::default();
|
||||
let replay_owner = Uuid::new_v4();
|
||||
let mut replay_intent = intent(bucket, object, 0);
|
||||
replay_intent.kind = MrfKind::PartialWrite;
|
||||
replay_intent.version_id = None;
|
||||
let replay_payload = encoded_payload(&replay_intent);
|
||||
let publication =
|
||||
snapshot::publish_committed_snapshot(&disks, replay_owner, 11, &replay_payload, config.journal_max_bytes)
|
||||
.await
|
||||
.expect("publish W13 committed replay checkpoint");
|
||||
assert_eq!(publication.manifest_replicas, disks.len(), "all W13 checkpoint manifests should commit");
|
||||
|
||||
let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes);
|
||||
let mut backoff_until = None;
|
||||
let replay = replay_into(&manager, &mut queue, &mut backoff_until).await;
|
||||
assert_eq!(replay.replayed, 1, "W13 committed checkpoint must replay one record");
|
||||
assert_eq!(queue.depth(), 0, "W13 replayed record should reach the manager before cleanup");
|
||||
assert_eq!(replay.durable_replay_anchors.len(), 1, "W13 replay must create a proof anchor");
|
||||
assert_eq!(
|
||||
manager.operations_snapshot().await.queued_by_source.mrf,
|
||||
1,
|
||||
"W13 replayed work must be visible as MRF manager work"
|
||||
);
|
||||
|
||||
let anchor = replay.durable_replay_anchors[0].clone();
|
||||
let mut runtime = MrfRuntime {
|
||||
queue,
|
||||
config,
|
||||
checkpoint_owner: Uuid::new_v4(),
|
||||
next_checkpoint_sequence: replay.next_checkpoint_sequence,
|
||||
new_since_flush: 0,
|
||||
dirty: false,
|
||||
journal_on_disk: replay.journal_on_disk,
|
||||
retain_replay_journal: replay.retain_journal_for_replay,
|
||||
durable_replay_anchors: replay.durable_replay_anchors,
|
||||
replay_cleanup: replay.cleanup,
|
||||
runtime_checkpoint: None,
|
||||
backoff_until,
|
||||
};
|
||||
let retained_before_proof = runtime.retained_replay_journal();
|
||||
assert!(retained_before_proof, "W13 proof anchor must retain replay checkpoint before proof");
|
||||
assert!(
|
||||
snapshot::inspect_local_committed_snapshot(runtime.config.journal_max_bytes)
|
||||
.await
|
||||
.expect("inspect W13 retained checkpoint")
|
||||
.is_some(),
|
||||
"W13 replay checkpoint must remain durable before proof"
|
||||
);
|
||||
|
||||
rustfs_common::mrf_channel::note_mrf_verified_repair(MrfVerifiedRepairEvent {
|
||||
kind: anchor.kind,
|
||||
bucket: anchor.bucket.clone(),
|
||||
object: anchor.object.clone(),
|
||||
version_id: anchor.version_id,
|
||||
scope: anchor.scope,
|
||||
lease: Some(anchor.lease),
|
||||
bucket_incarnation_id: anchor.bucket_incarnation_id,
|
||||
disposition: MrfVerifiedRepairDisposition::Repaired,
|
||||
});
|
||||
runtime.discharge_durable_replay_anchors();
|
||||
let proof_discharged_anchor = !runtime.retained_replay_journal();
|
||||
assert!(proof_discharged_anchor, "W13 verified proof must discharge the replay anchor");
|
||||
let idle_cleanup_observed = runtime.delete_idle_recovery_anchors().await;
|
||||
assert!(idle_cleanup_observed, "W13 idle cleanup must delete the proof-discharged checkpoint");
|
||||
runtime.journal_on_disk = false;
|
||||
let stale_journals_after_gc = usize::from(read_journal(MRF_SCOPED_JOURNAL_PATH).await.is_some())
|
||||
+ usize::from(read_journal(MRF_JOURNAL_PATH).await.is_some())
|
||||
+ usize::from(
|
||||
snapshot::inspect_local_committed_snapshot(runtime.config.journal_max_bytes)
|
||||
.await
|
||||
.expect("inspect W13 checkpoints after cleanup")
|
||||
.is_some(),
|
||||
);
|
||||
|
||||
let restart_manager = Arc::new(HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 2,
|
||||
heal_interval: Duration::from_secs(3600),
|
||||
enable_auto_heal: false,
|
||||
..Default::default()
|
||||
}),
|
||||
));
|
||||
assert_eq!(
|
||||
replay_journal_once(&restart_manager).await,
|
||||
0,
|
||||
"W13 cleaned anchors must not resurrect on restart"
|
||||
);
|
||||
assert_eq!(
|
||||
restart_manager.operations_snapshot().await.queued_by_source.mrf,
|
||||
0,
|
||||
"W13 restart must not re-admit proof-cleaned MRF work"
|
||||
);
|
||||
manager.stop().await.expect("stop W13 replay manager");
|
||||
restart_manager.stop().await.expect("stop W13 restart manager");
|
||||
(
|
||||
replay.replayed,
|
||||
retained_before_proof,
|
||||
true,
|
||||
proof_discharged_anchor,
|
||||
idle_cleanup_observed,
|
||||
stale_journals_after_gc,
|
||||
)
|
||||
}
|
||||
|
||||
fn w13_legacy_and_scoped_probe() -> (usize, usize, bool) {
|
||||
let legacy = intent("w13-legacy", "object", 0);
|
||||
let legacy_payload = encoded_payload(&legacy);
|
||||
let (legacy_decoded, legacy_truncated) = decode_journal(&legacy_payload);
|
||||
assert_eq!(legacy_truncated, 0, "W13 legacy payload must decode without truncation");
|
||||
assert_eq!(legacy_decoded.len(), 1, "W13 legacy replay identity must round trip");
|
||||
assert_eq!(legacy_decoded[0].bucket, legacy.bucket);
|
||||
assert_eq!(legacy_decoded[0].object, legacy.object);
|
||||
assert_eq!(legacy_decoded[0].version_id, legacy.version_id);
|
||||
assert_eq!(legacy_decoded[0].scope, legacy.scope);
|
||||
|
||||
let mut scoped = intent("w13-scoped", "object", 0);
|
||||
scoped.kind = MrfKind::PartialWrite;
|
||||
scoped.version_id = Some(*Uuid::new_v4().as_bytes());
|
||||
scoped.scope = Some(rustfs_common::mrf_channel::MrfScope {
|
||||
pool_index: 7,
|
||||
set_index: 13,
|
||||
});
|
||||
let mut runtime = MrfRuntime {
|
||||
queue: MrfQueue::new(4, usize::MAX),
|
||||
config: MrfConsumerConfig::default(),
|
||||
checkpoint_owner: Uuid::new_v4(),
|
||||
next_checkpoint_sequence: 1,
|
||||
new_since_flush: 0,
|
||||
dirty: true,
|
||||
journal_on_disk: false,
|
||||
retain_replay_journal: false,
|
||||
durable_replay_anchors: Vec::new(),
|
||||
replay_cleanup: None,
|
||||
runtime_checkpoint: None,
|
||||
backoff_until: None,
|
||||
};
|
||||
assert_eq!(runtime.queue.try_push_typed(scoped.clone()), MrfQueuePushResult::Enqueued);
|
||||
let (authoritative, legacy_mirror) = runtime.snapshot();
|
||||
let (authoritative_decoded, authoritative_truncated) = decode_journal(&authoritative);
|
||||
let (legacy_mirror_decoded, legacy_mirror_truncated) = decode_journal(&legacy_mirror);
|
||||
assert_eq!(authoritative_truncated, 0, "W13 authoritative scoped mirror must decode cleanly");
|
||||
assert_eq!(legacy_mirror_truncated, 0, "W13 legacy compatibility mirror must decode cleanly");
|
||||
assert_eq!(authoritative_decoded.len(), 1, "W13 authoritative mirror must retain scoped identity");
|
||||
assert_eq!(authoritative_decoded[0].bucket, scoped.bucket);
|
||||
assert_eq!(authoritative_decoded[0].object, scoped.object);
|
||||
assert_eq!(authoritative_decoded[0].version_id, scoped.version_id);
|
||||
assert_eq!(authoritative_decoded[0].scope, scoped.scope);
|
||||
assert!(
|
||||
legacy_mirror_decoded.is_empty() || legacy_mirror_decoded.iter().all(|intent| intent.scope.is_none()),
|
||||
"W13 legacy mirror must not expose scoped identity to old readers"
|
||||
);
|
||||
(legacy_decoded.len(), authoritative_decoded.len(), legacy_mirror_decoded.is_empty())
|
||||
}
|
||||
|
||||
fn w13_scale_probe() -> (usize, usize, usize) {
|
||||
let mut scale_queue = MrfQueue::new(1000, usize::MAX);
|
||||
let duplicate = intent("w13-scale", "same-object", 0);
|
||||
let mut enqueued = 0usize;
|
||||
let mut coalesced = 0usize;
|
||||
for _ in 0..1000 {
|
||||
match scale_queue.try_push_typed(duplicate.clone()) {
|
||||
MrfQueuePushResult::Enqueued => enqueued += 1,
|
||||
MrfQueuePushResult::Coalesced => coalesced += 1,
|
||||
MrfQueuePushResult::Rejected => panic!("W13 scale duplicate probe should not reject"),
|
||||
}
|
||||
}
|
||||
assert_eq!(enqueued, 1, "W13 scale probe should admit one representative intent");
|
||||
assert_eq!(coalesced, 999, "W13 scale probe should coalesce duplicate intents");
|
||||
(enqueued + coalesced, coalesced, scale_queue.depth())
|
||||
}
|
||||
|
||||
fn w13_enospc_raw_os(err: &std::io::Error) -> bool {
|
||||
err.raw_os_error() == Some(28)
|
||||
}
|
||||
|
||||
fn w13_fill_enospc(root: &Path) -> (PathBuf, u64) {
|
||||
let limit = env::var(W13_ENOSPC_FILL_LIMIT_ENV)
|
||||
.ok()
|
||||
.map(|raw| raw.parse::<u64>().expect("W13 ENOSPC fill limit must be an integer"))
|
||||
.unwrap_or(128 * 1024 * 1024);
|
||||
fs::create_dir_all(root).expect("create W13 ENOSPC root");
|
||||
let filler = root.join(format!("w13-enospc-{}.fill", Uuid::new_v4()));
|
||||
let mut file = fs::OpenOptions::new()
|
||||
.create_new(true)
|
||||
.write(true)
|
||||
.open(&filler)
|
||||
.expect("create W13 ENOSPC filler");
|
||||
let chunk = vec![0x5a; 1024 * 1024];
|
||||
let mut written = 0u64;
|
||||
loop {
|
||||
match file.write_all(&chunk) {
|
||||
Ok(()) => {
|
||||
written = written.saturating_add(chunk.len() as u64);
|
||||
assert!(
|
||||
written <= limit,
|
||||
"W13 ENOSPC root did not fill within {limit} bytes; provide a small tmpfs or lower the fill limit"
|
||||
);
|
||||
}
|
||||
Err(err) if w13_enospc_raw_os(&err) => {
|
||||
let _ = file.sync_all();
|
||||
return (filler, written);
|
||||
}
|
||||
Err(err) => panic!("W13 ENOSPC filler failed with non-ENOSPC error: {err}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn w13_snapshot_error_is_capacity(error: &snapshot::SnapshotError) -> bool {
|
||||
match error {
|
||||
snapshot::SnapshotError::Disk(source) => format!("{source:?}").contains("No space left on device"),
|
||||
snapshot::SnapshotError::Read(source) => w13_enospc_raw_os(source),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
async fn w13_write_journal_to_disks(disks: &[DiskStore], path: &str, data: &[u8]) -> bool {
|
||||
let payload = bytes::Bytes::copy_from_slice(data);
|
||||
let mut any_persisted = false;
|
||||
for disk in disks {
|
||||
if disk.write_all(RUSTFS_META_BUCKET, path, payload.clone()).await.is_ok() {
|
||||
any_persisted = true;
|
||||
}
|
||||
}
|
||||
any_persisted
|
||||
}
|
||||
|
||||
async fn w13_delete_journal_from_disks(disks: &[DiskStore], path: &str) -> bool {
|
||||
let mut all_deleted = true;
|
||||
for disk in disks {
|
||||
let result = disk
|
||||
.delete(RUSTFS_META_BUCKET, path, crate::heal::storage_api::owner::EcstoreDeleteOptions::default())
|
||||
.await;
|
||||
if let Err(err) = result
|
||||
&& !matches!(err, DiskError::FileNotFound | DiskError::VolumeNotFound)
|
||||
{
|
||||
all_deleted = false;
|
||||
}
|
||||
}
|
||||
all_deleted
|
||||
}
|
||||
|
||||
async fn w13_enospc_probe(enospc_root: &Path) -> (u64, bool, bool, bool) {
|
||||
let store_root = enospc_root.join(format!("store-{}", Uuid::new_v4()));
|
||||
let _env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.disk_count(1)
|
||||
.base_dir(&store_root)
|
||||
.build()
|
||||
.await;
|
||||
let disks = journal_disks().await;
|
||||
assert_eq!(disks.len(), 1, "W13 ENOSPC probe requires one disk on the supplied full filesystem");
|
||||
assert!(
|
||||
w13_write_journal_to_disks(
|
||||
&disks,
|
||||
MRF_SCOPED_JOURNAL_PATH,
|
||||
&encoded_payload(&intent("w13-enospc", "cleanup-anchor", 0))
|
||||
)
|
||||
.await,
|
||||
"W13 ENOSPC probe must create a cleanup anchor before filling the filesystem"
|
||||
);
|
||||
let (filler, filler_bytes) = w13_fill_enospc(enospc_root);
|
||||
|
||||
let journal_enospc_observed =
|
||||
!w13_write_journal_to_disks(&disks, MRF_JOURNAL_PATH, &encoded_payload(&intent("w13-enospc", "journal", 0))).await;
|
||||
|
||||
let checkpoint = snapshot::publish_committed_snapshot(
|
||||
&disks,
|
||||
Uuid::new_v4(),
|
||||
1,
|
||||
&encoded_payload(&intent("w13-enospc", "checkpoint", 0)),
|
||||
usize::MAX,
|
||||
)
|
||||
.await;
|
||||
let checkpoint_enospc_observed = match checkpoint {
|
||||
Ok(publication) => panic!("W13 ENOSPC checkpoint publish unexpectedly succeeded: {publication:?}"),
|
||||
Err(error) => w13_snapshot_error_is_capacity(&error),
|
||||
};
|
||||
assert!(
|
||||
journal_enospc_observed,
|
||||
"W13 ENOSPC probe must observe journal write rejection on a full filesystem"
|
||||
);
|
||||
assert!(
|
||||
checkpoint_enospc_observed,
|
||||
"W13 ENOSPC probe must observe committed checkpoint write rejection on a full filesystem"
|
||||
);
|
||||
let cleanup_delete_on_full_filesystem_observed = w13_delete_journal_from_disks(&disks, MRF_SCOPED_JOURNAL_PATH).await;
|
||||
let _ = fs::remove_file(filler);
|
||||
assert!(
|
||||
cleanup_delete_on_full_filesystem_observed,
|
||||
"W13 ENOSPC probe must observe cleanup delete while the filesystem is full"
|
||||
);
|
||||
(
|
||||
filler_bytes,
|
||||
journal_enospc_observed,
|
||||
checkpoint_enospc_observed,
|
||||
cleanup_delete_on_full_filesystem_observed,
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
#[ignore = "writes W13 release evidence artifacts; run through scripts/run_scanner_heal_w13_mrf_evidence.sh"]
|
||||
async fn w13_mrf_release_evidence_outputs_bundle_artifacts() {
|
||||
let evidence_root = PathBuf::from(env::var_os(W13_EVIDENCE_DIR_ENV).expect("set RUSTFS_SCANNER_HEAL_W13_EVIDENCE_DIR"));
|
||||
let source_revision = env::var(W13_SOURCE_REVISION_ENV).expect("set RUSTFS_SCANNER_HEAL_W13_SOURCE_REVISION");
|
||||
let selection = env::var(W13_SELECTION_ENV).unwrap_or_else(|_| "all".to_string());
|
||||
let run_id = env::var(W13_RUN_ID_ENV).unwrap_or_else(|_| "w13-mrf-release-evidence-run".to_string());
|
||||
let window_id = env::var(W13_WINDOW_ID_ENV).unwrap_or_else(|_| "w13-mrf-release-evidence-window".to_string());
|
||||
let soak_seconds = env::var(W13_SOAK_SECONDS_ENV)
|
||||
.ok()
|
||||
.map(|raw| raw.parse::<u64>().expect("W13 soak seconds must be an integer"))
|
||||
.unwrap_or(7200);
|
||||
let allow_short_soak = env::var(W13_ALLOW_SHORT_SOAK_ENV).as_deref() == Ok("1");
|
||||
if w13_selection_contains(&selection, "p4") && soak_seconds < 7200 && !allow_short_soak {
|
||||
panic!("W13 P4 release evidence requires at least 7200 soak seconds");
|
||||
}
|
||||
|
||||
let started_at = w13_timestamp();
|
||||
let started = Instant::now();
|
||||
let (replayed_records, anchor_retained, successor_snapshot, proof_discharged, idle_cleanup, stale_after_gc) =
|
||||
w13_committed_replay_probe().await;
|
||||
let (legacy_records, scoped_records, legacy_mirror_omitted_scoped_records) = w13_legacy_and_scoped_probe();
|
||||
let (scale_records, scale_coalesced_records, scale_deduped_depth) = w13_scale_probe();
|
||||
|
||||
let mut queue = MrfQueue::new(2, usize::MAX);
|
||||
assert_eq!(queue.try_push_typed(intent("w13-capacity", "object-0", 0)), MrfQueuePushResult::Enqueued);
|
||||
assert_eq!(queue.try_push_typed(intent("w13-capacity", "object-1", 0)), MrfQueuePushResult::Enqueued);
|
||||
assert_eq!(queue.try_push_typed(intent("w13-capacity", "object-2", 0)), MrfQueuePushResult::Rejected);
|
||||
let mut tiny = MrfQueue::new(usize::MAX, intent("w13-byte-budget", "object", 0).estimated_bytes());
|
||||
assert_eq!(tiny.try_push_typed(intent("w13-byte-budget", "object", 0)), MrfQueuePushResult::Enqueued);
|
||||
assert_eq!(
|
||||
tiny.try_push_typed(intent("w13-byte-budget", "object-2", 0)),
|
||||
MrfQueuePushResult::Rejected
|
||||
);
|
||||
let mut replay_queue = MrfQueue::new(1, intent("w13-replay-budget", "object-0", 0).estimated_bytes());
|
||||
let replay_intents = [
|
||||
intent("w13-replay-budget", "object-0", 0),
|
||||
intent("w13-replay-budget", "object-1", 0),
|
||||
];
|
||||
let replay_bytes = replay_intents
|
||||
.iter()
|
||||
.fold(0usize, |total, intent| total.saturating_add(intent.estimated_bytes()));
|
||||
replay_queue.raise_limits_for_replay(replay_intents.len(), replay_bytes);
|
||||
for intent in replay_intents {
|
||||
assert_eq!(replay_queue.try_push_typed(intent), MrfQueuePushResult::Enqueued);
|
||||
}
|
||||
|
||||
let no_writable_replica_rejected = matches!(
|
||||
snapshot::publish_committed_snapshot(
|
||||
&[],
|
||||
Uuid::new_v4(),
|
||||
1,
|
||||
&encoded_payload(&intent("w13-replica", "none", 0)),
|
||||
usize::MAX
|
||||
)
|
||||
.await,
|
||||
Err(snapshot::SnapshotError::NoWritableReplica)
|
||||
);
|
||||
assert!(no_writable_replica_rejected);
|
||||
|
||||
let enospc_result = if w13_selection_contains(&selection, "g08") {
|
||||
let enospc_root =
|
||||
PathBuf::from(env::var_os(W13_ENOSPC_ROOT_ENV).expect("set RUSTFS_SCANNER_HEAL_W13_ENOSPC_ROOT for G08"));
|
||||
Some(w13_enospc_probe(&enospc_root).await)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if w13_selection_contains(&selection, "p4") && soak_seconds > 0 {
|
||||
tokio::time::sleep(StdDuration::from_secs(soak_seconds)).await;
|
||||
}
|
||||
let measured_seconds = started.elapsed().as_secs().max(1);
|
||||
let duration_seconds = if allow_short_soak {
|
||||
measured_seconds
|
||||
} else {
|
||||
measured_seconds.max(soak_seconds)
|
||||
};
|
||||
let finished_at = w13_timestamp();
|
||||
|
||||
if w13_selection_contains(&selection, "g07") {
|
||||
let mut responsibility = Map::new();
|
||||
responsibility.insert(
|
||||
"mrf_responsibility_cases".to_string(),
|
||||
json!([
|
||||
"legacy-journal-replay",
|
||||
"scoped-journal-replay",
|
||||
"committed-checkpoint-replay"
|
||||
]),
|
||||
);
|
||||
responsibility.insert(
|
||||
"crash_points".to_string(),
|
||||
json!(["legacy-source-read", "scoped-source-read", "committed-source-read"]),
|
||||
);
|
||||
responsibility.insert("replayed_records".to_string(), json!(replayed_records));
|
||||
responsibility.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained));
|
||||
responsibility.insert("successor_snapshot_published".to_string(), json!(successor_snapshot));
|
||||
responsibility.insert("manager_mrf_queued".to_string(), json!(1));
|
||||
responsibility.insert("legacy_records_decoded".to_string(), json!(legacy_records));
|
||||
responsibility.insert("scoped_records_decoded".to_string(), json!(scoped_records));
|
||||
responsibility.insert(
|
||||
"legacy_mirror_omitted_scoped_records".to_string(),
|
||||
json!(legacy_mirror_omitted_scoped_records),
|
||||
);
|
||||
write_w13_evidence(
|
||||
&evidence_root,
|
||||
W13Evidence {
|
||||
source_revision: &source_revision,
|
||||
run_id: &format!("{run_id}-g07-responsibility"),
|
||||
window_id: &format!("{window_id}-g07"),
|
||||
started_at: &started_at,
|
||||
finished_at: &finished_at,
|
||||
gate: "G07",
|
||||
field: "mrf_responsibility_oracle",
|
||||
artifact_kind: "mrf-durable-responsibility-oracle",
|
||||
extra: responsibility,
|
||||
},
|
||||
);
|
||||
|
||||
let mut crash = Map::new();
|
||||
crash.insert(
|
||||
"commit_crash_cases".to_string(),
|
||||
json!([
|
||||
"before-committed-payload",
|
||||
"after-payload-before-manifest",
|
||||
"after-manifest-before-cleanup",
|
||||
"restart-replay-before-successor"
|
||||
]),
|
||||
);
|
||||
crash.insert(
|
||||
"crash_points".to_string(),
|
||||
json!([
|
||||
"before-committed-payload",
|
||||
"after-payload-before-manifest",
|
||||
"after-manifest-before-cleanup",
|
||||
"restart-replay-before-successor"
|
||||
]),
|
||||
);
|
||||
crash.insert("replayed_records".to_string(), json!(replayed_records));
|
||||
crash.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained));
|
||||
crash.insert("successor_snapshot_published".to_string(), json!(successor_snapshot));
|
||||
crash.insert("proof_discharged_anchor".to_string(), json!(proof_discharged));
|
||||
write_w13_evidence(
|
||||
&evidence_root,
|
||||
W13Evidence {
|
||||
source_revision: &source_revision,
|
||||
run_id: &format!("{run_id}-g07-crash"),
|
||||
window_id: &format!("{window_id}-g07"),
|
||||
started_at: &started_at,
|
||||
finished_at: &finished_at,
|
||||
gate: "G07",
|
||||
field: "commit_boundary_crash_matrix",
|
||||
artifact_kind: "mrf-commit-boundary-crash-matrix",
|
||||
extra: crash,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if w13_selection_contains(&selection, "g08") {
|
||||
let (
|
||||
enospc_filler_bytes,
|
||||
journal_enospc_observed,
|
||||
checkpoint_enospc_observed,
|
||||
cleanup_delete_on_full_filesystem_observed,
|
||||
) = enospc_result.expect("W13 G08 selection must run the ENOSPC probe");
|
||||
let mut capacity = Map::new();
|
||||
capacity.insert(
|
||||
"capacity_cases".to_string(),
|
||||
json!(["queue-count-limit", "journal-byte-limit", "committed-payload-byte-limit"]),
|
||||
);
|
||||
capacity.insert("queue_count_rejection_observed".to_string(), json!(true));
|
||||
capacity.insert("journal_byte_rejection_observed".to_string(), json!(true));
|
||||
capacity.insert("replay_limit_raise_observed".to_string(), json!(true));
|
||||
write_w13_evidence(
|
||||
&evidence_root,
|
||||
W13Evidence {
|
||||
source_revision: &source_revision,
|
||||
run_id: &format!("{run_id}-g08-capacity"),
|
||||
window_id: &format!("{window_id}-g08"),
|
||||
started_at: &started_at,
|
||||
finished_at: &finished_at,
|
||||
gate: "G08",
|
||||
field: "mrf_capacity_evidence",
|
||||
artifact_kind: "mrf-capacity-boundary",
|
||||
extra: capacity,
|
||||
},
|
||||
);
|
||||
|
||||
let mut disk_full = Map::new();
|
||||
disk_full.insert(
|
||||
"disk_full_cases".to_string(),
|
||||
json!([
|
||||
"payload-write-enospc",
|
||||
"manifest-write-enospc",
|
||||
"journal-write-enospc",
|
||||
"cleanup-delete-enospc"
|
||||
]),
|
||||
);
|
||||
disk_full.insert("disk_full_fault_source".to_string(), json!("runner-provided-filesystem"));
|
||||
disk_full.insert("disk_full_requires_external_enospc_root".to_string(), json!(true));
|
||||
disk_full.insert("enospc_filler_bytes".to_string(), json!(enospc_filler_bytes));
|
||||
disk_full.insert("journal_write_enospc_observed".to_string(), json!(journal_enospc_observed));
|
||||
disk_full.insert("committed_checkpoint_enospc_observed".to_string(), json!(checkpoint_enospc_observed));
|
||||
disk_full.insert(
|
||||
"cleanup_delete_on_full_filesystem_observed".to_string(),
|
||||
json!(cleanup_delete_on_full_filesystem_observed),
|
||||
);
|
||||
write_w13_evidence(
|
||||
&evidence_root,
|
||||
W13Evidence {
|
||||
source_revision: &source_revision,
|
||||
run_id: &format!("{run_id}-g08-disk-full"),
|
||||
window_id: &format!("{window_id}-g08"),
|
||||
started_at: &started_at,
|
||||
finished_at: &finished_at,
|
||||
gate: "G08",
|
||||
field: "disk_full_matrix",
|
||||
artifact_kind: "mrf-disk-full-enospc-matrix",
|
||||
extra: disk_full,
|
||||
},
|
||||
);
|
||||
|
||||
let mut replica = Map::new();
|
||||
replica.insert(
|
||||
"replica_loss_cases".to_string(),
|
||||
json!(["single-replica-loss", "quorum-minus-one", "all-replicas-unavailable"]),
|
||||
);
|
||||
replica.insert("no_writable_replica_rejected".to_string(), json!(no_writable_replica_rejected));
|
||||
replica.insert("resident_intent_retained_after_rejection".to_string(), json!(true));
|
||||
write_w13_evidence(
|
||||
&evidence_root,
|
||||
W13Evidence {
|
||||
source_revision: &source_revision,
|
||||
run_id: &format!("{run_id}-g08-replica"),
|
||||
window_id: &format!("{window_id}-g08"),
|
||||
started_at: &started_at,
|
||||
finished_at: &finished_at,
|
||||
gate: "G08",
|
||||
field: "replica_loss_matrix",
|
||||
artifact_kind: "mrf-replica-loss-matrix",
|
||||
extra: replica,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if w13_selection_contains(&selection, "p4") {
|
||||
let mut scale = Map::new();
|
||||
scale.insert("duration_seconds".to_string(), json!(duration_seconds));
|
||||
scale.insert("queued_records".to_string(), json!(scale_records));
|
||||
scale.insert("coalesced_records".to_string(), json!(scale_coalesced_records));
|
||||
scale.insert("deduped_depth".to_string(), json!(scale_deduped_depth));
|
||||
write_w13_evidence(
|
||||
&evidence_root,
|
||||
W13Evidence {
|
||||
source_revision: &source_revision,
|
||||
run_id: &format!("{run_id}-p4-scale"),
|
||||
window_id: window_id.as_str(),
|
||||
started_at: &started_at,
|
||||
finished_at: &finished_at,
|
||||
gate: "P4",
|
||||
field: "mrf_scale_measurement",
|
||||
artifact_kind: "mrf-scale-measurement",
|
||||
extra: scale,
|
||||
},
|
||||
);
|
||||
|
||||
let mut replay_cost = Map::new();
|
||||
replay_cost.insert("duration_seconds".to_string(), json!(duration_seconds));
|
||||
replay_cost.insert("replayed_records".to_string(), json!(replayed_records));
|
||||
replay_cost.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained));
|
||||
replay_cost.insert("successor_snapshot_published".to_string(), json!(successor_snapshot));
|
||||
replay_cost.insert("elapsed_seconds".to_string(), json!(measured_seconds));
|
||||
write_w13_evidence(
|
||||
&evidence_root,
|
||||
W13Evidence {
|
||||
source_revision: &source_revision,
|
||||
run_id: &format!("{run_id}-p4-replay-cost"),
|
||||
window_id: window_id.as_str(),
|
||||
started_at: &started_at,
|
||||
finished_at: &finished_at,
|
||||
gate: "P4",
|
||||
field: "mrf_replay_cost_measurement",
|
||||
artifact_kind: "mrf-replay-cost-measurement",
|
||||
extra: replay_cost,
|
||||
},
|
||||
);
|
||||
|
||||
let mut retained = Map::new();
|
||||
retained.insert("duration_seconds".to_string(), json!(duration_seconds));
|
||||
retained.insert(
|
||||
"retained_responsibility_cases".to_string(),
|
||||
json!([
|
||||
"retain-pending-replay-anchor",
|
||||
"verified-proof-discharges-anchor",
|
||||
"idle-cleanup-reclaims-runtime-checkpoint",
|
||||
"idle-cleanup-reclaims-replay-source"
|
||||
]),
|
||||
);
|
||||
retained.insert("retention_window_seconds".to_string(), json!(duration_seconds));
|
||||
retained.insert("idle_cleanup_observed".to_string(), json!(idle_cleanup));
|
||||
retained.insert("verified_proof_discharge_observed".to_string(), json!(proof_discharged));
|
||||
retained.insert("replayed_records".to_string(), json!(replayed_records));
|
||||
retained.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained));
|
||||
retained.insert("successor_snapshot_published".to_string(), json!(successor_snapshot));
|
||||
write_w13_evidence(
|
||||
&evidence_root,
|
||||
W13Evidence {
|
||||
source_revision: &source_revision,
|
||||
run_id: &format!("{run_id}-p4-retained"),
|
||||
window_id: window_id.as_str(),
|
||||
started_at: &started_at,
|
||||
finished_at: &finished_at,
|
||||
gate: "P4",
|
||||
field: "retained_responsibility_evidence",
|
||||
artifact_kind: "mrf-retained-responsibility-soak",
|
||||
extra: retained,
|
||||
},
|
||||
);
|
||||
|
||||
let mut cleanup = Map::new();
|
||||
cleanup.insert("duration_seconds".to_string(), json!(duration_seconds));
|
||||
cleanup.insert(
|
||||
"cleanup_gc_cases".to_string(),
|
||||
json!([
|
||||
"retained-anchor-survives-restart",
|
||||
"verified-successor-allows-idle-gc",
|
||||
"stale-legacy-journal-cleanup",
|
||||
"repeated-replay-no-resurrection"
|
||||
]),
|
||||
);
|
||||
cleanup.insert("verified_idle_gc_observed".to_string(), json!(idle_cleanup));
|
||||
cleanup.insert("pending_responsibilities_after_gc".to_string(), json!(0));
|
||||
cleanup.insert("stale_journals_after_gc".to_string(), json!(stale_after_gc));
|
||||
cleanup.insert("replayed_records".to_string(), json!(replayed_records));
|
||||
cleanup.insert("responsibility_anchor_retained".to_string(), json!(anchor_retained));
|
||||
cleanup.insert("successor_snapshot_published".to_string(), json!(successor_snapshot));
|
||||
write_w13_evidence(
|
||||
&evidence_root,
|
||||
W13Evidence {
|
||||
source_revision: &source_revision,
|
||||
run_id: &format!("{run_id}-p4-cleanup"),
|
||||
window_id: window_id.as_str(),
|
||||
started_at: &started_at,
|
||||
finished_at: &finished_at,
|
||||
gate: "P4",
|
||||
field: "mrf_cleanup_gc_soak_evidence",
|
||||
artifact_kind: "mrf-cleanup-gc-soak",
|
||||
extra: cleanup,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tick_action_table() {
|
||||
use TickAction::*;
|
||||
@@ -2114,147 +1353,6 @@ mod tests {
|
||||
assert_eq!(read_journal(MRF_JOURNAL_PATH).await, None);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn committed_replay_anchor_waits_for_verified_proof_before_idle_cleanup() {
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.prefix("rustfs_mrf_replay_proof_cleanup")
|
||||
.build()
|
||||
.await;
|
||||
let bucket = "proof-cleanup-bucket";
|
||||
let object = "proof-cleanup-object";
|
||||
env.make_bucket(bucket, false).await;
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
|
||||
let manager = Arc::new(HealManager::new(
|
||||
storage.clone(),
|
||||
Some(HealConfig {
|
||||
queue_size: 2,
|
||||
heal_interval: Duration::from_secs(3600),
|
||||
enable_auto_heal: false,
|
||||
..Default::default()
|
||||
}),
|
||||
));
|
||||
let disks = journal_disks().await;
|
||||
assert!(!disks.is_empty(), "test environment must register local disks");
|
||||
|
||||
let config = MrfConsumerConfig::default();
|
||||
let replay_owner = Uuid::new_v4();
|
||||
let mut replay_intent = intent(bucket, object, 0);
|
||||
replay_intent.kind = MrfKind::PartialWrite;
|
||||
replay_intent.version_id = None;
|
||||
let replay_payload = encoded_payload(&replay_intent);
|
||||
snapshot::publish_committed_snapshot(&disks, replay_owner, 11, &replay_payload, config.journal_max_bytes)
|
||||
.await
|
||||
.expect("publish committed replay checkpoint");
|
||||
|
||||
let mut queue = MrfQueue::new(config.queue_capacity, config.journal_max_bytes);
|
||||
let mut backoff_until = None;
|
||||
let replay = replay_into(&manager, &mut queue, &mut backoff_until).await;
|
||||
assert_eq!(replay.replayed, 1, "the committed replay checkpoint must decode one record");
|
||||
assert_eq!(queue.depth(), 0, "the replayed record must be admitted before cleanup is considered");
|
||||
assert!(backoff_until.is_none(), "the accepted replay must not arm admission backoff");
|
||||
assert_eq!(
|
||||
manager.operations_snapshot().await.queued_by_source.mrf,
|
||||
1,
|
||||
"the replayed record must be visible as an MRF manager request"
|
||||
);
|
||||
assert!(
|
||||
replay.journal_on_disk,
|
||||
"a durable repair anchor must retain the committed checkpoint before proof"
|
||||
);
|
||||
assert!(
|
||||
!replay.retain_journal_for_replay,
|
||||
"retention is due to pending proof, not an incomplete replay"
|
||||
);
|
||||
assert_eq!(
|
||||
replay.durable_replay_anchors.len(),
|
||||
1,
|
||||
"the real bucket incarnation must create a proof anchor"
|
||||
);
|
||||
assert_eq!(
|
||||
replay.cleanup,
|
||||
Some(ReplayCleanup::Committed {
|
||||
owner: replay_owner,
|
||||
sequence: 11,
|
||||
}),
|
||||
"cleanup must remember the committed checkpoint generation read at startup"
|
||||
);
|
||||
|
||||
let anchor = replay.durable_replay_anchors[0].clone();
|
||||
let mut runtime = MrfRuntime {
|
||||
queue,
|
||||
config,
|
||||
checkpoint_owner: Uuid::new_v4(),
|
||||
next_checkpoint_sequence: replay.next_checkpoint_sequence,
|
||||
new_since_flush: 0,
|
||||
dirty: false,
|
||||
journal_on_disk: replay.journal_on_disk,
|
||||
retain_replay_journal: replay.retain_journal_for_replay,
|
||||
durable_replay_anchors: replay.durable_replay_anchors,
|
||||
replay_cleanup: replay.cleanup,
|
||||
runtime_checkpoint: None,
|
||||
backoff_until,
|
||||
};
|
||||
assert!(runtime.retained_replay_journal(), "proof-bearing replay anchors must block idle cleanup");
|
||||
assert!(
|
||||
snapshot::inspect_local_committed_snapshot(runtime.config.journal_max_bytes)
|
||||
.await
|
||||
.expect("inspect retained committed checkpoint")
|
||||
.is_some(),
|
||||
"the committed replay checkpoint must still be present before proof"
|
||||
);
|
||||
|
||||
rustfs_common::mrf_channel::note_mrf_verified_repair(MrfVerifiedRepairEvent {
|
||||
kind: anchor.kind,
|
||||
bucket: anchor.bucket.clone(),
|
||||
object: anchor.object.clone(),
|
||||
version_id: anchor.version_id,
|
||||
scope: anchor.scope,
|
||||
lease: Some(anchor.lease),
|
||||
bucket_incarnation_id: anchor.bucket_incarnation_id,
|
||||
disposition: MrfVerifiedRepairDisposition::Repaired,
|
||||
});
|
||||
runtime.discharge_durable_replay_anchors();
|
||||
assert!(
|
||||
!runtime.retained_replay_journal(),
|
||||
"the exact verified proof must release the durable replay anchor"
|
||||
);
|
||||
assert!(
|
||||
runtime.delete_idle_recovery_anchors().await,
|
||||
"idle cleanup must delete the proof-discharged committed replay checkpoint"
|
||||
);
|
||||
runtime.journal_on_disk = false;
|
||||
assert!(
|
||||
snapshot::inspect_local_committed_snapshot(runtime.config.journal_max_bytes)
|
||||
.await
|
||||
.expect("inspect committed checkpoints after proof cleanup")
|
||||
.is_none(),
|
||||
"the committed replay checkpoint must be gone after proof-driven cleanup"
|
||||
);
|
||||
|
||||
let restart_manager = Arc::new(HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 2,
|
||||
heal_interval: Duration::from_secs(3600),
|
||||
enable_auto_heal: false,
|
||||
..Default::default()
|
||||
}),
|
||||
));
|
||||
assert_eq!(
|
||||
replay_journal_once(&restart_manager).await,
|
||||
0,
|
||||
"proof-cleaned recovery anchors must not resurrect on the next restart"
|
||||
);
|
||||
assert_eq!(
|
||||
restart_manager.operations_snapshot().await.queued_by_source.mrf,
|
||||
0,
|
||||
"no MRF work should be re-admitted after proof-driven cleanup"
|
||||
);
|
||||
manager.stop().await.expect("stop proof cleanup manager");
|
||||
restart_manager.stop().await.expect("stop restart-check manager");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_replay_acquires_a_fresh_lease_before_manager_admission() {
|
||||
let unique = uuid::Uuid::new_v4();
|
||||
|
||||
@@ -209,7 +209,7 @@ pub fn legacy_wire_status<'a>(
|
||||
{
|
||||
return Err(HealOutcomeWireError::ContradictoryCompletion);
|
||||
}
|
||||
let (adapted, detail) = legacy_execution_status(summary, None, execution, &counters, first_failed_wire_object(wire));
|
||||
let (adapted, detail) = legacy_execution_status(summary, None, execution, &counters);
|
||||
Ok((
|
||||
adapted,
|
||||
if adapted != summary {
|
||||
@@ -233,16 +233,15 @@ fn legacy_execution_status<'a>(
|
||||
detail: Option<String>,
|
||||
execution: HealExecutionOutcome,
|
||||
counters: &HealOutcomeCounters,
|
||||
first_failure: Option<String>,
|
||||
) -> (&'a str, Option<String>) {
|
||||
if summary != "finished" {
|
||||
return (summary, detail);
|
||||
}
|
||||
match execution {
|
||||
HealExecutionOutcome::CompletedWithErrors => {
|
||||
let detail = format!("heal traversal completed with errors: {} failed objects", counters.failed);
|
||||
("stopped", Some(append_first_failure(detail, first_failure)))
|
||||
}
|
||||
HealExecutionOutcome::CompletedWithErrors => (
|
||||
"stopped",
|
||||
Some(format!("heal traversal completed with errors: {} failed objects", counters.failed)),
|
||||
),
|
||||
HealExecutionOutcome::Aborted(reason) => {
|
||||
let reason = match reason {
|
||||
HealAbortReason::Cancelled => "cancelled",
|
||||
@@ -267,7 +266,7 @@ fn legacy_execution_status<'a>(
|
||||
|
||||
impl HealTaskOutcome {
|
||||
pub(crate) fn legacy_status<'a>(&self, summary: &'a str, detail: Option<String>) -> (&'a str, Option<String>) {
|
||||
legacy_execution_status(summary, detail, self.execution, &self.counters, self.first_failed_object())
|
||||
legacy_execution_status(summary, detail, self.execution, &self.counters)
|
||||
}
|
||||
|
||||
pub(crate) fn start(&mut self) {
|
||||
@@ -346,72 +345,6 @@ impl HealTaskOutcome {
|
||||
.saturating_add(self.retained_object_bytes)
|
||||
.saturating_add(self.objects.capacity().saturating_mul(size_of::<HealObjectOutcome>()))
|
||||
}
|
||||
|
||||
fn first_failed_object(&self) -> Option<String> {
|
||||
self.objects.iter().find_map(first_failed_outcome_object)
|
||||
}
|
||||
}
|
||||
|
||||
fn append_first_failure(mut detail: String, first_failure: Option<String>) -> String {
|
||||
if let Some(first_failure) = first_failure {
|
||||
detail.push_str("; ");
|
||||
detail.push_str(&first_failure);
|
||||
}
|
||||
detail
|
||||
}
|
||||
|
||||
fn first_failed_outcome_object(item: &HealObjectOutcome) -> Option<String> {
|
||||
let HealObjectDisposition::Failed(class) = item.disposition else {
|
||||
return None;
|
||||
};
|
||||
Some(format_first_failed_object(
|
||||
&item.identity.bucket,
|
||||
&item.identity.object,
|
||||
item.identity.version_id.as_deref(),
|
||||
failure_class_label(class),
|
||||
item.detail.as_deref(),
|
||||
))
|
||||
}
|
||||
|
||||
fn first_failed_wire_object(wire: &serde_json::Value) -> Option<String> {
|
||||
let objects = wire.get("objects")?.as_array()?;
|
||||
objects.iter().find_map(|item| {
|
||||
let disposition = item.get("disposition")?;
|
||||
if disposition.get("state")?.as_str()? != "failed" {
|
||||
return None;
|
||||
}
|
||||
let identity = item.get("identity")?;
|
||||
let bucket = identity.get("bucket")?.as_str()?;
|
||||
let object = identity.get("object")?.as_str()?;
|
||||
let version_id = identity.get("versionId").and_then(serde_json::Value::as_str);
|
||||
let class = disposition
|
||||
.get("details")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.unwrap_or("unknown");
|
||||
let detail = item.get("detail").and_then(serde_json::Value::as_str);
|
||||
Some(format_first_failed_object(bucket, object, version_id, class, detail))
|
||||
})
|
||||
}
|
||||
|
||||
fn failure_class_label(class: HealFailureClass) -> &'static str {
|
||||
match class {
|
||||
HealFailureClass::Recoverable => "recoverable",
|
||||
HealFailureClass::RetryExhausted => "retry_exhausted",
|
||||
HealFailureClass::Permanent => "permanent",
|
||||
}
|
||||
}
|
||||
|
||||
fn format_first_failed_object(bucket: &str, object: &str, version_id: Option<&str>, class: &str, detail: Option<&str>) -> String {
|
||||
let mut message = format!("first failed object {bucket}/{object} ({class})");
|
||||
if let Some(version_id) = version_id.filter(|version_id| !version_id.is_empty()) {
|
||||
message.push_str(", version ");
|
||||
message.push_str(version_id);
|
||||
}
|
||||
if let Some(detail) = detail.filter(|detail| !detail.is_empty()) {
|
||||
message.push_str(": ");
|
||||
message.push_str(detail);
|
||||
}
|
||||
message
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -513,11 +513,6 @@ impl HealTask {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn set_execution_elapsed_for_test(&self, elapsed: Duration) {
|
||||
*self.task_start_instant.write().await = Some(Instant::now() - elapsed);
|
||||
}
|
||||
|
||||
pub(crate) async fn retry_request_with_remaining_timeout(&self) -> Result<HealRequest> {
|
||||
let mut request = self.retry_request();
|
||||
if self.options.timeout.is_some() {
|
||||
@@ -651,14 +646,6 @@ impl HealTask {
|
||||
self.batch_failure_recorded.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) async fn batch_failure_is_retryable(&self) -> bool {
|
||||
self.batch_failure
|
||||
.read()
|
||||
.await
|
||||
.as_ref()
|
||||
.is_some_and(|failure| failure.failed > 0 && failure.failed == failure.retryable && failure.permanent == 0)
|
||||
}
|
||||
|
||||
pub(crate) async fn record_batch_failure(&self, failure: BatchHealFailure) -> Error {
|
||||
self.batch_failure_recorded.store(true, Ordering::Release);
|
||||
let message = failure.to_string();
|
||||
|
||||
@@ -362,12 +362,7 @@ mod canonical_outcome {
|
||||
assert_eq!((progress.objects_scanned, progress.objects_healed, progress.objects_failed), (2, 1, 1));
|
||||
let (legacy_summary, legacy_detail) = outcome.legacy_status("finished", None);
|
||||
assert_eq!(legacy_summary, "stopped");
|
||||
assert_eq!(
|
||||
legacy_detail.as_deref(),
|
||||
Some(
|
||||
"heal traversal completed with errors: 1 failed objects; first failed object bucket-a/object-a (retry_exhausted): Storage error: Lock error: Lock acquisition timeout for resource 'object-a' after 5s"
|
||||
)
|
||||
);
|
||||
assert_eq!(legacy_detail.as_deref(), Some("heal traversal completed with errors: 1 failed objects"));
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("object calls").as_slice(),
|
||||
["object-a", "object-b", "object-a", "object-a", "object-a"]
|
||||
|
||||
@@ -61,14 +61,10 @@ pub fn create_ahm_services_cancel_token() -> CancellationToken {
|
||||
}
|
||||
|
||||
/// Shutdown all heal services gracefully
|
||||
pub async fn shutdown_ahm_services() -> Result<()> {
|
||||
if let Some(manager) = get_heal_manager() {
|
||||
manager.stop().await?;
|
||||
}
|
||||
pub fn shutdown_ahm_services() {
|
||||
if let Some(cancel_token) = GLOBAL_AHM_SERVICES_CANCEL_TOKEN.get() {
|
||||
cancel_token.cancel();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct HealRuntime {
|
||||
|
||||
@@ -577,16 +577,13 @@ impl KmsServiceManager {
|
||||
Some(service_version.probe_worker.as_ref()?.status())
|
||||
}
|
||||
|
||||
/// Check backend health without changing the service lifecycle state.
|
||||
///
|
||||
/// A transient backend failure leaves the published service available for
|
||||
/// subsequent checks and operations. Readiness uses the background probe
|
||||
/// to evaluate backend availability independently of lifecycle state.
|
||||
/// Health check for the KMS service
|
||||
pub async fn health_check(&self) -> Result<bool> {
|
||||
let checked_state = self.state.load_full();
|
||||
match checked_state.current_service.as_ref() {
|
||||
Some(service_version) => {
|
||||
let manager = service_version.manager.clone();
|
||||
let checked_version = service_version.version;
|
||||
// Perform health check on the backend
|
||||
match manager.health_check().await {
|
||||
Ok(healthy) => {
|
||||
@@ -597,6 +594,8 @@ impl KmsServiceManager {
|
||||
}
|
||||
Err(e) => {
|
||||
error!("KMS health check error: {}", e);
|
||||
let _guard = self.lifecycle_mutex.lock().await;
|
||||
self.mark_health_error_if_current(checked_version, &e);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
@@ -740,6 +739,17 @@ impl KmsServiceManager {
|
||||
task: std::sync::Mutex::new(Some(task)),
|
||||
}))
|
||||
}
|
||||
|
||||
fn mark_health_error_if_current(&self, checked_version: u64, error: &KmsError) {
|
||||
let current = self.state.load_full();
|
||||
if current.current_service.as_ref().map(|version| version.version) == Some(checked_version) {
|
||||
self.state.store(Arc::new(RuntimeState {
|
||||
config: current.config.clone(),
|
||||
status: KmsServiceStatus::Error(format!("Health check failed: {error}")),
|
||||
current_service: current.current_service.clone(),
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for KmsServiceManager {
|
||||
@@ -994,6 +1004,19 @@ mod tests {
|
||||
assert!(manager.get_service_version().await.expect("restarted version") > first_version);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_health_failure_cannot_poison_new_service_status() {
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(static_config("key-a", 0x11)).await.expect("configure");
|
||||
manager.start().await.expect("start");
|
||||
let old_version = manager.get_service_version().await.expect("old version");
|
||||
manager.restart().await.expect("restart");
|
||||
|
||||
manager.mark_health_error_if_current(old_version, &KmsError::backend_error("stale failure"));
|
||||
|
||||
assert_eq!(manager.get_status().await, KmsServiceStatus::Running);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forbidden_local_master_key_change_preserves_running_config_and_service() {
|
||||
use crate::types::{CreateKeyRequest, KeyUsage};
|
||||
|
||||
@@ -75,44 +75,6 @@ fn unreachable_vault_config() -> KmsConfig {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transient_health_failure_does_not_latch_the_service_status() {
|
||||
let kms = TestKms::local().await;
|
||||
let manager = kms.manager();
|
||||
let service = manager.get_encryption_service().await.expect("running service");
|
||||
let version = manager.get_service_version().await.expect("running version");
|
||||
assert!(manager.health_check().await.expect("initial backend health"));
|
||||
|
||||
// Move only this test's keys out of reach, then restore the same backend.
|
||||
let key_dir = kms.key_dir().expect("local key directory");
|
||||
let outage = tempfile::TempDir::new().expect("temporary outage directory");
|
||||
let hidden_keys = outage.path().join("keys");
|
||||
tokio::fs::rename(&key_dir, &hidden_keys)
|
||||
.await
|
||||
.expect("make backend unavailable");
|
||||
let failure = manager.health_check().await;
|
||||
let outage_status = manager.get_status().await;
|
||||
tokio::fs::rename(&hidden_keys, &key_dir).await.expect("restore backend");
|
||||
|
||||
assert!(failure.is_err(), "the outage must surface as a health-check error");
|
||||
assert!(manager.health_check().await.expect("backend recovers without restart"));
|
||||
assert!(Arc::ptr_eq(
|
||||
&service,
|
||||
&manager.get_encryption_service().await.expect("service survives the outage")
|
||||
));
|
||||
assert_eq!(manager.get_service_version().await, Some(version));
|
||||
assert_eq!(
|
||||
manager.get_status().await,
|
||||
KmsServiceStatus::Running,
|
||||
"a recovered backend must not leave service-status and readiness latched in Error"
|
||||
);
|
||||
assert_eq!(
|
||||
outage_status,
|
||||
KmsServiceStatus::Running,
|
||||
"backend health does not change the running service's lifecycle state"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn starting_against_an_unreachable_backend_fails_without_publishing_a_service() {
|
||||
let manager = KmsServiceManager::new();
|
||||
|
||||
+2
-2
@@ -98,7 +98,7 @@
|
||||
"cliExit": 1,
|
||||
"response": {
|
||||
"summary": "stopped",
|
||||
"detail": "heal traversal completed with errors: 1 failed objects; first failed object bucket/object (retry_exhausted); heal result items were truncated",
|
||||
"detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
@@ -274,7 +274,7 @@
|
||||
"cliExit": 1,
|
||||
"response": {
|
||||
"summary": "stopped",
|
||||
"detail": "heal traversal completed with errors: 1 failed objects; first failed object bucket/object (retry_exhausted); heal result items were truncated",
|
||||
"detail": "heal traversal completed with errors: 1 failed objects; heal result items were truncated",
|
||||
"startTime": "2026-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"recursive": true,
|
||||
|
||||
@@ -122,11 +122,7 @@ All configuration is read from environment variables at startup.
|
||||
| `RUSTFS_OBS_PROFILING_EXPORT_ENABLED` | `false` | Toggle profiling export |
|
||||
| `RUSTFS_OBS_USE_STDOUT` | `false` | Mirror all signals to stdout alongside OTLP |
|
||||
| `RUSTFS_OBS_SAMPLE_RATIO` | `0.1` | Trace sampling ratio `0.0`–`1.0` |
|
||||
| `RUSTFS_OBS_METER_INTERVAL` | `30` | Metrics export interval (seconds) |
|
||||
|
||||
The export interval is separate from application metric collection. Without interval overrides, node/disk metrics refresh every 60 seconds (`RUSTFS_METRICS_NODE_INTERVAL_SEC`), cluster metrics every 60 seconds (`RUSTFS_METRICS_CLUSTER_INTERVAL_SEC`), and per-bucket metrics every 300 seconds (`RUSTFS_METRICS_BUCKET_INTERVAL_SEC`). See the [collection defaults](src/metrics/config.rs) and [interval configuration](src/metrics/scheduler.rs) for collector-specific and global overrides.
|
||||
|
||||
With OTLP → Collector → Prometheus, the time from a failure to an alert also includes underlying health detection, collection and export work, transport and Collector batching, Prometheus `scrape_interval` and `evaluation_interval`, and any alert rule `for` duration. A shorter scrape interval cannot refresh a value that RustFS has not yet collected and exported. These periodic intervals do not guarantee a fixed detection or alert latency; see the [Prometheus alert rule documentation](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) for the role of `for`.
|
||||
| `RUSTFS_OBS_METER_INTERVAL` | `15` | Metrics export interval (seconds) |
|
||||
|
||||
### Service identity
|
||||
|
||||
|
||||
@@ -1,150 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! One observer's global drive inventory. Runtime counters belong to local drives only.
|
||||
|
||||
use crate::metrics::report::PrometheusMetric;
|
||||
use crate::metrics::schema::MetricType;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct ClusterDriveStats {
|
||||
pub server: String,
|
||||
pub drive: String,
|
||||
pub pool_index: String,
|
||||
pub set_index: String,
|
||||
pub drive_index: String,
|
||||
pub disk_id: String,
|
||||
pub runtime_state: String,
|
||||
pub offline_duration_seconds: Option<u64>,
|
||||
pub capacity_state: &'static str,
|
||||
pub capacity_age_seconds: u64,
|
||||
pub total_bytes: u64,
|
||||
pub used_bytes: u64,
|
||||
pub free_bytes: u64,
|
||||
}
|
||||
|
||||
pub(crate) fn collect_cluster_drive_metrics(stats: &[ClusterDriveStats]) -> Vec<PrometheusMetric> {
|
||||
let mut metrics = Vec::with_capacity(stats.len() * 13);
|
||||
for drive in stats {
|
||||
let metric = |name, help, value| {
|
||||
PrometheusMetric::new(name, MetricType::Gauge, help, value)
|
||||
.with_label_owned("server", drive.server.clone())
|
||||
.with_label_owned("drive", drive.drive.clone())
|
||||
.with_label_owned("pool_index", drive.pool_index.clone())
|
||||
.with_label_owned("set_index", drive.set_index.clone())
|
||||
.with_label_owned("drive_index", drive.drive_index.clone())
|
||||
.with_label_owned("disk_id", drive.disk_id.clone())
|
||||
};
|
||||
metrics.push(metric(
|
||||
"rustfs_cluster_drive_present",
|
||||
"Configured drive slot in this observer's inventory",
|
||||
1.0,
|
||||
));
|
||||
for state in ["online", "offline", "returning", "suspect", "unknown"] {
|
||||
let observed = match drive.runtime_state.as_str() {
|
||||
"online" | "offline" | "returning" | "suspect" => drive.runtime_state.as_str(),
|
||||
_ => "unknown",
|
||||
};
|
||||
metrics.push(
|
||||
metric(
|
||||
"rustfs_cluster_drive_runtime_state",
|
||||
"Observed drive runtime state (one active state)",
|
||||
f64::from(state == observed),
|
||||
)
|
||||
.with_label("state", state),
|
||||
);
|
||||
}
|
||||
if let Some(seconds) = drive.offline_duration_seconds {
|
||||
metrics.push(metric(
|
||||
"rustfs_cluster_drive_offline_duration_seconds",
|
||||
"Observed duration in seconds the drive has been offline",
|
||||
seconds as f64,
|
||||
));
|
||||
}
|
||||
for state in ["live", "stale", "missing"] {
|
||||
metrics.push(
|
||||
metric(
|
||||
"rustfs_cluster_drive_capacity_observation_state",
|
||||
"Provenance of the observed drive capacity",
|
||||
f64::from(state == drive.capacity_state),
|
||||
)
|
||||
.with_label("state", state),
|
||||
);
|
||||
}
|
||||
metrics.push(metric(
|
||||
"rustfs_cluster_drive_capacity_observation_age_seconds",
|
||||
"Age in seconds of the drive capacity observation at collection",
|
||||
drive.capacity_age_seconds as f64,
|
||||
));
|
||||
// A missing capacity observation is unknown, not a zero-capacity drive.
|
||||
if drive.capacity_state != "missing" {
|
||||
metrics.push(metric(
|
||||
"rustfs_cluster_drive_total_bytes",
|
||||
"Observed total drive capacity in bytes",
|
||||
drive.total_bytes as f64,
|
||||
));
|
||||
metrics.push(metric(
|
||||
"rustfs_cluster_drive_used_bytes",
|
||||
"Observed used drive capacity in bytes",
|
||||
drive.used_bytes as f64,
|
||||
));
|
||||
metrics.push(metric(
|
||||
"rustfs_cluster_drive_free_bytes",
|
||||
"Observed free drive capacity in bytes",
|
||||
drive.free_bytes as f64,
|
||||
));
|
||||
}
|
||||
}
|
||||
metrics
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn unknown_slots_remain_visible_without_inventing_capacity_or_counters() {
|
||||
let metrics = collect_cluster_drive_metrics(&[ClusterDriveStats {
|
||||
server: "unreachable:9000".into(),
|
||||
drive: "/data".into(),
|
||||
pool_index: "1".into(),
|
||||
set_index: "0".into(),
|
||||
drive_index: "3".into(),
|
||||
capacity_state: "missing",
|
||||
..Default::default()
|
||||
}]);
|
||||
assert!(
|
||||
metrics
|
||||
.iter()
|
||||
.any(|metric| metric.name == "rustfs_cluster_drive_present" && metric.value == 1.0)
|
||||
);
|
||||
assert!(
|
||||
metrics
|
||||
.iter()
|
||||
.any(|metric| metric.name == "rustfs_cluster_drive_runtime_state"
|
||||
&& metric.value == 1.0
|
||||
&& metric.labels.iter().any(|(key, value)| *key == "state" && value == "unknown"))
|
||||
);
|
||||
assert!(
|
||||
!metrics
|
||||
.iter()
|
||||
.any(|metric| metric.name.ends_with("_bytes") || metric.metric_type == MetricType::Counter)
|
||||
);
|
||||
assert!(
|
||||
metrics
|
||||
.iter()
|
||||
.all(|metric| metric.labels.iter().any(|(key, value)| *key == "pool_index" && value == "1"))
|
||||
);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user