mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-27 08:38:58 +00:00
houseme
08e44b95f8
* fix(targets): make persistent queue store crash-safe and replay lifecycle correct Harden the target notification persistent queue (store.rs) and the replay worker lifecycle (runtime) against data loss, silent truncation, ordering drift, orphaned tasks, and a few low-risk robustness gaps. store.rs - Atomic, durable writes: write to a per-key temp file, fsync (sync_all), then rename into place; best-effort parent-dir fsync. A crash mid-write can no longer lose an acknowledged event or leave a half-written payload that reads as a valid entry. - open() now removes leftover .tmp residue and zero-byte files, and only indexes files matching the queue extension, so ghosts/foreign files are never replayed. - FIFO ordering is derived from time-ordered UUIDv7 entry names instead of coarse, clock-dependent file mtimes, so replay order is stable and identical after a restart. - Clamp HashMap/Vec pre-allocation derived from untrusted inputs (entry_limit, batch item_count) to avoid capacity-overflow panics / giant allocations. target/mod.rs - QueuedPayload::decode validates body length against the recorded payload_len, rejecting torn/truncated writes instead of delivering a silently truncated body. - send_from_store purges a NotFound/empty entry (index + file) instead of skipping it, so it cannot occupy a queue slot and be replayed forever. - sanitize_queue_dir_component appends a stable hash suffix when the id was lossy, so distinct target ids can no longer collapse onto the same queue directory; path-safe ids are unchanged (no migration). runtime - Replay backoff, idle waits, and inter-scan pauses are now cancel-aware, so reload/shutdown is not blocked for the full retry delay. - ReplayWorkerManager::stop_all signals cancellation and then awaits each worker's exit (bounded, with abort fallback), preventing orphaned tasks and overlapping drain of the same store. - Fix the always-true replay flush condition so batching is real (size/timeout based, one semaphore permit per batch) rather than one permit per entry; dedup keys already pending in the batch. - clear_and_close aggregates and reports per-target close failures instead of swallowing them; explicit shutdown surfaces them. Relates to rustfs/backlog#966 Relates to rustfs/backlog#967 Relates to rustfs/backlog#975 Relates to rustfs/backlog#970 Relates to rustfs/backlog#983 Co-Authored-By: heihutu <heihutu@gmail.com> * style(targets): apply rustfmt to replay batch dedup guard Fixes the Quick Checks rustfmt failure on the multi-line `.iter().any(...)` closure in the replay batch dedup guard. Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
66 lines
2.2 KiB
TOML
66 lines
2.2 KiB
TOML
[package]
|
|
name = "rustfs-targets"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
repository.workspace = true
|
|
rust-version.workspace = true
|
|
version.workspace = true
|
|
homepage.workspace = true
|
|
description = "Notification target abstraction and implementations for RustFS"
|
|
keywords = ["file-system", "notification", "target", "rustfs", "Minio"]
|
|
categories = ["web-programming", "development-tools", "filesystem"]
|
|
documentation = "https://docs.rs/rustfs-targets/latest/rustfs_targets/"
|
|
|
|
[dependencies]
|
|
rustfs-config = { workspace = true, features = ["notify", "constants", "audit", "server-config-model"] }
|
|
rustfs-extension-schema = { workspace = true }
|
|
rustfs-tls-runtime = { workspace = true }
|
|
rustfs-s3-types = { workspace = true }
|
|
rustfs-utils = { workspace = true, features = ["egress"] }
|
|
async-trait = { workspace = true }
|
|
async-nats = { workspace = true }
|
|
deadpool-postgres = { workspace = true }
|
|
hyper = { workspace = true }
|
|
hyper-rustls = { workspace = true }
|
|
lapin = { workspace = true }
|
|
libc = { workspace = true }
|
|
pulsar = { workspace = true }
|
|
regex = { workspace = true }
|
|
reqwest = { workspace = true }
|
|
rumqttc = { workspace = true }
|
|
redis = { workspace = true }
|
|
rustls = { workspace = true }
|
|
rustls-native-certs = { workspace = true }
|
|
s3s = { workspace = true }
|
|
serde = { workspace = true }
|
|
serde_json = { workspace = true }
|
|
snap = { workspace = true }
|
|
thiserror = { workspace = true }
|
|
tokio = { workspace = true, features = ["fs", "rt-multi-thread", "sync", "time"] }
|
|
tokio-postgres = { workspace = true }
|
|
tokio-postgres-rustls = { workspace = true }
|
|
tracing = { workspace = true }
|
|
url = { workspace = true }
|
|
urlencoding = { workspace = true }
|
|
uuid = { workspace = true, features = ["v4", "v7", "serde"] }
|
|
sysinfo = { workspace = true, features = ["multithread"] }
|
|
rustfs-kafka-async = { workspace = true }
|
|
mysql_async = { workspace = true }
|
|
chrono = { workspace = true }
|
|
parking_lot = { workspace = true }
|
|
hashbrown = { workspace = true }
|
|
arc-swap = { workspace = true }
|
|
metrics = { workspace = true }
|
|
|
|
[dev-dependencies]
|
|
criterion = { workspace = true }
|
|
tempfile = { workspace = true }
|
|
rcgen = { workspace = true }
|
|
|
|
[[bench]]
|
|
name = "queue_store_benchmark"
|
|
harness = false
|
|
|
|
[lints]
|
|
workspace = true
|