mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-07 13:53:12 +00:00
feat(table-catalog): add manifest reachability cleanup (#3484)
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
Henry Guo
Generated
+53
-2
@@ -275,6 +275,30 @@ version = "1.0.102"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
|
||||
|
||||
[[package]]
|
||||
name = "apache-avro"
|
||||
version = "0.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf"
|
||||
dependencies = [
|
||||
"bigdecimal",
|
||||
"bon",
|
||||
"digest 0.10.7",
|
||||
"log",
|
||||
"miniz_oxide",
|
||||
"num-bigint",
|
||||
"quad-rand",
|
||||
"rand 0.9.4",
|
||||
"regex-lite",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"serde_json",
|
||||
"strum 0.27.2",
|
||||
"strum_macros 0.27.2",
|
||||
"thiserror 2.0.18",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ar_archive_writer"
|
||||
version = "0.5.2"
|
||||
@@ -1441,6 +1465,7 @@ dependencies = [
|
||||
"num-bigint",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6640,6 +6665,7 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
|
||||
dependencies = [
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8172,6 +8198,12 @@ dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quad-rand"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40"
|
||||
|
||||
[[package]]
|
||||
name = "quanta"
|
||||
version = "0.12.6"
|
||||
@@ -9046,6 +9078,7 @@ version = "1.0.0-beta.8"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
"apache-avro",
|
||||
"astral-tokio-tar",
|
||||
"async-trait",
|
||||
"async_zip",
|
||||
@@ -9766,7 +9799,7 @@ dependencies = [
|
||||
"rustfs-crypto",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum",
|
||||
"strum 0.28.0",
|
||||
"temp-env",
|
||||
"test-case",
|
||||
"thiserror 2.0.18",
|
||||
@@ -11264,13 +11297,31 @@ version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "strum"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
|
||||
|
||||
[[package]]
|
||||
name = "strum"
|
||||
version = "0.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd"
|
||||
dependencies = [
|
||||
"strum_macros",
|
||||
"strum_macros 0.28.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strum_macros"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -160,6 +160,7 @@ tower = { version = "0.5.3", features = ["timeout"] }
|
||||
tower-http = { version = "0.6.11", features = ["cors"] }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = "0.21.0"
|
||||
bytes = { version = "1.11.1", features = ["serde"] }
|
||||
bytesize = "2.4.0"
|
||||
byteorder = "1.5.0"
|
||||
|
||||
@@ -123,6 +123,7 @@ tower.workspace = true
|
||||
tower-http = { workspace = true, features = ["trace", "compression-full", "cors", "catch-panic", "timeout", "limit", "request-id", "add-extension"] }
|
||||
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
flatbuffers.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
|
||||
+1295
-24
File diff suppressed because it is too large
Load Diff
@@ -150,10 +150,10 @@ current unsupported inventory is:
|
||||
|
||||
- credential vending: automated after table bootstrap with exact-prefix validation and a data-plane scope probe; full no-long-term-data-credential bootstrap is not claimed
|
||||
- background maintenance worker: controlled run-once and heartbeat endpoints are registered; continuous in-process scheduling is not claimed
|
||||
- manifest/data reachability cleanup: fail-closed reachability graph reporting only; metadata cleanup must not delete manifest, data, or delete files
|
||||
- manifest/data reachability cleanup: metadata maintenance reads manifest-list and manifest Avro references, reports manifest/data/delete reachability, and deletes only unreferenced table objects that pass the safety window
|
||||
- snapshot expiration dry-run planning and manual catalog commit: supported through metadata maintenance reports
|
||||
- automatic maintenance scheduling: external scheduler hook supported through the worker run endpoint; built-in periodic scheduling is not claimed
|
||||
- compaction rewrite: unsupported; planning reports fail closed until manifest Avro reading and rewrite support are implemented
|
||||
- compaction rewrite: unsupported; planning reports fail closed until data file rewrite support is implemented
|
||||
- Iceberg views: stable unsupported routes are registered and return explicit unsupported JSON
|
||||
- external catalog bridges: metadata import/register is supported, but Polaris/Glue/DLF/Hive synchronization is unsupported
|
||||
- multi-table transactions: not a short-term production claim
|
||||
|
||||
@@ -140,9 +140,9 @@ UNSUPPORTED_INVENTORY: list[dict[str, str]] = [
|
||||
},
|
||||
{
|
||||
"capability": "manifest-data-reachability-cleanup",
|
||||
"status": "fail-closed-reporting",
|
||||
"status": "conservative-cleanup-supported",
|
||||
"roadmap_area": "reachability-cleanup",
|
||||
"expected_behavior": "metadata maintenance reports expose reachability graph status and referenced manifest-list objects, but cleanup must not delete manifest, data, or delete files",
|
||||
"expected_behavior": "metadata maintenance reads manifest-list and manifest Avro references, reports retained manifest/data/delete objects, and deletes only unreferenced table objects that pass the safety window",
|
||||
},
|
||||
{
|
||||
"capability": "snapshot-expiration",
|
||||
@@ -154,7 +154,7 @@ UNSUPPORTED_INVENTORY: list[dict[str, str]] = [
|
||||
"capability": "compaction-rewrite",
|
||||
"status": "planning-only",
|
||||
"roadmap_area": "snapshot-maintenance",
|
||||
"expected_behavior": "compaction planning fails closed when manifest Avro reading is required; no data file rewrite or automatic compaction is claimed",
|
||||
"expected_behavior": "compaction planning fails closed until data file rewrite support is implemented; no data file rewrite or automatic compaction is claimed",
|
||||
},
|
||||
{
|
||||
"capability": "iceberg-views",
|
||||
|
||||
Reference in New Issue
Block a user