mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-05 11:45:39 +00:00
test(scanner): add durable checkpoint diagnostics (#7175)
* chore(deps): refresh SDKs and pin clock skew regression coverage Refresh compatible dependencies for Scanner/Heal V2 batch 1 and verify the production S3 retry/signing path with a deterministic clock. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> * test(scanner): add durable checkpoint diagnostics Refs rustfs/backlog#2260 and rustfs/backlog#2240. Co-Authored-By: heihutu <heihutu@gmail.com> Co-Authored-By: zhi22915 <qiuzgang@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com> Co-authored-by: zhi22915 <qiuzgang@gmail.com>
This commit is contained in:
houseme
Generated
+104
-103
@@ -506,7 +506,7 @@ dependencies = [
|
||||
"arrow-select",
|
||||
"chrono",
|
||||
"half",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"itoa",
|
||||
"lexical-core",
|
||||
"memchr",
|
||||
@@ -809,7 +809,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -891,9 +891,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "aws-config"
|
||||
version = "1.11.0"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a767267da9e2c2e189b2f9df8b5657e850ecf5352644734ba130d4a57095cf1b"
|
||||
checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
@@ -958,9 +958,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-runtime"
|
||||
version = "1.9.1"
|
||||
version = "1.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c9007227e10b5fed2f3e0a2beff489211e2b5604c400b7a9d5d81ca9d64c24bb"
|
||||
checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-sigv4",
|
||||
@@ -986,9 +986,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-kms"
|
||||
version = "1.117.0"
|
||||
version = "1.118.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83b602641be84ebe5f96606cfefe4b96efaae1fd947c1b34ea8513b8ac0d2d8d"
|
||||
checksum = "c243864bc3754be9f0001414e0162fdf1370fa62c70b0cfbe1da6a6221d553c7"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1012,9 +1012,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.144.0"
|
||||
version = "1.145.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "30dc8bf6baaf7d46336a0ca2c69f223d9b90d7a801fb3e28f7ea17b00dc6b1de"
|
||||
checksum = "f0e6320417a37c8a62f78b443d0b4cf628b57cd340a09b0eb56173d47cc94e93"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1049,9 +1049,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sso"
|
||||
version = "1.108.0"
|
||||
version = "1.109.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c15301b04372832947916607983b114b3374b9db0be058a00fb7513800de1f05"
|
||||
checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1075,9 +1075,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-ssooidc"
|
||||
version = "1.110.0"
|
||||
version = "1.111.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72cc2c205cb27108183cf1856333f7d584c2ba0f505421b4209ca5828f9ea899"
|
||||
checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1101,9 +1101,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sts"
|
||||
version = "1.113.0"
|
||||
version = "1.114.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68182ecb449f7537db0f4d5d25917789cf41e32074a9fe47b6a0b847fe1d2032"
|
||||
checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
@@ -1235,7 +1235,7 @@ dependencies = [
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"hyper-util",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"pin-project-lite",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
@@ -1406,9 +1406,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-types"
|
||||
version = "1.5.0"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94"
|
||||
checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-smithy-async",
|
||||
@@ -1660,7 +1660,7 @@ version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1679,7 +1679,7 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1734,7 +1734,7 @@ dependencies = [
|
||||
"prettyplease 0.3.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1951,9 +1951,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.4.4"
|
||||
version = "1.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273"
|
||||
checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
@@ -2061,7 +2061,7 @@ version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.6",
|
||||
"crypto-common 0.1.7",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
@@ -2108,7 +2108,7 @@ dependencies = [
|
||||
"heck 0.5.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2531,7 +2531,7 @@ version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -2556,11 +2556,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.6"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
@@ -2759,7 +2759,7 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2803,7 +2803,7 @@ checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785"
|
||||
dependencies = [
|
||||
"darling_core 0.24.1",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2868,7 +2868,7 @@ dependencies = [
|
||||
"datafusion-session",
|
||||
"datafusion-sql",
|
||||
"futures",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"itertools 0.15.0",
|
||||
"log",
|
||||
"object_store",
|
||||
@@ -2943,7 +2943,7 @@ dependencies = [
|
||||
"foldhash 0.2.0",
|
||||
"half",
|
||||
"hashbrown 0.17.1",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"itertools 0.15.0",
|
||||
"libc",
|
||||
"log",
|
||||
@@ -3148,7 +3148,7 @@ dependencies = [
|
||||
"datafusion-functions-aggregate-common",
|
||||
"datafusion-functions-window-common",
|
||||
"datafusion-physical-expr-common",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"itertools 0.15.0",
|
||||
"recursive",
|
||||
"serde_json",
|
||||
@@ -3163,7 +3163,7 @@ checksum = "2604994999d5aeca1d1df645ffc98bc787447aaff05dde27aad0342b48fc1fe0"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"itertools 0.15.0",
|
||||
]
|
||||
|
||||
@@ -3304,7 +3304,7 @@ checksum = "15192effab05d38cce10e92a6fb48c967b5f166b27b7195a165a72b232569c58"
|
||||
dependencies = [
|
||||
"datafusion-doc",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3319,7 +3319,7 @@ dependencies = [
|
||||
"datafusion-expr",
|
||||
"datafusion-expr-common",
|
||||
"datafusion-physical-expr",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"itertools 0.15.0",
|
||||
"log",
|
||||
"recursive",
|
||||
@@ -3341,7 +3341,7 @@ dependencies = [
|
||||
"datafusion-physical-expr-common",
|
||||
"half",
|
||||
"hashbrown 0.17.1",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"itertools 0.15.0",
|
||||
"parking_lot",
|
||||
"petgraph 0.8.3",
|
||||
@@ -3375,7 +3375,7 @@ dependencies = [
|
||||
"datafusion-common",
|
||||
"datafusion-expr-common",
|
||||
"hashbrown 0.17.1",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"itertools 0.15.0",
|
||||
"parking_lot",
|
||||
"pin-project",
|
||||
@@ -3426,7 +3426,7 @@ dependencies = [
|
||||
"futures",
|
||||
"half",
|
||||
"hashbrown 0.17.1",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"itertools 0.15.0",
|
||||
"log",
|
||||
"num-traits",
|
||||
@@ -3479,7 +3479,7 @@ dependencies = [
|
||||
"datafusion-common",
|
||||
"datafusion-expr",
|
||||
"datafusion-functions-nested",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"log",
|
||||
"recursive",
|
||||
"regex",
|
||||
@@ -3852,7 +3852,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.6",
|
||||
"crypto-common 0.1.7",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -3929,7 +3929,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4117,7 +4117,7 @@ dependencies = [
|
||||
"crypto-bigint 0.5.5",
|
||||
"digest 0.10.7",
|
||||
"ff 0.13.1",
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"group 0.13.0",
|
||||
"hkdf 0.12.4",
|
||||
"pem-rfc7468 0.7.0",
|
||||
@@ -4341,9 +4341,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.11"
|
||||
version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890"
|
||||
checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d"
|
||||
|
||||
[[package]]
|
||||
name = "findshlibs"
|
||||
@@ -4517,7 +4517,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4562,9 +4562,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.9"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -4577,7 +4577,7 @@ version = "1.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"rustversion",
|
||||
"typenum",
|
||||
]
|
||||
@@ -4656,7 +4656,7 @@ checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c"
|
||||
dependencies = [
|
||||
"fnv",
|
||||
"hashbrown 0.16.1",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
@@ -4934,7 +4934,7 @@ dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"http 1.5.0",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
@@ -5583,9 +5583,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.14.1"
|
||||
version = "2.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb"
|
||||
checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.17.1",
|
||||
@@ -5600,7 +5600,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"block-padding 0.3.3",
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5847,9 +5847,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.104"
|
||||
version = "0.3.105"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a"
|
||||
checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
@@ -5885,7 +5885,7 @@ dependencies = [
|
||||
"crc",
|
||||
"crc32c",
|
||||
"flate2",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"lz4",
|
||||
"snap",
|
||||
"uuid",
|
||||
@@ -5918,7 +5918,7 @@ version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d"
|
||||
dependencies = [
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6398,7 +6398,7 @@ dependencies = [
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
"hashbrown 0.16.1",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"metrics",
|
||||
"ordered-float 5.5.0",
|
||||
"quanta",
|
||||
@@ -7027,7 +7027,7 @@ version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"base64 0.21.7",
|
||||
"chrono",
|
||||
"getrandom 0.2.17",
|
||||
"http 1.5.0",
|
||||
@@ -7703,7 +7703,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772"
|
||||
dependencies = [
|
||||
"fixedbitset",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7714,7 +7714,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455"
|
||||
dependencies = [
|
||||
"fixedbitset",
|
||||
"hashbrown 0.15.5",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"serde",
|
||||
]
|
||||
|
||||
@@ -7989,9 +7989,9 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic-util"
|
||||
version = "0.2.7"
|
||||
version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618"
|
||||
checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715"
|
||||
dependencies = [
|
||||
"portable-atomic",
|
||||
]
|
||||
@@ -8095,7 +8095,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8943,7 +8943,7 @@ checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9662,6 +9662,7 @@ dependencies = [
|
||||
"async-trait",
|
||||
"aws-credential-types",
|
||||
"aws-sdk-s3",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http-client",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
@@ -9956,7 +9957,7 @@ dependencies = [
|
||||
"bytes",
|
||||
"fnv",
|
||||
"hmac 0.13.0",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"kafka-protocol",
|
||||
"metrics",
|
||||
"pbkdf2 0.13.0",
|
||||
@@ -11293,7 +11294,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
||||
dependencies = [
|
||||
"base16ct 0.2.0",
|
||||
"der 0.7.10",
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"pkcs8 0.10.2",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -11405,7 +11406,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11424,7 +11425,7 @@ version = "1.0.151"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||
dependencies = [
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
@@ -11469,7 +11470,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11495,7 +11496,7 @@ dependencies = [
|
||||
"chrono",
|
||||
"hex",
|
||||
"indexmap 1.9.3",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"jiff",
|
||||
"schemars 0.9.0",
|
||||
"schemars 1.2.2",
|
||||
@@ -11549,7 +11550,7 @@ checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12146,9 +12147,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.4"
|
||||
version = "3.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f"
|
||||
checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -12271,7 +12272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -12367,7 +12368,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12513,7 +12514,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12558,9 +12559,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio-rustls"
|
||||
version = "0.26.4"
|
||||
version = "0.26.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
|
||||
checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67"
|
||||
dependencies = [
|
||||
"rustls",
|
||||
"tokio",
|
||||
@@ -12642,7 +12643,7 @@ version = "0.25.13+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b"
|
||||
dependencies = [
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"toml_datetime",
|
||||
"toml_parser",
|
||||
"winnow",
|
||||
@@ -12736,7 +12737,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"pin-project-lite",
|
||||
"slab",
|
||||
"sync_wrapper",
|
||||
@@ -13239,9 +13240,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.127"
|
||||
version = "0.2.128"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70"
|
||||
checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
@@ -13252,9 +13253,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-futures"
|
||||
version = "0.4.77"
|
||||
version = "0.4.78"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950"
|
||||
checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
@@ -13262,9 +13263,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.127"
|
||||
version = "0.2.128"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1"
|
||||
checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
@@ -13272,22 +13273,22 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.127"
|
||||
version = "0.2.128"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284"
|
||||
checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
"syn 3.0.5",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.127"
|
||||
version = "0.2.128"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf"
|
||||
checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
@@ -13307,9 +13308,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.104"
|
||||
version = "0.3.105"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30"
|
||||
checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
@@ -13856,7 +13857,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
"syn 3.0.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -13873,7 +13874,7 @@ dependencies = [
|
||||
"flate2",
|
||||
"getrandom 0.4.3",
|
||||
"hmac 0.13.0",
|
||||
"indexmap 2.14.1",
|
||||
"indexmap 2.14.2",
|
||||
"lzma-rust2",
|
||||
"memchr",
|
||||
"pbkdf2 0.13.0",
|
||||
@@ -13921,18 +13922,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zstd-safe"
|
||||
version = "7.2.4"
|
||||
version = "7.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
|
||||
checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882"
|
||||
dependencies = [
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-sys"
|
||||
version = "2.0.16+zstd.1.5.7"
|
||||
version = "2.1.0+zstd.1.5.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
|
||||
checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
|
||||
+6
-5
@@ -168,7 +168,7 @@ reqwest = "0.13.4"
|
||||
rustfs-kafka-async = { version = "1.3.1" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.53.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.4" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.5" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.5"
|
||||
tokio-util = { version = "0.7.19" }
|
||||
@@ -238,11 +238,12 @@ arc-swap = "1.9.2"
|
||||
astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
|
||||
atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.11.0" }
|
||||
aws-config = { version = "1.12.0" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.117.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.144.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.113.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.118.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.145.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.114.0" }
|
||||
aws-smithy-async = { version = "1.3.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
|
||||
aws-smithy-runtime-api = { version = "1.16.0" }
|
||||
aws-smithy-types = { version = "1.6.3" }
|
||||
|
||||
@@ -244,6 +244,7 @@ windows-sys = { workspace = true, features = [
|
||||
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
|
||||
|
||||
[dev-dependencies]
|
||||
aws-smithy-async.workspace = true
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
|
||||
@@ -652,9 +652,10 @@ async fn build_aws_s3_http_client_from_tls_path() -> Option<SharedHttpClient> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_smithy_async::time::TimeSource;
|
||||
use aws_smithy_runtime_api::http::StatusCode as SmithyStatusCode;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
|
||||
fn spec(endpoint: &str, secure: bool) -> RemoteS3EndpointSpec {
|
||||
RemoteS3EndpointSpec {
|
||||
@@ -824,6 +825,174 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct ClockSkewTimeSource(Arc<AtomicU64>);
|
||||
|
||||
impl TimeSource for ClockSkewTimeSource {
|
||||
fn now(&self) -> SystemTime {
|
||||
SystemTime::UNIX_EPOCH + Duration::from_secs(self.0.load(Ordering::SeqCst))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct ClockSkewConnector {
|
||||
request_headers: RecordedHeaders,
|
||||
error_code: &'static str,
|
||||
skew_seconds: i64,
|
||||
clock: ClockSkewTimeSource,
|
||||
}
|
||||
|
||||
fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> &'a str {
|
||||
headers
|
||||
.iter()
|
||||
.find(|(key, _)| key.eq_ignore_ascii_case(name))
|
||||
.map(|(_, value)| value.as_str())
|
||||
.unwrap_or_else(|| panic!("signed request must contain {name}"))
|
||||
}
|
||||
|
||||
fn signing_time(headers: &[(String, String)]) -> chrono::NaiveDateTime {
|
||||
chrono::NaiveDateTime::parse_from_str(recorded_header(headers, "x-amz-date"), "%Y%m%dT%H%M%SZ")
|
||||
.expect("SDK signing timestamp must use the SigV4 format")
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for ClockSkewConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
let mut headers = self.request_headers.lock().expect("clock skew request capture lock");
|
||||
assert!(headers.len() < 3, "clock skew fixture must not exceed two GET attempts and one HEAD");
|
||||
headers.push(
|
||||
request
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(key, value)| (key.to_string(), value.to_string()))
|
||||
.collect(),
|
||||
);
|
||||
let server_time = chrono::DateTime::<chrono::Utc>::from(self.clock.now()).naive_utc()
|
||||
+ chrono::Duration::seconds(self.skew_seconds);
|
||||
let (status, body) = if headers.len() == 1 {
|
||||
(
|
||||
403,
|
||||
format!("<Error><Code>{}</Code><Message>Clock skew fixture</Message></Error>", self.error_code),
|
||||
)
|
||||
} else {
|
||||
(200, String::new())
|
||||
};
|
||||
let response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("date", server_time.format("%a, %d %b %Y %H:%M:%S GMT").to_string())
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len())
|
||||
.body(SdkBody::from(body))
|
||||
.expect("clock skew fixture response");
|
||||
HttpConnectorFuture::ready(Ok(HttpResponse::try_from(response).expect("Smithy fixture response")))
|
||||
}
|
||||
}
|
||||
|
||||
async fn clock_skew_client(
|
||||
error_code: &'static str,
|
||||
skew_seconds: i64,
|
||||
retry: RemoteS3RetryPolicy,
|
||||
) -> (S3Client, RecordedHeaders, ClockSkewTimeSource) {
|
||||
let headers: RecordedHeaders = Arc::new(Mutex::new(Vec::new()));
|
||||
let clock = ClockSkewTimeSource(Arc::new(AtomicU64::new(1_700_000_000)));
|
||||
let connector = SharedHttpConnector::new(ClockSkewConnector {
|
||||
request_headers: Arc::clone(&headers),
|
||||
error_code,
|
||||
skew_seconds,
|
||||
clock: clock.clone(),
|
||||
});
|
||||
let mut spec = spec("s3.example.com", true);
|
||||
spec.retry = retry;
|
||||
let config = build_remote_s3_config(&spec)
|
||||
.await
|
||||
.expect("clock skew fixture uses the production outbound configuration")
|
||||
.http_client(http_client_fn(move |_settings, _components| connector.clone()))
|
||||
.time_source(clock.clone())
|
||||
.build();
|
||||
(S3Client::from_conf(config), headers, clock)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn remote_s3_clock_skew_retries_resign_and_seed_next_operation() {
|
||||
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
|
||||
for skew_seconds in [-600, 600] {
|
||||
let (client, headers, clock) = clock_skew_client(error_code, skew_seconds, REPLICATION_TARGET_RETRY_POLICY).await;
|
||||
let initial = chrono::DateTime::<chrono::Utc>::from(clock.now()).naive_utc();
|
||||
client
|
||||
.get_object()
|
||||
.bucket("bucket")
|
||||
.key("object")
|
||||
.send()
|
||||
.await
|
||||
.expect("clock skew GET must retry successfully");
|
||||
assert_eq!(
|
||||
headers.lock().expect("captured requests").len(),
|
||||
2,
|
||||
"{error_code}: GET needs exactly one retry"
|
||||
);
|
||||
clock.0.fetch_add(17, Ordering::SeqCst);
|
||||
// SDK signing time is independent of Tokio's retry/scheduler clock.
|
||||
tokio::time::advance(Duration::from_secs(61)).await;
|
||||
client
|
||||
.head_bucket()
|
||||
.bucket("bucket")
|
||||
.send()
|
||||
.await
|
||||
.expect("subsequent HEAD must use the client's cached skew");
|
||||
let headers = headers.lock().expect("captured signed requests");
|
||||
assert_eq!(headers.len(), 3, "subsequent operation must succeed on its first attempt");
|
||||
assert_eq!(signing_time(&headers[0]), initial, "the first attempt must use the injected clock");
|
||||
assert_eq!(
|
||||
signing_time(&headers[1]),
|
||||
initial + chrono::Duration::seconds(skew_seconds),
|
||||
"{error_code}: retry must apply the measured offset exactly"
|
||||
);
|
||||
assert_eq!(
|
||||
signing_time(&headers[2]),
|
||||
initial + chrono::Duration::seconds(skew_seconds + 17),
|
||||
"{error_code}: the next operation must apply cached skew to the advanced signing clock"
|
||||
);
|
||||
let signature = |index: usize| {
|
||||
recorded_header(&headers[index], "authorization")
|
||||
.rsplit_once("Signature=")
|
||||
.expect("SigV4 authorization contains a signature")
|
||||
.1
|
||||
};
|
||||
assert_ne!(
|
||||
signature(0),
|
||||
signature(1),
|
||||
"{error_code}: retry must be signed again after adjusting its date"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn remote_s3_clock_skew_respects_one_attempt_policy() {
|
||||
use aws_smithy_types::error::metadata::ProvideErrorMetadata;
|
||||
|
||||
for error_code in ["RequestTimeTooSkewed", "SignatureDoesNotMatch"] {
|
||||
for retry in [
|
||||
RemoteS3RetryPolicy::Disabled,
|
||||
RemoteS3RetryPolicy::Standard { max_attempts: 1 },
|
||||
] {
|
||||
let (client, headers, _clock) = clock_skew_client(error_code, 600, retry).await;
|
||||
let error = client
|
||||
.get_object()
|
||||
.bucket("bucket")
|
||||
.key("object")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("clock skew must not override the caller's one-attempt budget");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some(error_code));
|
||||
assert_eq!(
|
||||
headers.lock().expect("captured requests").len(),
|
||||
1,
|
||||
"{error_code}: {retry:?} must send exactly one request"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_style_auto_and_path_force_path_style() {
|
||||
assert!(PathStyle::Auto.force_path_style());
|
||||
|
||||
@@ -52,6 +52,9 @@ static REMOTE_SCANNER_CYCLE_REFRESH: LazyLock<AsyncMutex<()>> = LazyLock::new(||
|
||||
|
||||
mod stream;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use stream::checkpoint_fixture_partial_return;
|
||||
|
||||
pub use stream::{RemoteScannerAdmission, RemoteScannerRequest, serve_remote_scanner_request};
|
||||
pub(crate) use stream::{RemoteScannerOutcome, RemoteScannerScanSpec, scan_remote_bucket};
|
||||
use stream::{RemoteScannerReplayCache, RemoteScannerRequestWire, RemoteScannerValidatedCycle};
|
||||
|
||||
@@ -1017,6 +1017,48 @@ fn finish_remote_scanner_stream(
|
||||
#[cfg(test)]
|
||||
const TEST_NEXT_CYCLE: u64 = 11;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn checkpoint_fixture_partial_return(progress: (u64, u64), entries_visited: u64) {
|
||||
let request_id = Uuid::new_v4();
|
||||
let writer_auth = FrameAuthenticator::for_test(request_id);
|
||||
let reader_auth = FrameAuthenticator::for_test(request_id);
|
||||
let mut bytes = Vec::new();
|
||||
write_frame(
|
||||
&mut bytes,
|
||||
&writer_auth,
|
||||
&mut 0,
|
||||
&RemoteScannerFrame::terminal(
|
||||
RemoteScannerProgress {
|
||||
objects_scanned: progress.0,
|
||||
directories_started: progress.1,
|
||||
entries_visited,
|
||||
},
|
||||
RemoteScannerFrameResult::Partial,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("checkpoint partial frame must encode");
|
||||
let frame = read_frame(&mut std::io::Cursor::new(bytes.as_slice()), &reader_auth, &mut 0)
|
||||
.await
|
||||
.expect("checkpoint progress frame must authenticate");
|
||||
assert_eq!(frame.progress.entries_visited, entries_visited);
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default());
|
||||
let result = consume_remote_scanner_stream(
|
||||
std::io::Cursor::new(bytes),
|
||||
parent,
|
||||
budget.clone(),
|
||||
"bucket",
|
||||
DataUsageCacheSource::new(0, 0),
|
||||
DataUsageScanPlanDigest([17; 32]),
|
||||
reader_auth,
|
||||
)
|
||||
.await
|
||||
.expect("checkpoint partial frame must decode");
|
||||
assert!(matches!(result, RemoteScannerOutcome::Partial));
|
||||
assert_eq!(budget.progress(), progress);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn consume_remote_scanner_stream<R>(
|
||||
reader: R,
|
||||
|
||||
@@ -24,6 +24,8 @@ use std::io::Write;
|
||||
use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
use std::sync::Mutex;
|
||||
|
||||
mod checkpoint_fixture;
|
||||
|
||||
/// Reset the process-global alert cooldown map; test-only.
|
||||
fn reset_alert_cooldowns() {
|
||||
*SCANNER_ALERT_EMISSION_COOLDOWN
|
||||
|
||||
@@ -0,0 +1,410 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::scanner_budget::ScannerCycleBudgetConfig;
|
||||
use crate::scanner_io::{ScannerDiskScanOutcome, ScannerIODisk};
|
||||
use crate::storage_api::scanner_io::ObjectIO;
|
||||
use crate::{DataUsageCacheSource, DataUsageScanPlanDigest};
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const CACHE_NAME: &str = "bucket/checkpoint-fixture.bin";
|
||||
const STATIC_OBJECTS: u64 = 24;
|
||||
const MAX_CACHE_BYTES: u64 = 1024 * 1024;
|
||||
const SOURCE: DataUsageCacheSource = DataUsageCacheSource::new(0, 0);
|
||||
const PLAN: DataUsageScanPlanDigest = DataUsageScanPlanDigest([17; 32]);
|
||||
|
||||
/// Real cache persistence codec and CAS calls, backed by two bounded local files.
|
||||
#[derive(Debug)]
|
||||
struct FixtureStore {
|
||||
root: tempfile::TempDir,
|
||||
reject_save: AtomicBool,
|
||||
}
|
||||
|
||||
impl FixtureStore {
|
||||
fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
root: tempfile::tempdir().expect("checkpoint fixture storage directory"),
|
||||
reject_save: AtomicBool::new(false),
|
||||
})
|
||||
}
|
||||
|
||||
fn path(&self, object: &str) -> std::path::PathBuf {
|
||||
assert!(object.ends_with(CACHE_NAME) || object.ends_with(&format!("{CACHE_NAME}.bkp")));
|
||||
self.root
|
||||
.path()
|
||||
.join(if object.ends_with(".bkp") { "backup" } else { "main" })
|
||||
}
|
||||
|
||||
async fn strict_load(&self) -> DataUsageCache {
|
||||
let bytes = tokio::fs::read(self.root.path().join("main"))
|
||||
.await
|
||||
.expect("saved checkpoint fixture must exist");
|
||||
decode_fixture(&bytes).expect("saved checkpoint fixture must contain a valid bucket root")
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ObjectIO for FixtureStore {
|
||||
type Error = crate::EcstoreError;
|
||||
type RangeSpec = crate::storage_api::scanner_io::HTTPRangeSpec;
|
||||
type HeaderMap = http::HeaderMap;
|
||||
type ObjectOptions = crate::ScannerObjectOptions;
|
||||
type ObjectInfo = crate::ScannerObjectInfo;
|
||||
type GetObjectReader = crate::ScannerGetObjectReader;
|
||||
type PutObjectReader = crate::ScannerPutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
object: &str,
|
||||
_range: Option<Self::RangeSpec>,
|
||||
_headers: Self::HeaderMap,
|
||||
_options: &Self::ObjectOptions,
|
||||
) -> crate::EcstoreResult<Self::GetObjectReader> {
|
||||
let bytes = tokio::fs::read(self.path(object)).await.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
crate::EcstoreError::FileNotFound
|
||||
} else {
|
||||
crate::EcstoreError::from(error)
|
||||
}
|
||||
})?;
|
||||
assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES);
|
||||
Ok(crate::ScannerGetObjectReader {
|
||||
stream: Box::new(Cursor::new(bytes)),
|
||||
object_info: crate::ScannerObjectInfo {
|
||||
etag: Some("fixture".into()),
|
||||
..Default::default()
|
||||
},
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
object: &str,
|
||||
data: &mut Self::PutObjectReader,
|
||||
options: &Self::ObjectOptions,
|
||||
) -> crate::EcstoreResult<Self::ObjectInfo> {
|
||||
if self.reject_save.load(Ordering::SeqCst) {
|
||||
return Err(crate::EcstoreError::PreconditionFailed);
|
||||
}
|
||||
let path = self.path(object);
|
||||
let exists = tokio::fs::try_exists(&path).await?;
|
||||
let preconditions = options.http_preconditions.as_ref().expect("checkpoint writes must use CAS");
|
||||
if (exists && preconditions.if_none_match_value() == Some("*"))
|
||||
|| (!exists && preconditions.if_match_value().is_some())
|
||||
|| (exists && preconditions.if_match_value() != Some("fixture"))
|
||||
{
|
||||
return Err(crate::EcstoreError::PreconditionFailed);
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
(&mut data.stream).take(MAX_CACHE_BYTES + 1).read_to_end(&mut bytes).await?;
|
||||
assert!(u64::try_from(bytes.len()).expect("cache length") <= MAX_CACHE_BYTES);
|
||||
tokio::fs::write(path, bytes).await?;
|
||||
Ok(crate::ScannerObjectInfo {
|
||||
etag: Some("fixture".into()),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::ScannerConfigObjectDelete for FixtureStore {
|
||||
async fn delete_config_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_options: crate::ScannerObjectOptions,
|
||||
) -> crate::EcstoreResult<crate::ScannerObjectInfo> {
|
||||
Err(crate::EcstoreError::NotImplemented)
|
||||
}
|
||||
|
||||
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
|
||||
Some(crate::ScannerDataUsagePublicationAdmission::unfenced())
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_fixture(bytes: &[u8]) -> Result<DataUsageCache, &'static str> {
|
||||
if bytes.is_empty() || bytes.len() > usize::try_from(MAX_CACHE_BYTES).expect("fixture bound") {
|
||||
return Err("missing or oversized checkpoint fixture");
|
||||
}
|
||||
let cache = DataUsageCache::unmarshal(bytes).map_err(|_| "corrupt checkpoint fixture")?;
|
||||
if cache.info.name != "bucket" || cache.checked_flatten("bucket").is_none() {
|
||||
return Err("checkpoint fixture has no valid bucket root");
|
||||
}
|
||||
Ok(cache)
|
||||
}
|
||||
|
||||
fn retained(cache: &DataUsageCache) -> u64 {
|
||||
assert!(
|
||||
!cache.root().is_some_and(|root| root.compacted),
|
||||
"a compacted bucket root cannot prove static-prefix coverage"
|
||||
);
|
||||
cache
|
||||
.checked_flatten("bucket/static")
|
||||
.map_or(0, |entry| u64::try_from(entry.objects).expect("fixture object count fits u64"))
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum CoverageDiagnosis {
|
||||
Progress,
|
||||
NoNewWork,
|
||||
LostAtPrepare,
|
||||
LostAtReload,
|
||||
WalkWithoutRetention,
|
||||
}
|
||||
|
||||
fn diagnose(previous: u64, prepared: u64, walked: u64, scanned: u64, reloaded: u64) -> CoverageDiagnosis {
|
||||
if reloaded < scanned {
|
||||
CoverageDiagnosis::LostAtReload
|
||||
} else if prepared < previous {
|
||||
CoverageDiagnosis::LostAtPrepare
|
||||
} else if walked > 0 && reloaded <= previous {
|
||||
CoverageDiagnosis::WalkWithoutRetention
|
||||
} else if reloaded > previous {
|
||||
CoverageDiagnosis::Progress
|
||||
} else {
|
||||
CoverageDiagnosis::NoNewWork
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_fixture_diagnosis_rejects_walk_without_retention() {
|
||||
assert_eq!(diagnose(4, 4, 9, 8, 8), CoverageDiagnosis::Progress);
|
||||
assert_eq!(diagnose(4, 4, 9, 4, 4), CoverageDiagnosis::WalkWithoutRetention);
|
||||
assert_eq!(diagnose(4, 0, 9, 4, 4), CoverageDiagnosis::LostAtPrepare);
|
||||
assert_eq!(diagnose(4, 4, 9, 8, 4), CoverageDiagnosis::LostAtReload);
|
||||
assert_eq!(diagnose(4, 4, 0, 4, 4), CoverageDiagnosis::NoNewWork);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_fixture_missing_and_corrupt_inputs_fail() {
|
||||
for bytes in [
|
||||
vec![],
|
||||
vec![0xc1],
|
||||
DataUsageCache::default().marshal_msg().expect("empty cache encoding"),
|
||||
vec![0; usize::try_from(MAX_CACHE_BYTES + 1).expect("oversized fixture")],
|
||||
] {
|
||||
assert!(decode_fixture(&bytes).is_err(), "invalid fixture must not become an empty complete root");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_fixture_compaction_preserves_aggregate_not_child_enumeration() {
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.info.name = "bucket".to_string();
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
cache.replace("bucket/static", "bucket", DataUsageEntry::default());
|
||||
for index in 0..4 {
|
||||
cache.replace(
|
||||
&format!("bucket/static/{index}"),
|
||||
"bucket/static",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
cache.reduce_children_of(&hash_path("bucket/static"), 1, true);
|
||||
let decoded = decode_fixture(&cache.marshal_msg().expect("encode compacted cache")).expect("decode compacted fixture");
|
||||
let entry = decoded
|
||||
.find("bucket/static")
|
||||
.expect("compaction must retain the static subtree root");
|
||||
assert!(entry.compacted);
|
||||
assert!(entry.children.is_empty());
|
||||
assert_eq!(
|
||||
retained(&decoded),
|
||||
4,
|
||||
"compaction retains aggregate coverage even when leaf keys are absent"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn checkpoint_fixture_save_reload_resume() {
|
||||
run_checkpoint_fixture(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn checkpoint_fixture_hot_digest_diagnostic() {
|
||||
run_checkpoint_fixture(true).await;
|
||||
}
|
||||
|
||||
async fn run_checkpoint_fixture(change_digest: bool) {
|
||||
let (scanner, root) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
for index in 0..STATIC_OBJECTS {
|
||||
write_test_object_metadata(&root, "bucket", &format!("static/{index:04}")).await;
|
||||
}
|
||||
let store = FixtureStore::new();
|
||||
let mut previous = 0;
|
||||
let mut visited = 0;
|
||||
for round in 0..3_u8 {
|
||||
write_test_object_metadata(&root, "bucket", "hot/current").await;
|
||||
let mut cache = DataUsageCache::default();
|
||||
let revisions = cache
|
||||
.load_with_revisions(store.clone(), CACHE_NAME)
|
||||
.await
|
||||
.expect("load checkpoint revisions");
|
||||
if round > 0 {
|
||||
assert_eq!(retained(&store.strict_load().await), previous);
|
||||
}
|
||||
let plan = crate::scanner_io::checkpoint_fixture_bucket_digest(PLAN, change_digest.then_some(u64::from(round)));
|
||||
crate::scanner_io::current_cache_root_or_prepare_with_generation(
|
||||
&mut cache,
|
||||
"bucket",
|
||||
SOURCE,
|
||||
11,
|
||||
7,
|
||||
plan,
|
||||
crate::scanner_io::DataUsageCacheReuseOptions {
|
||||
require_source: true,
|
||||
tier_registry_generation: None,
|
||||
},
|
||||
);
|
||||
let prepared = retained(&cache);
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(4),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let outcome = scanner
|
||||
.local_disk
|
||||
.clone()
|
||||
.nsscanner_disk(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
vec![scanner.local_disk.clone()],
|
||||
cache,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
)
|
||||
.await
|
||||
.expect("budgeted local disk scan returns partial cache");
|
||||
let ScannerDiskScanOutcome::Partial(cache) = outcome else {
|
||||
panic!("budgeted fixture must remain partial")
|
||||
};
|
||||
assert!(!cache.info.snapshot_complete, "partial must never publish a complete root");
|
||||
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects));
|
||||
let scanned = retained(&cache);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist partial checkpoint");
|
||||
let mut loaded = DataUsageCache::default();
|
||||
loaded
|
||||
.load(store.clone(), CACHE_NAME)
|
||||
.await
|
||||
.expect("reload persisted partial checkpoint");
|
||||
let reloaded = retained(&loaded);
|
||||
assert_eq!(reloaded, retained(&store.strict_load().await));
|
||||
assert_eq!(scanned, reloaded, "save/load must retain static subtree coverage");
|
||||
assert!(!loaded.info.snapshot_complete);
|
||||
visited += budget.entries_visited();
|
||||
let diagnosis = diagnose(previous, prepared, budget.entries_visited(), scanned, reloaded);
|
||||
eprintln!(
|
||||
"checkpoint_fixture round={round} hot_digest={change_digest} visited_total={visited} before={previous} prepared={prepared} scanned={scanned} reloaded={reloaded} diagnosis={diagnosis:?}"
|
||||
);
|
||||
if !change_digest || std::env::var_os("RUSTFS_CHECKPOINT_REQUIRE_PROGRESS").is_some() {
|
||||
assert_eq!(
|
||||
diagnosis,
|
||||
CoverageDiagnosis::Progress,
|
||||
"visited growth must produce durable static coverage"
|
||||
);
|
||||
}
|
||||
crate::remote_scanner::checkpoint_fixture_partial_return(budget.progress(), budget.entries_visited()).await;
|
||||
previous = reloaded;
|
||||
}
|
||||
assert!(visited > 0, "fixture must exercise the directory walk");
|
||||
assert!(previous > 0, "fixture must retain and enumerate static subtree entries");
|
||||
|
||||
let mut loaded = DataUsageCache::default();
|
||||
let revisions = loaded
|
||||
.load_with_revisions(store.clone(), CACHE_NAME)
|
||||
.await
|
||||
.expect("load final checkpoint");
|
||||
let before = tokio::fs::read(store.root.path().join("main"))
|
||||
.await
|
||||
.expect("read durable checkpoint bytes");
|
||||
let epoch_error = loaded
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 1)
|
||||
.await
|
||||
.expect_err("stale publication epoch must reject persistence");
|
||||
assert!(epoch_error.to_string().contains(crate::SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
store.reject_save.store(true, Ordering::SeqCst);
|
||||
loaded.info.next_cycle += 1;
|
||||
loaded
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect_err("injected save failure must not report durable progress");
|
||||
assert_eq!(
|
||||
tokio::fs::read(store.root.path().join("main"))
|
||||
.await
|
||||
.expect("read unchanged checkpoint bytes"),
|
||||
before
|
||||
);
|
||||
|
||||
let parent = CancellationToken::new();
|
||||
parent.cancel();
|
||||
let budget = ScannerCycleBudget::new(&parent, Default::default());
|
||||
let result = scanner
|
||||
.local_disk
|
||||
.clone()
|
||||
.nsscanner_disk(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
vec![scanner.local_disk.clone()],
|
||||
loaded.clone(),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "pre-scan cancellation must not produce a complete root");
|
||||
assert_eq!(budget.reason(), None, "parent cancellation is not object budget exhaustion");
|
||||
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(&parent, Default::default());
|
||||
let result = scanner
|
||||
.local_disk
|
||||
.clone()
|
||||
.nsscanner_disk(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![scanner.local_disk.clone()],
|
||||
loaded,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
)
|
||||
.await
|
||||
.expect("unbounded scan must complete after durable partial progress");
|
||||
let ScannerDiskScanOutcome::Complete(cache) = result else {
|
||||
panic!("unbounded fixture must produce a complete disk cache");
|
||||
};
|
||||
assert!(cache.info.snapshot_complete);
|
||||
assert!(cache.info.scan_checkpoint.is_none());
|
||||
assert_eq!(
|
||||
cache.checked_flatten("bucket").expect("complete bucket root").objects,
|
||||
usize::try_from(STATIC_OBJECTS + 1).expect("fixture object count fits usize")
|
||||
);
|
||||
}
|
||||
@@ -209,6 +209,14 @@ fn scanner_bucket_cache_digest(
|
||||
DataUsageScanPlanDigest(hasher.finalize().into())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn checkpoint_fixture_bucket_digest(
|
||||
scan_plan_digest: DataUsageScanPlanDigest,
|
||||
dirty_generation: Option<u64>,
|
||||
) -> DataUsageScanPlanDigest {
|
||||
scanner_bucket_cache_digest(scan_plan_digest, dirty_generation)
|
||||
}
|
||||
|
||||
fn finalize_nsscanner_result(results: &[DataUsageCache], first_err: Option<Error>) -> Result<()> {
|
||||
if results.iter().any(|result| result.info.last_update.is_some()) {
|
||||
return Ok(());
|
||||
|
||||
@@ -1048,6 +1048,27 @@ fn scanner_cycle_status_requires_a_clean_complete_snapshot() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_fixture_superseded_is_distinct_from_partial_and_cancel() {
|
||||
for (budget, cancelled, bucket, expected) in [
|
||||
(false, false, ScannerBucketScanStatus::Complete, ScannerCycleStatus::Superseded),
|
||||
(true, false, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete),
|
||||
(false, true, ScannerBucketScanStatus::Partial, ScannerCycleStatus::Incomplete),
|
||||
] {
|
||||
assert_eq!(
|
||||
classify_nsscanner_cycle(
|
||||
true,
|
||||
budget,
|
||||
cancelled,
|
||||
bucket,
|
||||
DirtyUsageSnapshotStatus::Changed,
|
||||
ScannerCycleActivityStatus::Unchanged
|
||||
),
|
||||
expected,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unverified_activity_defers_partial_and_floor_cycles() {
|
||||
let expected = ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable);
|
||||
|
||||
@@ -21,6 +21,8 @@ Pick the lowest layer that can prove the change; add a higher-layer test only wh
|
||||
|
||||
Every script named above is indexed with status and wiring in [`scripts/README.md`](../../scripts/README.md). Fixed GHSA advisories map to named regression tests in [security-regressions.md](security-regressions.md).
|
||||
|
||||
The [scanner checkpoint fixture](scanner-checkpoint-fixture.md) diagnoses retained subtree coverage across budget interruption, persistence, reload, and plan invalidation.
|
||||
|
||||
## Naming conventions
|
||||
|
||||
### Reserved test-name substrings (migration gate)
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# Scanner Checkpoint Fixture
|
||||
|
||||
The `checkpoint_fixture` tests exercise a bounded namespace of 24 static objects and one repeatedly updated hot object. Each of three rounds runs the production local disk scanner with an object budget, saves the returned partial cache through the production persistence codec and revision checks to a two-file test backend, and reloads it before preparing the next round. The fixture prints static-subtree coverage at each boundary and cumulative visited entries. This is a diagnostic of retained coverage, not a throughput benchmark.
|
||||
|
||||
Run the fixture and confirm the test filter selects a nonzero number of tests:
|
||||
|
||||
```sh
|
||||
cargo test -p rustfs-scanner --lib checkpoint_fixture -- --list
|
||||
RUST_MIN_STACK=4194304 cargo test -p rustfs-scanner --lib checkpoint_fixture -- --nocapture
|
||||
```
|
||||
|
||||
The unchanged-plan case requires durable static coverage to increase each round. The hot-plan diagnostic changes the bucket plan digest between rounds and reports where coverage is lost without asserting that a particular defect must remain present. To require progress in this diagnostic as well:
|
||||
|
||||
```sh
|
||||
RUST_MIN_STACK=4194304 RUSTFS_CHECKPOINT_REQUIRE_PROGRESS=1 cargo test -p rustfs-scanner --lib checkpoint_fixture_hot_digest_diagnostic -- --nocapture
|
||||
```
|
||||
|
||||
A nonzero exit from the strict command means that walked work did not become additional retained static coverage. `LostAtPrepare` identifies invalidation before traversal; `LostAtReload` identifies loss between the returned cache and persisted data; `WalkWithoutRetention` identifies visited growth without durable coverage growth. Missing, corrupt, empty-root, and oversized checkpoint inputs are rejected by the strict fixture reader. Save failure and publication-epoch rejection must preserve the preceding file bytes. Parent cancellation is checked separately from object-budget exhaustion. Superseded classification is tested separately from either incomplete outcome.
|
||||
|
||||
For every saved partial cache, the fixture also passes its progress through the production authenticated remote terminal-frame writer and stream consumer. A remote partial result must remain partial even when its progress reports visited objects. This covers the return-frame contract; it does not execute the remote RPC server, distributed locks, EC quorum persistence, mixed-version peers, process crashes, or fsync durability. The file backend models revision preconditions and persistence errors, not a concurrent object store.
|
||||
|
||||
The synthetic namespace contains no customer data. Temporary files are removed with their owning fixture. Production scan semantics and persistent formats are unchanged, so rollback consists of removing these tests and this guide. A passing fixture alone does not establish that the field report in [issue #7108](https://github.com/rustfs/rustfs/issues/7108) has been independently reproduced or fixed. A field diagnosis must separately identify the source capture, cycle and leader identity, and decoded bucket/set caches.
|
||||
Reference in New Issue
Block a user