mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-20 19:42:17 +00:00
Compare commits
66 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 72f8e90e17 | |||
| bfd7deb269 | |||
| 63cbe45bd5 | |||
| bae3d2022f | |||
| 762919b1ba | |||
| cee0d5cf9b | |||
| 35af688cd9 | |||
| 205337151a | |||
| 105b6fbfde | |||
| b2e573c48b | |||
| 114bf5148c | |||
| 830e553a3c | |||
| d65fba9142 | |||
| 198a07d3fa | |||
| 8bd2d5d967 | |||
| efbef700ea | |||
| a247c79359 | |||
| cd399d1e72 | |||
| 095bf34086 | |||
| 2cf0ad0f85 | |||
| 2b9dcc646f | |||
| 129677f0b3 | |||
| c620a74230 | |||
| 319a03e638 | |||
| 621fcb93c7 | |||
| 76eb9c72e4 | |||
| 51023dc258 | |||
| 0d129ec4e7 | |||
| fec9e8980a | |||
| c428c6615e | |||
| 769e66511f | |||
| e6fc661162 | |||
| 305d291037 | |||
| 22b4ef9f0c | |||
| c10d74c78b | |||
| 898aa4db95 | |||
| b1b4e443b2 | |||
| d6efb65588 | |||
| 99c3811d93 | |||
| 3a46baab13 | |||
| 81332718e6 | |||
| 0126f359e3 | |||
| 10603d0870 | |||
| 7f8a8cdbac | |||
| cc0254d8de | |||
| 1f23fd17b6 | |||
| be7f684718 | |||
| 3bde70d5b4 | |||
| 5a1b0fe9df | |||
| 24cfce12ed | |||
| 9fed675185 | |||
| 1f8359537b | |||
| d7609b68a6 | |||
| 3958781320 | |||
| 5cb12300bc | |||
| bce0c05f3c | |||
| 07cef6789b | |||
| 05e6dc5f4a | |||
| 6f3f2f5f62 | |||
| b97fb02180 | |||
| f7073d0191 | |||
| d404e1bb8a | |||
| ceb6f779fb | |||
| e4eae22a70 | |||
| d030719dbc | |||
| 1741f79d7d |
@@ -53,9 +53,9 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install ripgrep
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ripgrep
|
||||
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
|
||||
with:
|
||||
tool: ripgrep@15.2.0
|
||||
|
||||
- name: Check architecture migration rules
|
||||
run: ./scripts/check_architecture_migration_rules.sh
|
||||
|
||||
@@ -83,7 +83,9 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install ripgrep
|
||||
run: sudo apt-get update && sudo apt-get install -y ripgrep
|
||||
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
|
||||
with:
|
||||
tool: ripgrep@15.2.0
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
|
||||
@@ -118,7 +118,9 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install ripgrep
|
||||
run: sudo apt-get update && sudo apt-get install -y ripgrep
|
||||
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
|
||||
with:
|
||||
tool: ripgrep@15.2.0
|
||||
|
||||
- name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
|
||||
Generated
+176
-144
@@ -1198,9 +1198,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-http-client"
|
||||
version = "1.3.0"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c1c8a04cb31ba74d0115af5a890bb8c0d48fba64b52812fa13929a6ef0cc83c"
|
||||
checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-protocol-test",
|
||||
@@ -1280,9 +1280,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime"
|
||||
version = "1.13.1"
|
||||
version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "483b858ff67522011c4786310c5cd8fd88d0be7ea3d5f1a48328446300c4269e"
|
||||
checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
@@ -1306,9 +1306,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.14.0"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb"
|
||||
checksum = "954c563ce84507722d2679f07a35d21b9c6466b3872d513020d0281fc8112ac9"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api-macros",
|
||||
@@ -1598,7 +1598,7 @@ version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1617,7 +1617,7 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1968,7 +1968,7 @@ version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.7",
|
||||
"crypto-common 0.1.6",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
@@ -2428,7 +2428,7 @@ version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -2453,11 +2453,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
@@ -2703,8 +2703,9 @@ checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06"
|
||||
|
||||
[[package]]
|
||||
name = "datafusion"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96f76f0167ed0842b29a3d1e41be3c034c0a46409a3a703cc4cc84ee8c24abf4"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-schema",
|
||||
@@ -2751,8 +2752,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-catalog"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d79ec3460f6ed5c58f9b3f2d873fbc77748b82653bff1b4cdaf06de33bb4e05f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2775,8 +2777,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-catalog-listing"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b48cef241e2efcfd496fe05ae4d0d5de20793451862faefe406c397a467e12d4"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2798,8 +2801,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-common"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f72810485975c258f1b4d00baab31728470676c60c5546f366ebd0d99f05ab6"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -2824,8 +2828,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-common-runtime"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "533c28e75dba52f41bde187d23a1cb24ab91c7c097966824fa471e67b60320ea"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"log",
|
||||
@@ -2834,8 +2839,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b00a1fa0da26f6087136a82fea7f13c76a672cbab452d4086952a7cf770a19b"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2863,8 +2869,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-arrow"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ad17ec881bff2ed7768b4bfe971d3efbf3473f2fd1f9d365447bccbdf908678"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -2886,8 +2893,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-csv"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5345285b0c3eaab412e7539b706973c083bd7e5bce575de5e0a3da488d08d1d"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2908,8 +2916,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-json"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da02fb9324f56bd8c53f1ee2e949547425cb66f76adc6832b10d44f80a1221d2"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2930,8 +2939,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-parquet"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c0b0dc1453952952fd5c69ad1c7f6042176e69ed233011d47e07cf74ed0949e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-schema",
|
||||
@@ -2961,13 +2971,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-doc"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a88fd985bc0550c36f557db69543cc9d6393b1509783520b30e902f23c555da6"
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-execution"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a98f1052f91b4991f0bf2ce1e4e36dfbdcda454a956b8c8d562c7c845e8fce1d"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-buffer",
|
||||
@@ -2991,8 +3003,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-expr"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "464625a1f0e4b9df552d894fafcc8aac953ebbc8b0fa0acdaf20975fd615040e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-schema",
|
||||
@@ -3013,8 +3026,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-expr-common"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2604994999d5aeca1d1df645ffc98bc787447aaff05dde27aad0342b48fc1fe0"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3024,8 +3038,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "051e97533e6af53e4aa0a0667cadc886abcaf36c4a5925019c55c0aa4c218fde"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-buffer",
|
||||
@@ -3051,8 +3066,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-aggregate"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d0f1bb166d3572b6ed40e1afb2faaacade962abc08c2fcf04babee74681c56b"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3071,8 +3087,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-aggregate-common"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ed756770f5f98369e181d692fd5ee6b1127ffd7322caba92f3730f9f5c92333"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3082,8 +3099,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-nested"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91173fdb5c0ff2a41169a8ffa1b385b8844f18728747bb0a37e35ad7d5772a4f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ord",
|
||||
@@ -3106,8 +3124,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-table"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1bcdfb286a745461b126719c32700777e83df4f17cc44db5d71ebce5731e840"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -3121,8 +3140,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-window"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ec4b508f1f93f00038ba3e737e894ec6c775528b4369413386655ae6125f0fc"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3137,8 +3157,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-window-common"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b352020834140073fbf5b46ee0ceb926e5074a9d0bcae1dbd91d0586d999cde"
|
||||
dependencies = [
|
||||
"datafusion-common",
|
||||
"datafusion-physical-expr-common",
|
||||
@@ -3146,8 +3167,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-macros"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15192effab05d38cce10e92a6fb48c967b5f166b27b7195a165a72b232569c58"
|
||||
dependencies = [
|
||||
"datafusion-doc",
|
||||
"quote",
|
||||
@@ -3156,8 +3178,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-optimizer"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "854445d9f7847e1e46089cf61b8d341a64382f14484e912c83a0f23b31216896"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"chrono",
|
||||
@@ -3175,8 +3198,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-expr"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "671558dad1d2aa253c39c0a4c52515958b99eb91abf649f4b88d5e69cc55282f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3196,8 +3220,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-expr-adapter"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ffae3d78c2da80ecc829cb58536cc5aca2e99cf1365eda694fc75bfe288861e0"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3210,8 +3235,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-expr-common"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d9092ed15e7203fbd0903215172f7c9d18f10d94cba35137f3b3836f7c46f16"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"chrono",
|
||||
@@ -3226,8 +3252,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-optimizer"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9005b6cf50b57b72d476c6ed4662b04be7ca6be5320ba9127c6d0b7e4218095b"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3245,8 +3272,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-plan"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5787e4fcff4adc4fce8948441103a99705018b49c8dff0720b650bd7a15da112"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-data",
|
||||
@@ -3279,8 +3307,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-pruning"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e651c8df0b90daed6a7be5921ec0ee379e6909705f063eeff70fd4e35010e4c"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -3294,8 +3323,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-session"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb56667ee38217efab19b895d9a936052cfb47ed438a19663351bdc42a6214a1"
|
||||
dependencies = [
|
||||
"arrow-schema",
|
||||
"async-trait",
|
||||
@@ -3308,8 +3338,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-sql"
|
||||
version = "54.1.0"
|
||||
source = "git+https://github.com/apache/datafusion.git?rev=e08aed1e5de41dcf81d529140dae07723b942a5e#e08aed1e5de41dcf81d529140dae07723b942a5e"
|
||||
version = "55.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c29067cb9d32f8e603c45e15d61ea18f1069f96ceafeceb4e18466b8e5b31d9"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"bigdecimal",
|
||||
@@ -3664,7 +3695,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.7",
|
||||
"crypto-common 0.1.6",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -3764,7 +3795,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "e2e_test"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"astral-tokio-tar",
|
||||
@@ -3924,7 +3955,7 @@ dependencies = [
|
||||
"crypto-bigint 0.5.5",
|
||||
"digest 0.10.7",
|
||||
"ff 0.13.1",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"group 0.13.0",
|
||||
"hkdf 0.12.4",
|
||||
"pem-rfc7468 0.7.0",
|
||||
@@ -4369,9 +4400,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -4384,7 +4415,7 @@ version = "1.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "337d46834ee672ab3e48caca2cb0c78cc174fb12b3a68d0d88f99a0519a5e36e"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"rustversion",
|
||||
"typenum",
|
||||
]
|
||||
@@ -4726,9 +4757,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.16"
|
||||
version = "0.4.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27"
|
||||
checksum = "839c0e8a181239723652be9062bb56ca5bf5f64011f73b623f6f4fc59086a228"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
@@ -5418,7 +5449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"block-padding 0.3.3",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8628,18 +8659,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast"
|
||||
version = "1.0.26"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d"
|
||||
checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3"
|
||||
dependencies = [
|
||||
"ref-cast-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast-impl"
|
||||
version = "1.0.26"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c"
|
||||
checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -9094,7 +9125,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -9142,6 +9173,7 @@ dependencies = [
|
||||
"mime_guess",
|
||||
"opentelemetry",
|
||||
"opentelemetry_sdk",
|
||||
"p256 0.13.2",
|
||||
"parking_lot",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
@@ -9232,7 +9264,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-audit"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"const-str",
|
||||
@@ -9255,7 +9287,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-checksums"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -9271,14 +9303,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-common"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"hotpath",
|
||||
"jiff",
|
||||
"metrics",
|
||||
"rmp-serde",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
@@ -9290,7 +9321,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-concurrency"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"insta",
|
||||
@@ -9303,7 +9334,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-config"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"hotpath",
|
||||
@@ -9313,7 +9344,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-credentials"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -9327,7 +9358,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-crypto"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
@@ -9348,7 +9379,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rmp-serde",
|
||||
@@ -9358,13 +9389,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-ecstore"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
"aws-config",
|
||||
"aws-credential-types",
|
||||
"aws-sdk-s3",
|
||||
"aws-smithy-http-client",
|
||||
@@ -9402,6 +9432,7 @@ dependencies = [
|
||||
"md-5 0.11.0",
|
||||
"memmap2",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"moka",
|
||||
"num_cpus",
|
||||
"opentelemetry",
|
||||
@@ -9498,7 +9529,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-extension-schema"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -9508,7 +9539,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-filemeta"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"byteorder",
|
||||
@@ -9535,7 +9566,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.23.1",
|
||||
@@ -9568,7 +9599,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-iam"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -9609,7 +9640,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-core"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hotpath",
|
||||
@@ -9621,7 +9652,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-metrics"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"hotpath",
|
||||
@@ -9685,7 +9716,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-keystone"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -9712,7 +9743,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-kms"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -9761,7 +9792,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lifecycle"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -9784,7 +9815,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lock"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"compact_str",
|
||||
@@ -9807,7 +9838,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-log-analyzer"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"flate2",
|
||||
@@ -9826,7 +9857,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-madmin"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
@@ -9846,7 +9877,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -9881,7 +9912,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-capacity"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"futures",
|
||||
@@ -9901,7 +9932,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-data-cache"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"criterion",
|
||||
@@ -9918,7 +9949,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-obs"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"crossbeam-channel",
|
||||
@@ -9933,6 +9964,7 @@ dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"num_cpus",
|
||||
"nvml-wrapper",
|
||||
"opentelemetry",
|
||||
@@ -9973,7 +10005,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-policy"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -10004,7 +10036,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protocols"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -10066,7 +10098,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protos"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"hotpath",
|
||||
@@ -10090,7 +10122,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-replication"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -10108,7 +10140,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -10146,7 +10178,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio-v2"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"bytes",
|
||||
@@ -10169,7 +10201,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-ops"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-s3-types",
|
||||
@@ -10177,7 +10209,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -10186,7 +10218,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-api"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10216,7 +10248,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-query"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
@@ -10235,7 +10267,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10276,7 +10308,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-security-governance"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"thiserror 2.0.20",
|
||||
@@ -10284,7 +10316,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-signer"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -10302,7 +10334,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-storage-api"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -10317,7 +10349,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-targets"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-nats",
|
||||
@@ -10371,7 +10403,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-test-utils"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-data-usage",
|
||||
@@ -10387,7 +10419,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-tls-runtime"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"hotpath",
|
||||
@@ -10408,7 +10440,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-trusted-proxies"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -10445,7 +10477,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-utils"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"blake2",
|
||||
@@ -10487,7 +10519,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-zip"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"hotpath",
|
||||
@@ -10677,8 +10709,8 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
|
||||
[[package]]
|
||||
name = "s3s"
|
||||
version = "0.14.1"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=d358a68783096df1db0c3e314127f2704603b29e#d358a68783096df1db0c3e314127f2704603b29e"
|
||||
version = "0.15.0-alpha.1"
|
||||
source = "git+https://github.com/rustfs/s3s.git?rev=ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a#ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrayvec",
|
||||
@@ -10832,7 +10864,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
||||
dependencies = [
|
||||
"base16ct 0.2.0",
|
||||
"der 0.7.10",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"pkcs8 0.10.2",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -11617,9 +11649,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "suppaftp"
|
||||
version = "10.0.1"
|
||||
version = "10.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c890e698eaf58526b6e7105d74c5d91ebe76a4da1faac2e20ff10e8e5c8bcff"
|
||||
checksum = "821001051ea3d12a60fb790b8c7cb9a6f5f8698dcfdca4cd533a025fefb0b5b8"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
@@ -11810,7 +11842,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -13360,9 +13392,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerovec"
|
||||
version = "0.11.7"
|
||||
version = "0.11.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8"
|
||||
checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8"
|
||||
dependencies = [
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
|
||||
+54
-53
@@ -69,7 +69,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.97.1"
|
||||
version = "1.0.0-rc.2"
|
||||
version = "1.0.0-rc.3"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -86,52 +86,52 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.2" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.2" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.2" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.2" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.2" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.2" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.2" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.2" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.2" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.2" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.2" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.2" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.2" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.2" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.2" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.2" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.2" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.2" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.2" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.2" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.2" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.2" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.2", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.2" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.2" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.2" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.2" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.2" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.2" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.2" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.2" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.2" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.2" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.2" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.2" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.2" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.2" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.2" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.2" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.2" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.2" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.2" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.2" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.2" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.2" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.2" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.3" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.3" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.3" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.3" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.3" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.3" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.3" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.3" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.3" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.3" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.3" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.3" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.3" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.3" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.3" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.3" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.3" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.3" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.3" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.3" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.3" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.3" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.3", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.3" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.3" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.3" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.3" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.3" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.3" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.3" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.3" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.3" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.3" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.3" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.3" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.3" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.3" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.3" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.3" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.3" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.3" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.3" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.3" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.3" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.3" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.3" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -231,8 +231,8 @@ aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.115.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.142.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.111.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.3.0" }
|
||||
aws-smithy-runtime-api = { version = "1.14.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.4.0" }
|
||||
aws-smithy-runtime-api = { version = "1.15.0" }
|
||||
aws-smithy-types = { version = "1.6.2" }
|
||||
base64 = "0.23.1"
|
||||
base64-simd = "0.8.0"
|
||||
@@ -245,8 +245,7 @@ crossbeam-queue = "0.3.13"
|
||||
crossbeam-channel = "0.5.16"
|
||||
crossbeam-deque = "0.8.7"
|
||||
crossbeam-utils = "0.8.22"
|
||||
datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "e08aed1e5de41dcf81d529140dae07723b942a5e" }
|
||||
#datafusion = { default-features = false, version = "54.1.0" }
|
||||
datafusion = { default-features = false, version = "55.0.0" }
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.14"
|
||||
faster-hex = "0.10.0"
|
||||
@@ -264,6 +263,7 @@ lazy_static = "1.5.0"
|
||||
libc = "0.2.189"
|
||||
libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.13"
|
||||
log = "0.4"
|
||||
memmap2 = "0.9.11"
|
||||
lz4 = "1.28.1"
|
||||
matchit = "0.9.2"
|
||||
@@ -290,7 +290,7 @@ rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "d358a68783096df1db0c3e314127f2704603b29e" }
|
||||
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "ed70cb048cc4be168419d461cb9ac3c2c7fa6d5a" }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
@@ -326,6 +326,7 @@ zstd = "0.13.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9-tokio-telemetry = "0.3"
|
||||
opentelemetry = { version = "0.32.0" }
|
||||
opentelemetry-appender-tracing = { version = "0.32.0" }
|
||||
@@ -339,7 +340,7 @@ pyroscope = { version = "2.1.1" }
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.1" }
|
||||
suppaftp = { version = "10.0.2" }
|
||||
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.7" }
|
||||
russh-sftp = "2.4.0"
|
||||
|
||||
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.3
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.2
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.3
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -236,12 +236,19 @@ async fn audit_pipeline_reports_empty_runtime_snapshots() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn audit_runtime_facade_stops_empty_replay_workers() {
|
||||
async fn stopping_audit_replay_workers_is_a_no_op_when_there_are_none() {
|
||||
let registry = Arc::new(Mutex::new(AuditRegistry::new()));
|
||||
let replay_workers = Arc::new(RwLock::new(rustfs_targets::ReplayWorkerManager::new()));
|
||||
let facade = AuditRuntimeFacade::new(registry, replay_workers);
|
||||
let facade = AuditRuntimeFacade::new(registry, Arc::clone(&replay_workers));
|
||||
|
||||
facade.stop_replay_workers().await;
|
||||
|
||||
// The stop path takes the manager's workers and hands them to the adapter,
|
||||
// so an empty facade must leave it empty rather than wedge it, and a second
|
||||
// call — which shutdown paths make — must stay harmless (rustfs/backlog#1836).
|
||||
assert!(replay_workers.read().await.is_empty());
|
||||
facade.stop_replay_workers().await;
|
||||
assert!(replay_workers.read().await.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -44,7 +44,6 @@ metrics = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
smallvec = { workspace = true }
|
||||
rmp-serde = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
tracing = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleRule, ReplicationConfiguration, ReplicationRuleStatus};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
fmt::{self, Display},
|
||||
@@ -633,104 +632,6 @@ pub fn create_heal_response(
|
||||
}
|
||||
}
|
||||
|
||||
fn lc_get_prefix(rule: &LifecycleRule) -> String {
|
||||
if let Some(p) = &rule.prefix {
|
||||
return p.to_string();
|
||||
} else if let Some(filter) = &rule.filter {
|
||||
if let Some(p) = &filter.prefix {
|
||||
return p.to_string();
|
||||
} else if let Some(and) = &filter.and
|
||||
&& let Some(p) = &and.prefix
|
||||
{
|
||||
return p.to_string();
|
||||
}
|
||||
}
|
||||
|
||||
"".into()
|
||||
}
|
||||
|
||||
pub fn lc_has_active_rules(config: &BucketLifecycleConfiguration, prefix: &str) -> bool {
|
||||
if config.rules.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
for rule in config.rules.iter() {
|
||||
if rule.status == ExpirationStatus::from_static(ExpirationStatus::DISABLED) {
|
||||
continue;
|
||||
}
|
||||
let rule_prefix = lc_get_prefix(rule);
|
||||
if !prefix.is_empty() && !rule_prefix.is_empty() && !prefix.starts_with(&rule_prefix) && !rule_prefix.starts_with(prefix)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(e) = &rule.noncurrent_version_expiration {
|
||||
if e.noncurrent_days.is_some() {
|
||||
return true;
|
||||
}
|
||||
if let Some(true) = e.newer_noncurrent_versions.map(|d| d > 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if rule.noncurrent_version_transitions.is_some() {
|
||||
return true;
|
||||
}
|
||||
if let Some(true) = rule.expiration.as_ref().map(|e| e.date.is_some()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if let Some(true) = rule.expiration.as_ref().map(|e| e.days.is_some()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if let Some(Some(true)) = rule.expiration.as_ref().map(|e| e.expired_object_delete_marker) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if let Some(true) = rule.transitions.as_ref().map(|t| !t.is_empty()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if rule.transitions.is_some() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub fn rep_has_active_rules(config: &ReplicationConfiguration, prefix: &str, recursive: bool) -> bool {
|
||||
if config.rules.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
for rule in config.rules.iter() {
|
||||
if rule
|
||||
.status
|
||||
.eq(&ReplicationRuleStatus::from_static(ReplicationRuleStatus::DISABLED))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if !prefix.is_empty()
|
||||
&& let Some(filter) = &rule.filter
|
||||
&& let Some(r_prefix) = &filter.prefix
|
||||
&& !r_prefix.is_empty()
|
||||
{
|
||||
// incoming prefix must be in rule prefix
|
||||
if !recursive && !prefix.starts_with(r_prefix) {
|
||||
continue;
|
||||
}
|
||||
// If recursive, we can skip this rule if it doesn't match the tested prefix or level below prefix
|
||||
// does not match
|
||||
if recursive && !r_prefix.starts_with(prefix) && !prefix.starts_with(r_prefix) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub async fn send_heal_disk(set_disk_id: String, priority: Option<HealChannelPriority>) -> Result<(), String> {
|
||||
let req = HealChannelRequest {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
|
||||
@@ -13,82 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug, Default)]
|
||||
struct TimedAction {
|
||||
count: u64,
|
||||
acc_time: u64,
|
||||
min_time: Option<u64>,
|
||||
max_time: Option<u64>,
|
||||
bytes: u64,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl TimedAction {
|
||||
// Avg returns the average time spent on the action.
|
||||
pub fn avg(&self) -> Option<Duration> {
|
||||
if self.count == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(Duration::from_nanos(self.acc_time / self.count))
|
||||
}
|
||||
|
||||
// AvgBytes returns the average bytes processed.
|
||||
pub fn avg_bytes(&self) -> u64 {
|
||||
if self.count == 0 {
|
||||
return 0;
|
||||
}
|
||||
self.bytes / self.count
|
||||
}
|
||||
|
||||
// Merge other into t.
|
||||
pub fn merge(&mut self, other: TimedAction) {
|
||||
self.count += other.count;
|
||||
self.acc_time += other.acc_time;
|
||||
self.bytes += other.bytes;
|
||||
|
||||
if self.count == 0 {
|
||||
self.min_time = other.min_time;
|
||||
}
|
||||
if let Some(other_min) = other.min_time {
|
||||
self.min_time = self.min_time.map_or(Some(other_min), |min| Some(min.min(other_min)));
|
||||
}
|
||||
|
||||
self.max_time = self
|
||||
.max_time
|
||||
.map_or(other.max_time, |max| Some(max.max(other.max_time.unwrap_or(0))));
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Debug)]
|
||||
enum SizeCategory {
|
||||
SizeLessThan1KiB = 0,
|
||||
SizeLessThan1MiB,
|
||||
SizeLessThan10MiB,
|
||||
SizeLessThan100MiB,
|
||||
SizeLessThan1GiB,
|
||||
SizeGreaterThan1GiB,
|
||||
// Add new entries here
|
||||
SizeLastElemMarker,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SizeCategory {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let s = match *self {
|
||||
SizeCategory::SizeLessThan1KiB => "SizeLessThan1KiB",
|
||||
SizeCategory::SizeLessThan1MiB => "SizeLessThan1MiB",
|
||||
SizeCategory::SizeLessThan10MiB => "SizeLessThan10MiB",
|
||||
SizeCategory::SizeLessThan100MiB => "SizeLessThan100MiB",
|
||||
SizeCategory::SizeLessThan1GiB => "SizeLessThan1GiB",
|
||||
SizeCategory::SizeGreaterThan1GiB => "SizeGreaterThan1GiB",
|
||||
SizeCategory::SizeLastElemMarker => "SizeLastElemMarker",
|
||||
};
|
||||
write!(f, "{s}")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Copy)]
|
||||
pub struct AccElem {
|
||||
pub total: u64,
|
||||
|
||||
+110
-25
@@ -729,7 +729,7 @@ fn timestamp_elapsed_seconds_since(now: Timestamp, earlier: Timestamp) -> u64 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
u64::try_from(duration.as_secs()).map_or(u64::MAX, |seconds| seconds)
|
||||
u64::try_from(duration.as_secs()).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
@@ -781,6 +781,19 @@ struct ScannerBucketDriveResultValue {
|
||||
last_seen: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
|
||||
struct ScannerActiveBucketDriveKey {
|
||||
source: String,
|
||||
bucket: String,
|
||||
drive: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct ScannerActiveBucketDriveValue {
|
||||
count: u64,
|
||||
started_at: Timestamp,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Metrics
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -813,6 +826,7 @@ pub struct Metrics {
|
||||
scanner_set_scans_active: AtomicU64,
|
||||
scanner_disk_bucket_scan_states: Mutex<HashMap<ScannerDiskBucketScanKey, ScannerDiskBucketScanState>>,
|
||||
scanner_bucket_drive_results: Mutex<ScannerBucketDriveResults>,
|
||||
scanner_active_bucket_drive_scans: Mutex<HashMap<ScannerActiveBucketDriveKey, ScannerActiveBucketDriveValue>>,
|
||||
scanner_bucket_drive_result_clock: AtomicU64,
|
||||
current_scan_cycle_bucket_drive_results_start: Mutex<HashMap<ScannerBucketDriveResultKey, u64>>,
|
||||
last_scan_cycle_bucket_drive_results: Mutex<Vec<ScannerBucketDriveResultSnapshot>>,
|
||||
@@ -1045,6 +1059,15 @@ pub struct ScannerBucketDriveResultSnapshot {
|
||||
pub count: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ScannerActiveBucketDriveSnapshot {
|
||||
pub source: String,
|
||||
pub bucket: String,
|
||||
pub drive: String,
|
||||
pub count: u64,
|
||||
pub age_seconds: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ScannerReplicationRepairSnapshot {
|
||||
pub source: String,
|
||||
@@ -1387,6 +1410,8 @@ pub struct ScannerRuntimeDetailsReport {
|
||||
pub current_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
|
||||
#[serde(default)]
|
||||
pub last_cycle_bucket_drive_results: Vec<ScannerBucketDriveResultSnapshot>,
|
||||
#[serde(default)]
|
||||
pub active_bucket_drive_scans: Vec<ScannerActiveBucketDriveSnapshot>,
|
||||
}
|
||||
|
||||
impl CurrentCycle {
|
||||
@@ -1401,25 +1426,11 @@ impl CurrentCycle {
|
||||
}
|
||||
|
||||
/// OTEL metric name constants for scanner metrics
|
||||
const OTEL_SCANNER_OBJECTS_SCANNED: &str = "rustfs_scanner_objects_scanned_total";
|
||||
const OTEL_SCANNER_DIRECTORIES_SCANNED: &str = "rustfs_scanner_directories_scanned_total";
|
||||
const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total";
|
||||
const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
|
||||
const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
|
||||
const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
|
||||
|
||||
fn emit_otel_counter(metric: usize, count: u64) {
|
||||
match Metric::from_index(metric) {
|
||||
Some(Metric::ScanObject) => {
|
||||
metrics::counter!(OTEL_SCANNER_OBJECTS_SCANNED).increment(count);
|
||||
}
|
||||
Some(Metric::ScanFolder) => {
|
||||
metrics::counter!(OTEL_SCANNER_DIRECTORIES_SCANNED).increment(count);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_cycle_result_label(result: u8) -> &'static str {
|
||||
match result {
|
||||
SCAN_CYCLE_RESULT_SUCCESS => SCAN_CYCLE_RESULT_SUCCESS_LABEL,
|
||||
@@ -1760,7 +1771,7 @@ pub fn emit_scan_cycle_deferred(duration: Duration) {
|
||||
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_DEFERRED_LABEL).increment(1);
|
||||
}
|
||||
|
||||
pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str, duration: Duration) {
|
||||
pub fn emit_scan_bucket_drive_complete(_source: ScannerWorkSource, success: bool, bucket: &str, disk: &str, duration: Duration) {
|
||||
let result = if success { "success" } else { "error" };
|
||||
global_metrics().record_scanner_bucket_drive_result(bucket, disk, result);
|
||||
metrics::counter!(
|
||||
@@ -1778,7 +1789,7 @@ pub fn emit_scan_bucket_drive_complete(success: bool, bucket: &str, disk: &str,
|
||||
.record(duration.as_secs_f64());
|
||||
}
|
||||
|
||||
pub fn emit_scan_bucket_drive_partial(bucket: &str, disk: &str, duration: Duration) {
|
||||
pub fn emit_scan_bucket_drive_partial(_source: ScannerWorkSource, bucket: &str, disk: &str, duration: Duration) {
|
||||
global_metrics().record_scanner_bucket_drive_result(bucket, disk, SCAN_CYCLE_RESULT_PARTIAL_LABEL);
|
||||
metrics::counter!(
|
||||
OTEL_SCANNER_BUCKETS_SCANNED,
|
||||
@@ -1831,6 +1842,7 @@ impl Metrics {
|
||||
scanner_set_scans_active: AtomicU64::new(0),
|
||||
scanner_disk_bucket_scan_states: Mutex::new(HashMap::new()),
|
||||
scanner_bucket_drive_results: Mutex::new(ScannerBucketDriveResults::default()),
|
||||
scanner_active_bucket_drive_scans: Mutex::new(HashMap::new()),
|
||||
scanner_bucket_drive_result_clock: AtomicU64::new(0),
|
||||
current_scan_cycle_bucket_drive_results_start: Mutex::new(HashMap::new()),
|
||||
last_scan_cycle_bucket_drive_results: Mutex::new(Vec::new()),
|
||||
@@ -1960,7 +1972,6 @@ impl Metrics {
|
||||
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
||||
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
||||
global_metrics().record_source_work_for_metric(metric, 1);
|
||||
emit_otel_counter(metric_idx, 1);
|
||||
if metric_idx < Metric::LastRealtime as usize {
|
||||
global_metrics().latency[metric_idx].add(duration);
|
||||
}
|
||||
@@ -1976,7 +1987,6 @@ impl Metrics {
|
||||
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
||||
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
||||
global_metrics().record_source_work_for_metric(metric, 1);
|
||||
emit_otel_counter(metric_idx, 1);
|
||||
if metric_idx < Metric::LastRealtime as usize {
|
||||
global_metrics().latency[metric_idx].add_size(duration, size);
|
||||
}
|
||||
@@ -1992,7 +2002,6 @@ impl Metrics {
|
||||
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
||||
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
||||
global_metrics().record_source_work_for_metric(metric, 1);
|
||||
emit_otel_counter(metric_idx, 1);
|
||||
if metric_idx < Metric::LastRealtime as usize {
|
||||
global_metrics().latency[metric_idx].add(duration);
|
||||
}
|
||||
@@ -2010,7 +2019,6 @@ impl Metrics {
|
||||
let count = usize_to_u64_saturated(count);
|
||||
global_metrics().operations[metric_idx].fetch_add(count, Ordering::Relaxed);
|
||||
global_metrics().record_source_work_for_metric(metric, count);
|
||||
emit_otel_counter(metric_idx, count);
|
||||
if metric_idx < Metric::LastRealtime as usize {
|
||||
global_metrics().latency[metric_idx].add(duration);
|
||||
}
|
||||
@@ -2031,7 +2039,6 @@ impl Metrics {
|
||||
let duration = SystemTime::now().duration_since(start).unwrap_or_default();
|
||||
let metric_idx = Metric::Ilm as usize;
|
||||
global_metrics().operations[metric_idx].fetch_add(versions, Ordering::Relaxed);
|
||||
emit_otel_counter(metric_idx, versions);
|
||||
global_metrics().actions[a_idx].fetch_add(versions, Ordering::Relaxed);
|
||||
global_metrics().actions_latency[a_idx].add(duration);
|
||||
})
|
||||
@@ -2044,7 +2051,6 @@ impl Metrics {
|
||||
let metric_idx = metric as usize;
|
||||
global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed);
|
||||
global_metrics().record_source_work_for_metric(metric, 1);
|
||||
emit_otel_counter(metric_idx, 1);
|
||||
if metric_idx < Metric::LastRealtime as usize {
|
||||
global_metrics().latency[metric_idx].add(duration);
|
||||
}
|
||||
@@ -2328,8 +2334,45 @@ impl Metrics {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_scan_bucket_drive_start(&self) {
|
||||
pub fn record_scan_bucket_drive_start(&self, source: ScannerWorkSource, bucket: &str, drive: &str) {
|
||||
self.operations[Metric::ScanBucketDriveStart as usize].fetch_add(1, Ordering::Relaxed);
|
||||
if bucket.is_empty() || drive.is_empty() {
|
||||
return;
|
||||
}
|
||||
let key = ScannerActiveBucketDriveKey {
|
||||
source: source.as_str().to_string(),
|
||||
bucket: bucket.to_string(),
|
||||
drive: drive.to_string(),
|
||||
};
|
||||
let mut active = self
|
||||
.scanner_active_bucket_drive_scans
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
active
|
||||
.entry(key)
|
||||
.and_modify(|value| value.count = value.count.saturating_add(1))
|
||||
.or_insert(ScannerActiveBucketDriveValue {
|
||||
count: 1,
|
||||
started_at: Timestamp::now(),
|
||||
});
|
||||
}
|
||||
|
||||
pub fn record_scan_bucket_drive_end(&self, source: ScannerWorkSource, bucket: &str, drive: &str) {
|
||||
let key = ScannerActiveBucketDriveKey {
|
||||
source: source.as_str().to_string(),
|
||||
bucket: bucket.to_string(),
|
||||
drive: drive.to_string(),
|
||||
};
|
||||
let mut active = self
|
||||
.scanner_active_bucket_drive_scans
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if let Some(value) = active.get_mut(&key) {
|
||||
value.count = value.count.saturating_sub(1);
|
||||
if value.count == 0 {
|
||||
active.remove(&key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_scan_bucket_drive_failure(&self) {
|
||||
@@ -2802,6 +2845,26 @@ impl Metrics {
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let now = Timestamp::now();
|
||||
let mut active_bucket_drive_scans = self
|
||||
.scanner_active_bucket_drive_scans
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.iter()
|
||||
.map(|(key, value)| ScannerActiveBucketDriveSnapshot {
|
||||
source: key.source.clone(),
|
||||
bucket: key.bucket.clone(),
|
||||
drive: key.drive.clone(),
|
||||
count: value.count,
|
||||
age_seconds: timestamp_elapsed_seconds_since(now, value.started_at),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
active_bucket_drive_scans.sort_by(|left, right| {
|
||||
left.source
|
||||
.cmp(&right.source)
|
||||
.then_with(|| left.bucket.cmp(&right.bucket))
|
||||
.then_with(|| left.drive.cmp(&right.drive))
|
||||
});
|
||||
ScannerRuntimeDetailsReport {
|
||||
disk_bucket_scan_states: self.scanner_disk_bucket_scan_state_snapshots(),
|
||||
bucket_drive_results: self.scanner_bucket_drive_result_counter_snapshots(),
|
||||
@@ -2811,6 +2874,7 @@ impl Metrics {
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.clone(),
|
||||
active_bucket_drive_scans,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4391,7 +4455,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn report_includes_bucket_drive_scan_starts() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scan_bucket_drive_start();
|
||||
metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
||||
metrics.record_scan_bucket_drive_failure();
|
||||
|
||||
let report = metrics.report().await;
|
||||
@@ -4400,6 +4464,27 @@ mod tests {
|
||||
assert_eq!(report.life_time_ops.get("scan_bucket_drive_failure"), Some(&1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn active_bucket_drive_snapshot_is_structured_and_retired_on_end() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
||||
metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
||||
let active = metrics.scanner_runtime_details_report().active_bucket_drive_scans;
|
||||
assert_eq!(active.len(), 1);
|
||||
assert_eq!(active[0].source, ScannerWorkSource::Usage.as_str());
|
||||
assert_eq!(active[0].bucket, "bucket-a");
|
||||
assert_eq!(active[0].drive, "/mnt/data/1");
|
||||
assert_eq!(active[0].count, 2);
|
||||
|
||||
metrics.record_scan_bucket_drive_end(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
||||
assert_eq!(metrics.scanner_runtime_details_report().active_bucket_drive_scans[0].count, 1);
|
||||
metrics.record_scan_bucket_drive_end(ScannerWorkSource::Usage, "bucket-a", "/mnt/data/1");
|
||||
assert!(metrics.scanner_runtime_details_report().active_bucket_drive_scans.is_empty());
|
||||
|
||||
metrics.record_scan_bucket_drive_start(ScannerWorkSource::Usage, "", "/mnt/data/1");
|
||||
assert!(metrics.scanner_runtime_details_report().active_bucket_drive_scans.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_includes_structured_bucket_drive_results() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
@@ -148,6 +148,62 @@ fn unix_now_ms() -> u64 {
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// A repair the MRF consumer landed, fanned out so retry ledgers can drop
|
||||
/// entries the journal no longer tracks (backlog#1894 axis B). The payload
|
||||
/// mirrors the intent identity so consumers match without re-parsing.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct MrfRepairedEvent {
|
||||
pub bucket: Arc<str>,
|
||||
pub object: Arc<str>,
|
||||
pub version_id: Option<[u8; 16]>,
|
||||
}
|
||||
|
||||
/// Bound on the repaired-event backlog. Notices are best-effort hints; when
|
||||
/// the ring is full the oldest are dropped and the affected ledger entries
|
||||
/// simply expire through their own attempts/age limits.
|
||||
const MRF_REPAIRED_EVENT_CAP: usize = 4096;
|
||||
|
||||
static MRF_REPAIRED_EVENTS: OnceLock<std::sync::Mutex<std::collections::VecDeque<MrfRepairedEvent>>> = OnceLock::new();
|
||||
|
||||
/// Record that the MRF consumer landed a repair. Never blocks: the critical
|
||||
/// section is a deque push under a std mutex.
|
||||
pub fn note_mrf_repaired(bucket: &str, object: &str, version_id: Option<[u8; 16]>) {
|
||||
let registry = MRF_REPAIRED_EVENTS.get_or_init(|| std::sync::Mutex::new(std::collections::VecDeque::new()));
|
||||
let Ok(mut events) = registry.lock() else {
|
||||
return;
|
||||
};
|
||||
if events.len() >= MRF_REPAIRED_EVENT_CAP {
|
||||
events.pop_front();
|
||||
}
|
||||
events.push_back(MrfRepairedEvent {
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object),
|
||||
version_id,
|
||||
});
|
||||
}
|
||||
|
||||
/// Take the repair notices recorded for `bucket`, leaving other buckets'
|
||||
/// notices in place for their own scanners.
|
||||
pub fn take_mrf_repaired_events_for(bucket: &str) -> Vec<MrfRepairedEvent> {
|
||||
let Some(registry) = MRF_REPAIRED_EVENTS.get() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Ok(mut events) = registry.lock() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut taken = Vec::new();
|
||||
let mut retained = std::collections::VecDeque::with_capacity(events.len());
|
||||
while let Some(event) = events.pop_front() {
|
||||
if event.bucket.as_ref() == bucket {
|
||||
taken.push(event);
|
||||
} else {
|
||||
retained.push_back(event);
|
||||
}
|
||||
}
|
||||
*events = retained;
|
||||
taken
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -200,4 +256,32 @@ mod tests {
|
||||
assert!(!try_send_mrf_intent(MrfKind::MetadataCorruption, "b", "o", None));
|
||||
set_mrf_delivery_enabled(true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repaired_events_take_is_bucket_scoped_and_cap_bounded() {
|
||||
// Distinct buckets keep their notices until their own scanner takes
|
||||
// them; a take for one bucket leaves the others' notices in place.
|
||||
note_mrf_repaired("bucket-a", "object-1", None);
|
||||
note_mrf_repaired("bucket-b", "object-2", None);
|
||||
note_mrf_repaired("bucket-a", "object-3", None);
|
||||
|
||||
let taken_a = take_mrf_repaired_events_for("bucket-a");
|
||||
assert_eq!(taken_a.len(), 2);
|
||||
assert_eq!(taken_a[0].object.as_ref(), "object-1");
|
||||
assert_eq!(taken_a[1].object.as_ref(), "object-3");
|
||||
assert!(take_mrf_repaired_events_for("bucket-a").is_empty(), "take is destructive per bucket");
|
||||
|
||||
let taken_b = take_mrf_repaired_events_for("bucket-b");
|
||||
assert_eq!(taken_b.len(), 1);
|
||||
assert_eq!(taken_b[0].object.as_ref(), "object-2");
|
||||
|
||||
// Cap bound: flooding the ring drops the oldest notices rather than
|
||||
// growing unbounded.
|
||||
for i in 0..=(MRF_REPAIRED_EVENT_CAP + 8) {
|
||||
note_mrf_repaired("flood-bucket", &format!("object-{i}"), None);
|
||||
}
|
||||
let flooded = take_mrf_repaired_events_for("flood-bucket");
|
||||
assert_eq!(flooded.len(), MRF_REPAIRED_EVENT_CAP);
|
||||
assert_eq!(flooded[0].object.as_ref(), "object-9", "the oldest notices past the cap are dropped");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,6 +115,15 @@ Current guidance:
|
||||
- enables KMS readiness enforcement for `/health/ready`.
|
||||
- default is `false`.
|
||||
|
||||
## Object lock admission environment variables
|
||||
|
||||
- `RUSTFS_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS`
|
||||
- experimental same-object PUT commit namespace-lock admission budget.
|
||||
- default is `0`, which disables this override and keeps `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT` behavior.
|
||||
- when set, only `put_object_commit` write-lock acquisition is bounded by this millisecond budget; other namespace lock users keep the global object-lock timeout.
|
||||
- timeout returns S3 `SlowDown`, so clients should use normal SDK retry handling.
|
||||
- this is not a fdatasync or group-commit switch. Track fdatasync batching separately with `rustfs_s3_put_object_rename_fdatasync_batch_files`.
|
||||
|
||||
## Drive timeout environment variables
|
||||
|
||||
- `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS`
|
||||
|
||||
@@ -427,6 +427,19 @@ pub const ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT: &str = "RUSTFS_OBJECT_LOCK_ACQUIRE_TI
|
||||
/// Default lock acquisition timeout: 5 seconds.
|
||||
pub const DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT: u64 = 5;
|
||||
|
||||
/// Environment variable for the experimental PUT commit namespace lock acquire timeout in milliseconds.
|
||||
///
|
||||
/// A value of `0` disables the experiment and keeps
|
||||
/// `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT` as the timeout. This only bounds the
|
||||
/// `put_object_commit` namespace write-lock wait and is intended for #925
|
||||
/// tail-drain admission experiments.
|
||||
///
|
||||
/// Default: 0 milliseconds (disabled).
|
||||
pub const ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS: &str = "RUSTFS_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS";
|
||||
|
||||
/// Default: PUT commit namespace lock acquire timeout override is disabled.
|
||||
pub const DEFAULT_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS: u64 = 0;
|
||||
|
||||
/// Environment variable for remote namespace lock RPC transport timeout in milliseconds.
|
||||
///
|
||||
/// This timeout bounds the internode RPC call itself. It is intentionally
|
||||
|
||||
@@ -228,15 +228,6 @@ pub const DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS: usize = 4;
|
||||
/// Default object interval for cooperative scanner yields.
|
||||
pub const DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS: u64 = 128;
|
||||
|
||||
/// Compatibility flag kept for Patch 3 rollback windows.
|
||||
///
|
||||
/// Inline scanner heal execution has been removed in favor of heal-candidate enqueue.
|
||||
/// When this flag is enabled, RustFS logs a warning and continues to use enqueue-based heal.
|
||||
pub const ENV_SCANNER_INLINE_HEAL_ENABLE: &str = "RUSTFS_SCANNER_INLINE_HEAL_ENABLE";
|
||||
|
||||
/// Default inline scanner heal compatibility mode.
|
||||
pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false;
|
||||
|
||||
/// Scanner speed preset controlling throttling behavior.
|
||||
///
|
||||
/// Each preset defines three parameters:
|
||||
|
||||
@@ -92,15 +92,11 @@ pub const NOTIFY_SUB_SYSTEMS: &[&str] = &[
|
||||
pub const NOTIFY_KAFKA_SUB_SYS: &str = "notify_kafka";
|
||||
pub const NOTIFY_MQTT_SUB_SYS: &str = "notify_mqtt";
|
||||
pub const NOTIFY_MYSQL_SUB_SYS: &str = "notify_mysql";
|
||||
#[allow(dead_code)]
|
||||
pub const NOTIFY_NATS_SUB_SYS: &str = "notify_nats";
|
||||
#[allow(dead_code)]
|
||||
pub const NOTIFY_NSQ_SUB_SYS: &str = "notify_nsq";
|
||||
#[allow(dead_code)]
|
||||
pub const NOTIFY_ES_SUB_SYS: &str = "notify_elasticsearch";
|
||||
pub const NOTIFY_AMQP_SUB_SYS: &str = "notify_amqp";
|
||||
pub const NOTIFY_POSTGRES_SUB_SYS: &str = "notify_postgres";
|
||||
#[allow(dead_code)]
|
||||
pub const NOTIFY_REDIS_SUB_SYS: &str = "notify_redis";
|
||||
pub const NOTIFY_REDIS_DEFAULT_CHANNEL: &str = "rustfs_notify_channel";
|
||||
pub const NOTIFY_PULSAR_SUB_SYS: &str = "notify_pulsar";
|
||||
|
||||
@@ -6,9 +6,6 @@ use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
use std::time::Duration;
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::time::sleep;
|
||||
use tracing::info;
|
||||
|
||||
const COMPRESSION_TEST_BUCKET: &str = "compression-test-bucket";
|
||||
@@ -87,17 +84,7 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul
|
||||
env.process = Some(process);
|
||||
|
||||
info!("Waiting for RustFS server with compression enabled on {}", env.address);
|
||||
for i in 0..30 {
|
||||
if TcpStream::connect(&env.address).await.is_ok() {
|
||||
info!("RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
if i == 29 {
|
||||
return Err("RustFS server failed to become ready".into());
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
Ok(())
|
||||
env.wait_for_server_ready().await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -666,17 +653,7 @@ async fn start_rustfs_with_compression_and_sse(
|
||||
env.process = Some(process);
|
||||
|
||||
info!("Waiting for RustFS server with compression + SSE-S3 enabled on {}", env.address);
|
||||
for i in 0..30 {
|
||||
if TcpStream::connect(&env.address).await.is_ok() {
|
||||
info!("RustFS server is ready after {} attempts", i + 1);
|
||||
return Ok(());
|
||||
}
|
||||
if i == 29 {
|
||||
return Err("RustFS server failed to become ready".into());
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
Ok(())
|
||||
env.wait_for_server_ready().await
|
||||
}
|
||||
|
||||
/// SSE-S3 + disk compression multipart: each part is compressed and then encrypted, and every GET
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! E2E proof that a mid-stream GET failure is *reportable* — rustfs#4784.
|
||||
//!
|
||||
//! The functional invariant (a beyond-quorum read must fail rather than return
|
||||
//! a clean short body) is already covered by
|
||||
//! `degraded_read_eof_regression_test`. This suite covers the half that issue
|
||||
//! #4784 got stuck on for a month: whether an operator can tell, from the
|
||||
//! source server's log alone, that a GET failed mid-body and **which object**
|
||||
//! it failed on.
|
||||
//!
|
||||
//! The reporter saw only downstream symptoms — `rclone` reporting
|
||||
//! `unexpected EOF` on its PUT, and the receiving RustFS logging
|
||||
//! `Io error: error reading a body from connection` with a 500. In a cross-remote
|
||||
//! `rclone sync`, the source GET body *is* the destination PUT body, so a source
|
||||
//! read that ends short of its committed `Content-Length` surfaces as a PUT
|
||||
//! failure on the far side. Built-in replication and site replication have the
|
||||
//! same shape (read locally, PUT remotely), which is why every transport in that
|
||||
//! report failed the same way.
|
||||
//!
|
||||
//! The source side, meanwhile, said nothing:
|
||||
//! * `GetObjectReaderStream`'s short-read and read-error arms only incremented
|
||||
//! a metric; their log lines sat behind the `tracing-chunk-debug` cargo
|
||||
//! feature, which is not in the default feature set and therefore is not
|
||||
//! compiled into any released binary.
|
||||
//! * `GetObjectStreamingReader` did log mid-stream failures, but only under a
|
||||
//! `request_id` — with no bucket or object name, a failure could not be
|
||||
//! traced back to the object that caused it.
|
||||
//! * Those lines were `warn!`, while `DEFAULT_LOG_LEVEL` is `error`, so a
|
||||
//! default deployment filtered them out anyway.
|
||||
//!
|
||||
//! This test reproduces the source-side fault against a real server over the S3
|
||||
//! API and asserts the operator-visible evidence, at the **default** log level.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::chaos::DiskFaultHarness;
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, timeout};
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
|
||||
const MIB: usize = 1024 * 1024;
|
||||
const OP_TIMEOUT: Duration = Duration::from_secs(90);
|
||||
|
||||
/// The structured event name every GET body failure is tagged with.
|
||||
const STREAM_BODY_EVENT: &str = "get_object_stream_body";
|
||||
|
||||
fn payload(len: usize, seed: u8) -> Vec<u8> {
|
||||
(0..len)
|
||||
.map(|i| (i as u64).wrapping_mul(2654435761).wrapping_add(seed as u64) as u8)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Upload a multipart object so the data lands in real `part.*` shard files
|
||||
/// rather than being inlined into `xl.meta` (inlined objects cannot be
|
||||
/// corrupted shard-wise, and never exercise the streaming read path).
|
||||
async fn put_multipart(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
parts: Vec<Vec<u8>>,
|
||||
) -> Result<usize, Box<dyn Error + Send + Sync>> {
|
||||
let total_len = parts.iter().map(Vec::len).sum();
|
||||
|
||||
let create = client.create_multipart_upload().bucket(bucket).key(key).send().await?;
|
||||
let upload_id = create.upload_id().ok_or("missing upload id")?.to_string();
|
||||
|
||||
let mut completed = Vec::with_capacity(parts.len());
|
||||
for (index, part_body) in parts.into_iter().enumerate() {
|
||||
let part_number = (index + 1) as i32;
|
||||
let uploaded = timeout(
|
||||
OP_TIMEOUT,
|
||||
client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(part_body))
|
||||
.send(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| format!("upload_part {part_number} timed out"))??;
|
||||
completed.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.e_tag(uploaded.e_tag().ok_or("missing part etag")?)
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
|
||||
timeout(
|
||||
OP_TIMEOUT,
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed)).build())
|
||||
.send(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| "complete_multipart_upload timed out")??;
|
||||
|
||||
Ok(total_len)
|
||||
}
|
||||
|
||||
/// rustfs#4784: reproduce the source-side fault the reporter kept hitting —
|
||||
/// a GET that commits `200` + a full `Content-Length` and then cannot finish
|
||||
/// the body — and assert the server log names the object, at the log level a
|
||||
/// default deployment actually runs with.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn midstream_get_failure_is_logged_with_the_object_at_default_log_level() -> TestResult {
|
||||
init_logging();
|
||||
info!("rustfs#4784: a mid-stream GET failure must name its object in the source log");
|
||||
|
||||
let mut harness = DiskFaultHarness::new(4).await?;
|
||||
|
||||
// Capture the child's stdout so the test can read what an operator would.
|
||||
let log_path = format!("{}/server.log", harness.env.temp_dir);
|
||||
harness.env.capture_log_path = Some(log_path.clone());
|
||||
|
||||
// Reproduce a DEFAULT deployment's logging, not the e2e harness's
|
||||
// permissive `rustfs=info`: `DEFAULT_LOG_LEVEL` is `error`. Before the
|
||||
// #4784 fix these failures were `warn!`, so a default deployment
|
||||
// filtered them out entirely — which is why the reporter's source logs
|
||||
// were empty. extra_env is applied after the harness's own RUST_LOG, so
|
||||
// this wins.
|
||||
harness.set_env("RUST_LOG", "error");
|
||||
harness.set_env("RUSTFS_OBS_LOGGER_LEVEL", "error");
|
||||
|
||||
harness.start_server().await?;
|
||||
let client = harness.env.create_s3_client();
|
||||
|
||||
let bucket = "issue4784-source-read";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
// Named after the reporter's restic index objects, which is where they
|
||||
// saw the failures.
|
||||
let key = "index/3b18542ab3af4c3d03f804c7a24173e7836ef7fa447b5d1e9d634f975cc51611";
|
||||
let expected_len = put_multipart(
|
||||
&client,
|
||||
bucket,
|
||||
key,
|
||||
vec![payload(5 * MIB, 71), payload(5 * MIB, 72), payload(5 * MIB, 73)],
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Baseline: the object reads back completely before any corruption.
|
||||
let baseline = timeout(OP_TIMEOUT, client.get_object().bucket(bucket).key(key).send())
|
||||
.await
|
||||
.map_err(|_| "baseline GET timed out")??
|
||||
.body
|
||||
.collect()
|
||||
.await?;
|
||||
assert_eq!(baseline.into_bytes().len(), expected_len, "baseline GET must return the whole object");
|
||||
|
||||
// Corrupt three of four shards in a 2+2 set: below the 2-shard read
|
||||
// quorum. The corruption sits mid-file, so block 0 still reads clean —
|
||||
// the server commits 200 + the full Content-Length and only then cannot
|
||||
// reconstruct. That is the mid-stream window the reporter's downstream
|
||||
// saw as `unexpected EOF`.
|
||||
harness.corrupt_object_shard(0, bucket, key)?;
|
||||
harness.corrupt_object_shard(1, bucket, key)?;
|
||||
harness.corrupt_object_shard(2, bucket, key)?;
|
||||
|
||||
let response = timeout(OP_TIMEOUT, client.get_object().bucket(bucket).key(key).send())
|
||||
.await
|
||||
.map_err(|_| "degraded GET timed out")?;
|
||||
|
||||
// Either outcome is functionally correct (that invariant belongs to
|
||||
// degraded_read_eof_regression_test); this suite only needs the read to
|
||||
// have failed so there is something to report.
|
||||
let delivered = match response {
|
||||
Err(err) => {
|
||||
info!("degraded GET failed before the body: {err}");
|
||||
None
|
||||
}
|
||||
Ok(response) => match response.body.collect().await {
|
||||
Ok(aggregated) => Some(aggregated.into_bytes().len()),
|
||||
Err(err) => {
|
||||
info!("degraded GET failed mid-body as expected: {err}");
|
||||
None
|
||||
}
|
||||
},
|
||||
};
|
||||
assert_ne!(
|
||||
delivered,
|
||||
Some(expected_len),
|
||||
"the beyond-quorum read unexpectedly succeeded; this suite needs a failed read to have something to report"
|
||||
);
|
||||
|
||||
// Give the child a moment to flush its stdout.
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
let logged = std::fs::read_to_string(&log_path)?;
|
||||
|
||||
let failure_lines: Vec<&str> = logged.lines().filter(|line| line.contains(STREAM_BODY_EVENT)).collect();
|
||||
|
||||
assert!(
|
||||
!failure_lines.is_empty(),
|
||||
"a mid-stream GET failure produced no `{STREAM_BODY_EVENT}` line at the default log level. \
|
||||
This is the #4784 blind spot: the failure was only counted in a metric, or logged below \
|
||||
`error` and filtered out. Captured log:\n{logged}"
|
||||
);
|
||||
|
||||
// The identity is the whole point: a request_id alone cannot be resolved
|
||||
// back to an object once the request is over.
|
||||
assert!(
|
||||
failure_lines.iter().any(|line| line.contains(key)),
|
||||
"no `{STREAM_BODY_EVENT}` line named the failing object `{key}`, so the report is still \
|
||||
unactionable. Lines seen:\n{}",
|
||||
failure_lines.join("\n")
|
||||
);
|
||||
assert!(
|
||||
failure_lines.iter().any(|line| line.contains(bucket)),
|
||||
"no `{STREAM_BODY_EVENT}` line named the failing bucket `{bucket}`. Lines seen:\n{}",
|
||||
failure_lines.join("\n")
|
||||
);
|
||||
|
||||
info!(
|
||||
"source-side evidence now present: {} stream-body failure line(s) naming the object",
|
||||
failure_lines.len()
|
||||
);
|
||||
for line in &failure_lines {
|
||||
info!("operator-visible evidence: {line}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,7 @@ use std::time::Duration;
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
type S3OperationResult<T> = Result<T, Box<aws_sdk_s3::Error>>;
|
||||
|
||||
const ALLOWED_KEY: &str = "kms-matrix-allowed-key";
|
||||
const OTHER_KEY: &str = "kms-matrix-other-key";
|
||||
@@ -130,7 +131,7 @@ fn policy_document(statements: Vec<serde_json::Value>) -> String {
|
||||
serde_json::json!({ "Version": "2012-10-17", "Statement": statements }).to_string()
|
||||
}
|
||||
|
||||
async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> Result<(), aws_sdk_s3::Error> {
|
||||
async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> S3OperationResult<()> {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
@@ -141,14 +142,14 @@ async fn put_sse_kms(client: &Client, key: &str, kms_key_id: &str) -> Result<(),
|
||||
.send()
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(aws_sdk_s3::Error::from)
|
||||
.map_err(|err| Box::new(aws_sdk_s3::Error::from(err)))
|
||||
}
|
||||
|
||||
/// Assert the operation failed with `AccessDenied` rather than any other error.
|
||||
///
|
||||
/// A bare `is_err` would also accept `KMSKeyDisabled` or an internal error, which
|
||||
/// would hide both a leak of key state and an outage masquerading as a denial.
|
||||
fn assert_access_denied<T: std::fmt::Debug>(result: Result<T, aws_sdk_s3::Error>, what: &str) {
|
||||
fn assert_access_denied<T: std::fmt::Debug>(result: S3OperationResult<T>, what: &str) {
|
||||
let error = result.expect_err(&format!("{what} must be denied"));
|
||||
assert_eq!(error.code(), Some("AccessDenied"), "{what} must fail with AccessDenied: {error:?}");
|
||||
}
|
||||
@@ -296,7 +297,7 @@ async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult {
|
||||
.send()
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(aws_sdk_s3::Error::from),
|
||||
.map_err(|err| Box::new(aws_sdk_s3::Error::from(err))),
|
||||
"SSE-KMS read by an identity holding no kms grant",
|
||||
);
|
||||
|
||||
@@ -310,7 +311,7 @@ async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult {
|
||||
.send()
|
||||
.await
|
||||
.map(|_| ())
|
||||
.map_err(aws_sdk_s3::Error::from),
|
||||
.map_err(|err| Box::new(aws_sdk_s3::Error::from(err))),
|
||||
"SSE-KMS read by an identity holding kms:GenerateDataKey but not kms:Decrypt",
|
||||
);
|
||||
|
||||
|
||||
@@ -647,7 +647,10 @@ async fn test_multipart_upload_with_sse_c(
|
||||
}
|
||||
|
||||
/// Test large multipart upload to verify streaming encryption works correctly
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "parked behind the TODO in test_local_kms_multipart_upload until streaming encryption is fixed for large files (backlog#1823)"
|
||||
)]
|
||||
async fn test_large_multipart_upload(
|
||||
s3_client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
|
||||
@@ -48,6 +48,11 @@ mod replacement_privileged_e2e_test;
|
||||
#[cfg(test)]
|
||||
mod degraded_read_eof_regression_test;
|
||||
|
||||
// rustfs#4784: a mid-stream GET failure must be reportable from the source
|
||||
// server's log alone — naming the object, at the default log level.
|
||||
#[cfg(test)]
|
||||
mod get_stream_failure_observability_test;
|
||||
|
||||
// backlog#1183: GET codec-streaming fast path must be byte/header identical to
|
||||
// the legacy duplex path before its rollout gates can be flipped on by default.
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -19,32 +19,17 @@ use std::time::Instant;
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::{error, info};
|
||||
|
||||
/// Core test categories
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum TestCategory {
|
||||
SingleValue,
|
||||
MultiValue,
|
||||
Concatenation,
|
||||
Nested,
|
||||
DenyScenarios,
|
||||
}
|
||||
|
||||
impl TestCategory {}
|
||||
|
||||
/// Test case definition
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TestDefinition {
|
||||
pub name: String,
|
||||
#[allow(dead_code)]
|
||||
pub category: TestCategory,
|
||||
pub is_critical: bool,
|
||||
}
|
||||
|
||||
impl TestDefinition {
|
||||
pub fn new(name: impl Into<String>, category: TestCategory, is_critical: bool) -> Self {
|
||||
pub fn new(name: impl Into<String>, is_critical: bool) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
category,
|
||||
is_critical,
|
||||
}
|
||||
}
|
||||
@@ -92,12 +77,12 @@ impl PolicyTestSuite {
|
||||
/// Create default test suite
|
||||
pub fn new() -> Self {
|
||||
let tests = vec![
|
||||
TestDefinition::new("test_aws_policy_variables_single_value", TestCategory::SingleValue, true),
|
||||
TestDefinition::new("test_aws_policy_variables_multi_value", TestCategory::MultiValue, true),
|
||||
TestDefinition::new("test_aws_policy_variables_concatenation", TestCategory::Concatenation, true),
|
||||
TestDefinition::new("test_aws_policy_variables_nested", TestCategory::Nested, true),
|
||||
TestDefinition::new("test_aws_policy_variables_deny", TestCategory::DenyScenarios, true),
|
||||
TestDefinition::new("test_aws_policy_variables_sts", TestCategory::SingleValue, true),
|
||||
TestDefinition::new("test_aws_policy_variables_single_value", true),
|
||||
TestDefinition::new("test_aws_policy_variables_multi_value", true),
|
||||
TestDefinition::new("test_aws_policy_variables_concatenation", true),
|
||||
TestDefinition::new("test_aws_policy_variables_nested", true),
|
||||
TestDefinition::new("test_aws_policy_variables_deny", true),
|
||||
TestDefinition::new("test_aws_policy_variables_sts", true),
|
||||
];
|
||||
|
||||
Self {
|
||||
|
||||
@@ -233,6 +233,111 @@ pub async fn test_webdav_core_operations() -> Result<()> {
|
||||
);
|
||||
info!("PASS: PUT file '{}' successful", filename);
|
||||
|
||||
// Regression for #6260: a bucket-scoped policy must be able to discover its bucket at the
|
||||
// WebDAV root without the unrelated global ListAllMyBuckets permission.
|
||||
let scoped_bucket = "webdav-scoped-bucket";
|
||||
let scoped_file = "visible.txt";
|
||||
let scoped_user = "webdav-scoped-user";
|
||||
let scoped_secret = "webdav-scoped-secret";
|
||||
let scoped_policy_name = "webdav-scoped-policy";
|
||||
|
||||
let resp = client
|
||||
.request(reqwest::Method::from_bytes(b"MKCOL").unwrap(), format!("{}/{}", base_url, scoped_bucket))
|
||||
.header("Authorization", &auth_header)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(resp.status().as_u16(), 201, "scoped test bucket should be created");
|
||||
|
||||
let resp = client
|
||||
.put(format!("{}/{}/{}", base_url, scoped_bucket, scoped_file))
|
||||
.header("Authorization", &auth_header)
|
||||
.body("visible to the scoped principal")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(resp.status().as_u16(), 201, "scoped test object should be created");
|
||||
|
||||
admin_create_user(&admin_base_url, scoped_user, scoped_secret).await?;
|
||||
admin_add_canned_policy(
|
||||
&admin_base_url,
|
||||
scoped_policy_name,
|
||||
&serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:*"],
|
||||
"Resource": [
|
||||
format!("arn:aws:s3:::{}", scoped_bucket),
|
||||
format!("arn:aws:s3:::{}/*", scoped_bucket)
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Deny",
|
||||
"Action": ["s3:*"],
|
||||
"Resource": [
|
||||
format!("arn:aws:s3:::{}", scoped_bucket),
|
||||
format!("arn:aws:s3:::{}/*", scoped_bucket)
|
||||
],
|
||||
"Condition": { "Bool": { "aws:SecureTransport": "true" } }
|
||||
},
|
||||
{
|
||||
"Effect": "Deny",
|
||||
"Action": ["s3:*"],
|
||||
"Resource": [
|
||||
format!("arn:aws:s3:::{}", scoped_bucket),
|
||||
format!("arn:aws:s3:::{}/*", scoped_bucket)
|
||||
],
|
||||
"Condition": { "StringEquals": { "s3:signatureversion": "AWS4-HMAC-SHA256" } }
|
||||
}
|
||||
]
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
admin_attach_policy_to_user(&admin_base_url, scoped_policy_name, scoped_user).await?;
|
||||
|
||||
let scoped_auth = basic_auth_header_for(scoped_user, scoped_secret);
|
||||
let resp = client
|
||||
.request(reqwest::Method::from_bytes(b"PROPFIND").unwrap(), &base_url)
|
||||
.header("Authorization", &scoped_auth)
|
||||
.header("Depth", "1")
|
||||
.header("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(resp.status().as_u16(), 207, "bucket-scoped root PROPFIND should succeed");
|
||||
let root_listing = resp.text().await?;
|
||||
assert!(root_listing.contains(scoped_bucket), "the authorized bucket should be listed");
|
||||
assert!(!root_listing.contains(bucket_name), "an unauthorized bucket must not be listed");
|
||||
|
||||
let resp = client
|
||||
.request(
|
||||
reqwest::Method::from_bytes(b"PROPFIND").unwrap(),
|
||||
format!("{}/{}", base_url, scoped_bucket),
|
||||
)
|
||||
.header("Authorization", &scoped_auth)
|
||||
.header("Depth", "1")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(resp.status().as_u16(), 207, "authorized bucket PROPFIND should succeed");
|
||||
assert!(resp.text().await?.contains(scoped_file), "the authorized object should be listed");
|
||||
|
||||
let denied_user = "webdav-no-buckets-user";
|
||||
let denied_secret = "webdav-no-buckets-secret";
|
||||
admin_create_user(&admin_base_url, denied_user, denied_secret).await?;
|
||||
let resp = client
|
||||
.request(reqwest::Method::from_bytes(b"PROPFIND").unwrap(), &base_url)
|
||||
.header("Authorization", basic_auth_header_for(denied_user, denied_secret))
|
||||
.header("Depth", "1")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(
|
||||
resp.status().as_u16(),
|
||||
207,
|
||||
"PROPFIND keeps the root resource visible when the directory listing is forbidden"
|
||||
);
|
||||
let denied_body = resp.text().await?;
|
||||
assert!(!denied_body.contains(scoped_bucket), "a denied response must not leak the scoped bucket");
|
||||
assert!(!denied_body.contains(bucket_name), "a denied response must not leak the admin bucket");
|
||||
|
||||
// Test GET (download file)
|
||||
info!("Testing WebDAV: GET (download file '{}')", filename);
|
||||
let resp = client
|
||||
|
||||
@@ -168,6 +168,24 @@ async fn wait_for_version_expired(
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_key_versions_empty(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult {
|
||||
let start = std::time::Instant::now();
|
||||
loop {
|
||||
let listing = client.list_object_versions().bucket(bucket).prefix(key).send().await?;
|
||||
if listing.versions().is_empty() && listing.delete_markers().is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if start.elapsed() >= deadline {
|
||||
return Err(format!(
|
||||
"object {bucket}/{key} still had versions or delete markers after {}s: {listing:?}",
|
||||
deadline.as_secs()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
tokio::time::sleep(StdDuration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a prefix-scoped `Days`-based expiration rule.
|
||||
fn expiration_rule(id: &str, prefix: &str, days: i32) -> Result<LifecycleRule, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let rule = LifecycleRule::builder()
|
||||
@@ -193,6 +211,21 @@ fn noncurrent_expiration_rule(
|
||||
Ok(rule)
|
||||
}
|
||||
|
||||
fn noncurrent_expiration_with_delete_marker_cleanup_rule(
|
||||
id: &str,
|
||||
prefix: &str,
|
||||
days: i32,
|
||||
) -> Result<LifecycleRule, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let rule = LifecycleRule::builder()
|
||||
.id(id)
|
||||
.filter(LifecycleRuleFilter::builder().prefix(prefix).build())
|
||||
.expiration(LifecycleExpiration::builder().expired_object_delete_marker(true).build())
|
||||
.noncurrent_version_expiration(NoncurrentVersionExpiration::builder().noncurrent_days(days).build())
|
||||
.status(ExpirationStatus::Enabled)
|
||||
.build()?;
|
||||
Ok(rule)
|
||||
}
|
||||
|
||||
async fn put_expiration_config(client: &Client, bucket: &str, rule: LifecycleRule) -> TestResult {
|
||||
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
|
||||
client
|
||||
@@ -412,6 +445,143 @@ async fn test_lifecycle_noncurrent_version_expiry_removes_only_old_version() ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A combined `NoncurrentDays=1` and `ExpiredObjectDeleteMarker=true` rule
|
||||
/// must remove a noncurrent data version and then its sole latest delete
|
||||
/// marker, without expiring current-only objects. A second prefix with only
|
||||
/// noncurrent expiry proves that marker cleanup comes from EODM.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn test_lifecycle_noncurrent_expiry_then_cleans_expired_delete_marker() -> TestResult {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut extra_env = fast_lifecycle_env();
|
||||
extra_env.push(("RUSTFS_ILM_DEBUG_DAY_SECS", "2"));
|
||||
env.start_rustfs_server_with_env(vec![], &extra_env).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "ilm-expired-delete-marker";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let cascade_key = "cascade/deleted.txt";
|
||||
let cascade_put = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(cascade_key)
|
||||
.body(ByteStream::from_static(b"cascade payload"))
|
||||
.send()
|
||||
.await?;
|
||||
let cascade_data_version = cascade_put
|
||||
.version_id()
|
||||
.map(str::to_string)
|
||||
.expect("cascade PUT returns a version id");
|
||||
let cascade_delete = client.delete_object().bucket(bucket).key(cascade_key).send().await?;
|
||||
let cascade_marker_version = cascade_delete
|
||||
.version_id()
|
||||
.map(str::to_string)
|
||||
.expect("cascade DELETE returns a marker version id");
|
||||
assert_eq!(cascade_delete.delete_marker(), Some(true));
|
||||
|
||||
let survivor_key = "cascade/current-only.txt";
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(survivor_key)
|
||||
.body(ByteStream::from_static(b"current payload"))
|
||||
.send()
|
||||
.await?;
|
||||
let survivor_before = client.get_object().bucket(bucket).key(survivor_key).send().await?;
|
||||
assert_eq!(survivor_before.body.collect().await?.into_bytes().as_ref(), b"current payload");
|
||||
|
||||
let control_key = "nve-only/deleted.txt";
|
||||
let control_put = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(control_key)
|
||||
.body(ByteStream::from_static(b"control payload"))
|
||||
.send()
|
||||
.await?;
|
||||
let control_data_version = control_put
|
||||
.version_id()
|
||||
.map(str::to_string)
|
||||
.expect("control PUT returns a version id");
|
||||
let control_delete = client.delete_object().bucket(bucket).key(control_key).send().await?;
|
||||
let control_marker_version = control_delete
|
||||
.version_id()
|
||||
.map(str::to_string)
|
||||
.expect("control DELETE returns a marker version id");
|
||||
assert_eq!(control_delete.delete_marker(), Some(true));
|
||||
|
||||
let cascade_before = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(cascade_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
cascade_before
|
||||
.versions()
|
||||
.iter()
|
||||
.any(|version| version.version_id() == Some(cascade_data_version.as_str())),
|
||||
"cascade data version must exist before lifecycle is installed: {cascade_before:?}"
|
||||
);
|
||||
assert!(
|
||||
cascade_before
|
||||
.delete_markers()
|
||||
.iter()
|
||||
.any(|marker| { marker.version_id() == Some(cascade_marker_version.as_str()) && marker.is_latest() == Some(true) }),
|
||||
"cascade latest delete marker must exist before lifecycle is installed: {cascade_before:?}"
|
||||
);
|
||||
|
||||
let lifecycle = BucketLifecycleConfiguration::builder()
|
||||
.rules(noncurrent_expiration_with_delete_marker_cleanup_rule(
|
||||
"expire-and-clean-marker",
|
||||
"cascade/",
|
||||
1,
|
||||
)?)
|
||||
.rules(noncurrent_expiration_rule("expire-only", "nve-only/", 1)?)
|
||||
.build()?;
|
||||
client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(bucket)
|
||||
.lifecycle_configuration(lifecycle)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
wait_for_key_versions_empty(&client, bucket, cascade_key, StdDuration::from_secs(90)).await?;
|
||||
wait_for_version_expired(&client, bucket, control_key, &control_data_version, StdDuration::from_secs(90)).await?;
|
||||
|
||||
let survivor = client.get_object().bucket(bucket).key(survivor_key).send().await?;
|
||||
assert_eq!(survivor.body.collect().await?.into_bytes().as_ref(), b"current payload");
|
||||
|
||||
let control_after = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(control_key)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(
|
||||
control_after.versions().is_empty(),
|
||||
"NVE-only control must remove its data version: {control_after:?}"
|
||||
);
|
||||
assert!(
|
||||
control_after
|
||||
.delete_markers()
|
||||
.iter()
|
||||
.any(|marker| { marker.version_id() == Some(control_marker_version.as_str()) && marker.is_latest() == Some(true) }),
|
||||
"NVE-only control must preserve its latest delete marker: {control_after:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `Days=0` expiration is invalid per S3 semantics (`Days` must be a positive
|
||||
/// integer >= 1). A `PutBucketLifecycleConfiguration` carrying a zero-day rule
|
||||
/// must be rejected with `InvalidArgument` (HTTP 400) - see crates/lifecycle
|
||||
|
||||
@@ -200,11 +200,19 @@ async fn add_rustfs_tier(hot: &RustFSTestEnvironment, cold: &RustFSTestEnvironme
|
||||
|
||||
async fn remove_rustfs_tier_force(hot: &RustFSTestEnvironment) -> TestResult {
|
||||
let path = format!("/rustfs/admin/v3/tier/{TIER_NAME}?force=true");
|
||||
let (status, resp) = signed_admin_request(&hot.url, Method::DELETE, &path, None, &hot.access_key, &hot.secret_key).await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("RemoveTier(RustFS) failed: status={status}, body={resp}").into());
|
||||
let deadline = Instant::now() + StdDuration::from_secs(30);
|
||||
loop {
|
||||
let (status, resp) =
|
||||
signed_admin_request(&hot.url, Method::DELETE, &path, None, &hot.access_key, &hot.secret_key).await?;
|
||||
if status.is_success() {
|
||||
return Ok(());
|
||||
}
|
||||
if !resp.contains("TierNameBackendInUse") || Instant::now() >= deadline {
|
||||
return Err(format!("RemoveTier(RustFS) failed: status={status}, body={resp}").into());
|
||||
}
|
||||
// AddTier cleanup is asynchronous; wait until its committed mutation fence clears.
|
||||
tokio::time::sleep(StdDuration::from_millis(100)).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A current-version `Transition Days=0` rule scoped to the object's prefix.
|
||||
@@ -1477,15 +1485,6 @@ async fn test_manual_transition_async_tier_failure_reports_terminal_partial() ->
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
hot_client.create_bucket().bucket(MANUAL_TIER_FAILURE_BUCKET).send().await?;
|
||||
let due_mtime = OffsetDateTime::now_utc() - time::Duration::hours(25);
|
||||
put_backdated_single_part_object(
|
||||
&hot_client,
|
||||
MANUAL_TIER_FAILURE_BUCKET,
|
||||
MANUAL_TIER_FAILURE_KEY,
|
||||
b"manual tier failure object",
|
||||
due_mtime,
|
||||
)
|
||||
.await?;
|
||||
put_lifecycle_transition_rule(
|
||||
&hot_client,
|
||||
MANUAL_TIER_FAILURE_BUCKET,
|
||||
@@ -1496,6 +1495,15 @@ async fn test_manual_transition_async_tier_failure_reports_terminal_partial() ->
|
||||
.await?;
|
||||
remove_rustfs_tier_force(&hot).await?;
|
||||
|
||||
let due_mtime = OffsetDateTime::now_utc() - time::Duration::hours(25);
|
||||
put_backdated_single_part_object(
|
||||
&hot_client,
|
||||
MANUAL_TIER_FAILURE_BUCKET,
|
||||
MANUAL_TIER_FAILURE_KEY,
|
||||
b"manual tier failure object",
|
||||
due_mtime,
|
||||
)
|
||||
.await?;
|
||||
let before_remote_count = cold_tier_object_count(&cold_client).await?;
|
||||
let accepted = manual_transition_async_run(&hot, MANUAL_TIER_FAILURE_BUCKET, MANUAL_TIER_FAILURE_PREFIX, false, 10).await?;
|
||||
assert_eq!(accepted.state, "accepted");
|
||||
|
||||
@@ -229,7 +229,6 @@ base64-simd.workspace = true
|
||||
serde_urlencoded.workspace = true
|
||||
google-cloud-storage = { workspace = true }
|
||||
google-cloud-auth = { workspace = true }
|
||||
aws-config = { workspace = true }
|
||||
faster-hex = { workspace = true }
|
||||
ratelimit = { workspace = true }
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
@@ -268,6 +267,7 @@ tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time
|
||||
# dispatcher to keep tracing's process-global callsite-interest cache honest.
|
||||
tracing-core = { workspace = true }
|
||||
serial_test = { workspace = true }
|
||||
metrics-util = { workspace = true, features = ["debugging"] }
|
||||
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
|
||||
proptest = "1"
|
||||
rcgen.workspace = true
|
||||
|
||||
@@ -90,6 +90,12 @@ use uuid::Uuid;
|
||||
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||
|
||||
pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
|
||||
pub type GetObjectSdkError = Box<SdkError<GetObjectError>>;
|
||||
pub type GetObjectTaggingSdkError = Box<SdkError<GetObjectTaggingError>>;
|
||||
pub type PutObjectTaggingSdkError = Box<SdkError<PutObjectTaggingError>>;
|
||||
pub type DeleteObjectTaggingSdkError = Box<SdkError<DeleteObjectTaggingError>>;
|
||||
|
||||
pub static GLOBAL_BUCKET_TARGET_SYS: OnceLock<BucketTargetSys> = OnceLock::new();
|
||||
|
||||
fn replication_target_versioning_enabled(versioning: Option<&BucketVersioningStatus>) -> bool {
|
||||
@@ -1968,7 +1974,7 @@ impl TargetClient {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
) -> Result<HeadObjectOutput, SdkError<HeadObjectError>> {
|
||||
) -> Result<HeadObjectOutput, HeadObjectSdkError> {
|
||||
// Announce the replication check so a RustFS target returns SSE-C
|
||||
// object metadata (etag/size) without the customer key the replication
|
||||
// worker cannot hold; otherwise SSE-C replicas never converge on HEAD.
|
||||
@@ -2001,7 +2007,7 @@ impl TargetClient {
|
||||
.await
|
||||
{
|
||||
Ok(res) => Ok(res),
|
||||
Err(e) => Err(e),
|
||||
Err(e) => Err(Box::new(e)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2023,7 +2029,7 @@ impl TargetClient {
|
||||
range: Option<String>,
|
||||
part_number: Option<i32>,
|
||||
extra_headers: HeaderMap,
|
||||
) -> Result<HeadObjectOutput, SdkError<HeadObjectError>> {
|
||||
) -> Result<HeadObjectOutput, HeadObjectSdkError> {
|
||||
let headers = proxy_outbound_headers(extra_headers);
|
||||
self.client
|
||||
.head_object()
|
||||
@@ -2036,6 +2042,7 @@ impl TargetClient {
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// GET used by the read-proxy path (MinIO `proxyGetToReplicationTarget`).
|
||||
@@ -2051,7 +2058,7 @@ impl TargetClient {
|
||||
range: Option<String>,
|
||||
part_number: Option<i32>,
|
||||
extra_headers: HeaderMap,
|
||||
) -> Result<GetObjectOutput, SdkError<GetObjectError>> {
|
||||
) -> Result<GetObjectOutput, GetObjectSdkError> {
|
||||
let headers = proxy_outbound_headers(extra_headers);
|
||||
self.client
|
||||
.get_object()
|
||||
@@ -2064,6 +2071,7 @@ impl TargetClient {
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// GetObjectTagging for the tagging read-proxy path
|
||||
@@ -2073,7 +2081,7 @@ impl TargetClient {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
) -> Result<GetObjectTaggingOutput, SdkError<GetObjectTaggingError>> {
|
||||
) -> Result<GetObjectTaggingOutput, GetObjectTaggingSdkError> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.get_object_tagging()
|
||||
@@ -2084,6 +2092,7 @@ impl TargetClient {
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// PutObjectTagging for the tagging proxy path
|
||||
@@ -2094,7 +2103,7 @@ impl TargetClient {
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
tagging: SdkTagging,
|
||||
) -> Result<PutObjectTaggingOutput, SdkError<PutObjectTaggingError>> {
|
||||
) -> Result<PutObjectTaggingOutput, PutObjectTaggingSdkError> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.put_object_tagging()
|
||||
@@ -2106,6 +2115,7 @@ impl TargetClient {
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// DeleteObjectTagging for the tagging proxy path
|
||||
@@ -2115,7 +2125,7 @@ impl TargetClient {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
) -> Result<DeleteObjectTaggingOutput, SdkError<DeleteObjectTaggingError>> {
|
||||
) -> Result<DeleteObjectTaggingOutput, DeleteObjectTaggingSdkError> {
|
||||
let headers = proxy_outbound_headers(HeaderMap::new());
|
||||
self.client
|
||||
.delete_object_tagging()
|
||||
@@ -2126,6 +2136,7 @@ impl TargetClient {
|
||||
.map_request(move |req| apply_extra_headers(req, &headers))
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
}
|
||||
|
||||
/// On success returns the version id the target assigned (from
|
||||
|
||||
@@ -2180,7 +2180,7 @@ pub async fn recover_manual_transition_jobs_once(
|
||||
if limit == 0 {
|
||||
return Err(Error::other("manual transition job recovery limit must be greater than zero"));
|
||||
}
|
||||
let list_limit = i32::try_from(limit).map_or(i32::MAX, |value| value);
|
||||
let list_limit = i32::try_from(limit).unwrap_or(i32::MAX);
|
||||
let page = api
|
||||
.clone()
|
||||
.list_objects_v2(
|
||||
@@ -2386,7 +2386,7 @@ async fn replay_manual_transition_pending_tasks(
|
||||
version_id: task.version_id,
|
||||
etag: task.etag,
|
||||
mod_time,
|
||||
size: task.size.map_or(0, |size| size),
|
||||
size: task.size.unwrap_or(0),
|
||||
is_latest: task.is_latest.unwrap_or(false),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -4994,6 +4994,9 @@ pub async fn apply_expiry_on_transitioned_object(
|
||||
src: &LcEventSrc,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> bool {
|
||||
if lc_event.action.delete_all() {
|
||||
return apply_expiry_on_non_transitioned_objects(api, oi, lc_event, src, bucket_incarnation_id).await;
|
||||
}
|
||||
let time_ilm = Metrics::time_ilm(lc_event.action);
|
||||
if let Err(_err) = expire_transitioned_object(api, oi, lc_event, src, bucket_incarnation_id).await {
|
||||
return false;
|
||||
@@ -5047,13 +5050,24 @@ pub async fn apply_expiry_on_non_transitioned_objects(
|
||||
if lc_event.action.delete_all() {
|
||||
opts.delete_prefix = true;
|
||||
opts.delete_prefix_object = true;
|
||||
opts.lifecycle_delete_all = Some(crate::object_api::LifecycleDeleteAllRequest {
|
||||
version_id: oi.version_id.filter(|version_id| !version_id.is_nil()),
|
||||
delete_marker: oi.delete_marker,
|
||||
action: lc_event.action,
|
||||
rule_id: lc_event.rule_id.clone(),
|
||||
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
||||
});
|
||||
opts.ensure_lifecycle_delete_all_journal();
|
||||
}
|
||||
|
||||
let time_ilm = Metrics::time_ilm(lc_event.action);
|
||||
|
||||
//debug!("lc_event.action: {:?}", lc_event.action);
|
||||
debug!("expiry_on_non_transitioned_objects opts: {:?}", opts);
|
||||
let mut dobj = match api.delete_object(&oi.bucket, &encode_dir_object(&oi.name), opts).await {
|
||||
let mut dobj = match api
|
||||
.delete_object_with_tier_delete_journal(&oi.bucket, &encode_dir_object(&oi.name), opts)
|
||||
.await
|
||||
{
|
||||
Ok(dobj) => dobj,
|
||||
Err(e) => {
|
||||
error!(
|
||||
@@ -5283,7 +5297,7 @@ mod tests {
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_last_day_stats::LastDayTierStats;
|
||||
use crate::bucket::lifecycle::tier_sweeper::Jentry;
|
||||
use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
|
||||
use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG};
|
||||
use crate::bucket::metadata_sys;
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::client::transition_api::ReaderImpl;
|
||||
@@ -5304,6 +5318,7 @@ mod tests {
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
|
||||
lifecycle::ExpirationOptions,
|
||||
list::ListOperations as _,
|
||||
multipart::MultipartOperations as _,
|
||||
object::{ObjectIO as _, ObjectOperations as _},
|
||||
};
|
||||
@@ -10917,6 +10932,199 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn queued_delete_all_rechecks_a_same_id_rule_moved_into_the_future() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("stale-delete-all-rule-{}", Uuid::new_v4().simple());
|
||||
let object = "object";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
metadata_sys::update(
|
||||
&bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("bucket versioning should be enabled");
|
||||
let lifecycle_xml = |days| {
|
||||
format!(
|
||||
r#"<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>delete-marker-history</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix></Prefix></Filter>
|
||||
<DelMarkerExpiration><Days>{days}</Days></DelMarkerExpiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#
|
||||
)
|
||||
};
|
||||
metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml(1).into_bytes())
|
||||
.await
|
||||
.expect("initial lifecycle rule should be stored");
|
||||
|
||||
let old_time = OffsetDateTime::now_utc() - time::Duration::days(3);
|
||||
let mut reader = PutObjReader::from_vec(b"old version".to_vec());
|
||||
ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(old_time - time::Duration::hours(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("old version should be stored");
|
||||
let marker = ecstore
|
||||
.delete_object(
|
||||
&bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(old_time),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("delete marker should be created");
|
||||
|
||||
let queued_event = crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: IlmAction::DelMarkerDeleteAllVersionsAction,
|
||||
rule_id: "delete-marker-history".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml(30).into_bytes())
|
||||
.await
|
||||
.expect("updated lifecycle rule should be stored");
|
||||
let incarnation = ecstore
|
||||
.bucket_incarnation_id_from_disk(&bucket)
|
||||
.await
|
||||
.expect("bucket incarnation should be available");
|
||||
|
||||
let deleted = super::apply_expiry_on_non_transitioned_objects(
|
||||
ecstore.clone(),
|
||||
&marker,
|
||||
&queued_event,
|
||||
&LcEventSrc::Scanner,
|
||||
incarnation,
|
||||
)
|
||||
.await;
|
||||
assert!(!deleted, "the stale queued rule must be rejected");
|
||||
let versions = ecstore
|
||||
.clone()
|
||||
.list_object_versions(&bucket, object, None, None, None, 10)
|
||||
.await
|
||||
.expect("remaining versions should be listable");
|
||||
assert_eq!(versions.objects.iter().filter(|version| version.name == object).count(), 2);
|
||||
|
||||
metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml(1).into_bytes())
|
||||
.await
|
||||
.expect("due lifecycle rule should be restored");
|
||||
let deleted = super::apply_expiry_on_non_transitioned_objects(
|
||||
ecstore.clone(),
|
||||
&marker,
|
||||
&queued_event,
|
||||
&LcEventSrc::Scanner,
|
||||
incarnation,
|
||||
)
|
||||
.await;
|
||||
assert!(deleted, "the current due rule should purge marker and history");
|
||||
let versions = ecstore
|
||||
.clone()
|
||||
.list_object_versions(&bucket, object, None, None, None, 10)
|
||||
.await
|
||||
.expect("purged versions should be listable");
|
||||
assert_eq!(versions.objects.iter().filter(|version| version.name == object).count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn queued_expired_object_all_versions_purges_history_through_transitioned_dispatch() {
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("expired-all-versions-{}", Uuid::new_v4().simple());
|
||||
let object = "object";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
metadata_sys::update(
|
||||
&bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>".to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("bucket versioning should be enabled");
|
||||
metadata_sys::update(
|
||||
&bucket,
|
||||
BUCKET_LIFECYCLE_CONFIG,
|
||||
br#"<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>delete-all-versions</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix></Prefix></Filter>
|
||||
<Expiration><Days>1</Days><ExpiredObjectAllVersions>true</ExpiredObjectAllVersions></Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#
|
||||
.to_vec(),
|
||||
)
|
||||
.await
|
||||
.expect("delete-all lifecycle rule should be stored");
|
||||
|
||||
let old_time = OffsetDateTime::now_utc() - time::Duration::days(3);
|
||||
let mut old_reader = PutObjReader::from_vec(b"old version".to_vec());
|
||||
ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut old_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(old_time - time::Duration::hours(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("old version should be stored");
|
||||
let mut current_reader = PutObjReader::from_vec(b"current version".to_vec());
|
||||
let mut current = ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut current_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(old_time),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("current version should be stored");
|
||||
current.transitioned_object.status = crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string();
|
||||
|
||||
let incarnation = ecstore
|
||||
.bucket_incarnation_id_from_disk(&bucket)
|
||||
.await
|
||||
.expect("bucket incarnation should be available");
|
||||
let deleted = super::apply_expiry_on_transitioned_object(
|
||||
ecstore.clone(),
|
||||
¤t,
|
||||
&crate::bucket::lifecycle::lifecycle::Event {
|
||||
action: IlmAction::DeleteAllVersionsAction,
|
||||
rule_id: "delete-all-versions".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
&LcEventSrc::Scanner,
|
||||
incarnation,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(deleted, "delete-all must not degrade to transitioned single-version expiry");
|
||||
let versions = ecstore
|
||||
.list_object_versions(&bucket, object, None, None, None, 10)
|
||||
.await
|
||||
.expect("purged versions should be listable");
|
||||
assert_eq!(versions.objects.iter().filter(|version| version.name == object).count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_object_lifecycle_skips_current_expiration_for_explicit_legal_hold() {
|
||||
let lc = latest_expiration_lifecycle();
|
||||
|
||||
@@ -435,12 +435,88 @@ async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jen
|
||||
remove_tier_delete_journal_entry(api, je).await
|
||||
}
|
||||
|
||||
async fn reconcile_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
let (data, metadata) =
|
||||
config_boundary::read_config_with_metadata(api.clone(), &tier_delete_journal_object_name(je), &ObjectOptions::default())
|
||||
fn object_info_references_tier_delete(info: &ObjectInfo, je: &Jentry) -> std::io::Result<bool> {
|
||||
if info.transitioned_object.status != rustfs_filemeta::TRANSITION_COMPLETE
|
||||
|| info.transitioned_object.name != je.obj_name
|
||||
|| info.transitioned_object.tier != je.tier_name
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let source_backend_identity = tier_destination_id_from_metadata(&info.user_defined)?;
|
||||
if source_backend_identity.is_some() && source_backend_identity != je.backend_identity {
|
||||
return Ok(false);
|
||||
}
|
||||
if !je.version_id_exact {
|
||||
return Ok(true);
|
||||
}
|
||||
Ok(match info.transition_version_state {
|
||||
rustfs_filemeta::TransitionVersionState::Unknown => true,
|
||||
rustfs_filemeta::TransitionVersionState::KnownDisabled => false,
|
||||
rustfs_filemeta::TransitionVersionState::SuspendedNull | rustfs_filemeta::TransitionVersionState::Exact => {
|
||||
info.transitioned_object.version_id == je.version_id
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn prepared_tier_delete_has_live_source(
|
||||
api: &ECStore,
|
||||
source: &TierDeleteSourceIdentity,
|
||||
je: &Jentry,
|
||||
) -> std::io::Result<(bool, Vec<crate::store::ObjectLockDiagGuard>)> {
|
||||
let lock_object = rustfs_utils::path::encode_dir_object(&source.object);
|
||||
let mut lock_opts = ObjectOptions::default();
|
||||
let read_guards = api
|
||||
.acquire_all_object_read_locks("tier_delete_journal_recovery", &source.bucket, &lock_object, &mut lock_opts)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
if api.ctx.lock_manager().is_disabled() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier delete journal recovery requires namespace locking",
|
||||
));
|
||||
}
|
||||
let mut has_live_source = false;
|
||||
for pool in &api.pools {
|
||||
let set = pool.get_disks_by_key(&lock_object);
|
||||
let Some(versions) = set
|
||||
.load_file_info_versions_exact(&source.bucket, &source.object)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
.map_err(std::io::Error::other)?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
for version in versions.versions.iter().filter(|version| !version.tier_free_version()) {
|
||||
let info = ObjectInfo::from_file_info(version, &source.bucket, &source.object, source.versioned);
|
||||
if object_info_references_tier_delete(&info, je)? {
|
||||
has_live_source = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if has_live_source {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if read_guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier delete journal recovery object read lock was lost",
|
||||
));
|
||||
}
|
||||
Ok((has_live_source, read_guards))
|
||||
}
|
||||
|
||||
async fn reconcile_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jentry) -> std::io::Result<()> {
|
||||
let journal_name = tier_delete_journal_object_name(je);
|
||||
let (data, metadata) = config_boundary::read_config_with_metadata(api.clone(), &journal_name, &ObjectOptions::default())
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
let current = decode_tier_delete_journal_entry(&data).map_err(std::io::Error::other)?;
|
||||
if tier_delete_journal_object_name(¤t) != journal_name {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"prepared tier delete journal content does not match its object name",
|
||||
));
|
||||
}
|
||||
if current.state != TierDeleteJournalState::Prepared {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
@@ -453,16 +529,21 @@ async fn reconcile_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Je
|
||||
"prepared tier delete journal has no entity tag",
|
||||
));
|
||||
};
|
||||
let source = je
|
||||
let source = current
|
||||
.source
|
||||
.as_ref()
|
||||
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "prepared tier delete journal has no source"))?;
|
||||
match api
|
||||
.get_object_info(&source.bucket, &source.object, &source.lookup_options())
|
||||
.await
|
||||
{
|
||||
Ok(info) if source.matches(&info) => {
|
||||
match config_boundary::delete_config_if_match(api, &tier_delete_journal_object_name(¤t), &etag).await {
|
||||
match prepared_tier_delete_has_live_source(&api, source, ¤t).await {
|
||||
Ok((true, read_guards)) => {
|
||||
if read_guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier delete journal recovery object read lock was lost before abort",
|
||||
));
|
||||
}
|
||||
let result = config_boundary::delete_config_if_match(api, &tier_delete_journal_object_name(¤t), &etag).await;
|
||||
drop(read_guards);
|
||||
match result {
|
||||
Ok(()) => Ok(()),
|
||||
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
@@ -471,17 +552,32 @@ async fn reconcile_prepared_tier_delete_journal_entry(api: Arc<ECStore>, je: &Je
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
Ok(_info) if source.has_stable_identity() => {
|
||||
commit_prepared_tier_delete_journal_entry_if_current(api, current, etag).await
|
||||
Ok((false, read_guards)) if source.has_stable_identity() => {
|
||||
if read_guards.iter().any(crate::store::ObjectLockDiagGuard::is_lock_lost) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"tier delete journal recovery object read lock was lost before commit",
|
||||
));
|
||||
}
|
||||
let mut commit_opts = ObjectOptions::default();
|
||||
for signal in read_guards
|
||||
.iter()
|
||||
.filter_map(crate::store::ObjectLockDiagGuard::lock_lost_signal)
|
||||
{
|
||||
commit_opts.add_namespace_lock_lost_signal(signal);
|
||||
}
|
||||
let committed =
|
||||
commit_prepared_tier_delete_journal_entry_if_current(api.clone(), current, etag, &commit_opts).await?;
|
||||
// Keep namespace locks only through the journal CAS. Remote-tier IO
|
||||
// must not block writers for the object during recovery.
|
||||
drop(read_guards);
|
||||
process_committed_tier_delete_journal_entry(api, &committed).await
|
||||
}
|
||||
Ok(_) => Err(std::io::Error::new(
|
||||
Ok((false, _read_guards)) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal source identity is not sufficient to confirm deletion",
|
||||
)),
|
||||
Err(Error::ObjectNotFound(_, _)) | Err(Error::FileNotFound) | Err(Error::FileVersionNotFound) => {
|
||||
commit_prepared_tier_delete_journal_entry_if_current(api, current, etag).await
|
||||
}
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -489,7 +585,8 @@ async fn commit_prepared_tier_delete_journal_entry_if_current(
|
||||
api: Arc<ECStore>,
|
||||
mut committed: Jentry,
|
||||
etag: String,
|
||||
) -> std::io::Result<()> {
|
||||
lock_opts: &ObjectOptions,
|
||||
) -> std::io::Result<Jentry> {
|
||||
committed.state = TierDeleteJournalState::Committed;
|
||||
let data = encode_tier_delete_journal_entry(&committed).map_err(std::io::Error::other)?;
|
||||
match config_boundary::save_config_with_opts(
|
||||
@@ -502,12 +599,13 @@ async fn commit_prepared_tier_delete_journal_entry_if_current(
|
||||
if_match: Some(etag),
|
||||
..Default::default()
|
||||
}),
|
||||
namespace_lock_fence: lock_opts.namespace_lock_fence.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => process_committed_tier_delete_journal_entry(api, &committed).await,
|
||||
Ok(()) => Ok(committed),
|
||||
Err(Error::PreconditionFailed) => Err(std::io::Error::new(
|
||||
std::io::ErrorKind::WouldBlock,
|
||||
"prepared tier delete journal changed before commit",
|
||||
@@ -582,6 +680,18 @@ pub async fn recover_tier_delete_journal_entries(
|
||||
}
|
||||
};
|
||||
|
||||
if tier_delete_journal_object_name(&je) != object.name {
|
||||
stats.failed += 1;
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
journal_object = %object.name,
|
||||
"Tier delete journal content does not match its object name and will be retained"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if je.backend_identity.is_none() {
|
||||
stats.failed += 1;
|
||||
warn!(
|
||||
@@ -699,16 +809,14 @@ where
|
||||
mod tests {
|
||||
use super::{
|
||||
TIER_DELETE_JOURNAL_EXACT_VERSION, TIER_DELETE_JOURNAL_STATE_VERSION, await_tier_delete_journal_recovery,
|
||||
decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, record_tier_delete_journal_backend_identity,
|
||||
tier_delete_journal_object_name,
|
||||
decode_tier_delete_journal_entry, encode_tier_delete_journal_entry, object_info_references_tier_delete,
|
||||
record_tier_delete_journal_backend_identity, tier_delete_journal_object_name,
|
||||
};
|
||||
use crate::bucket::lifecycle::tier_sweeper::{Jentry, TierDeleteJournalState, TierDeleteSourceIdentity};
|
||||
use crate::error::Result;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn journal_entry() -> Jentry {
|
||||
Jentry {
|
||||
@@ -738,6 +846,16 @@ mod tests {
|
||||
assert_eq!(decoded.version_state, je.version_state);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_journal_object_name_binds_persisted_content() {
|
||||
let original = journal_entry();
|
||||
let original_name = tier_delete_journal_object_name(&original);
|
||||
let mut replaced = original;
|
||||
replaced.obj_name = "remote/replaced".to_string();
|
||||
|
||||
assert_ne!(tier_delete_journal_object_name(&replaced), original_name);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_transaction_roundtrips_prepared_source_identity() {
|
||||
let mut je = journal_entry();
|
||||
@@ -765,26 +883,35 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_delete_source_identity_rejects_recreated_object() {
|
||||
let version_id = Uuid::from_u128(1);
|
||||
let data_dir = Uuid::from_u128(2);
|
||||
let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1);
|
||||
let info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(version_id),
|
||||
data_dir: Some(data_dir),
|
||||
mod_time: Some(mod_time),
|
||||
fn prepared_recovery_blocks_any_live_reference_to_the_remote_version() {
|
||||
let je = journal_entry();
|
||||
let mut metadata = std::collections::HashMap::new();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex(je.backend_identity.expect("test journal should bind a backend")),
|
||||
);
|
||||
let mut info = ObjectInfo {
|
||||
user_defined: std::sync::Arc::new(metadata),
|
||||
transitioned_object: crate::storage_api_contracts::lifecycle::TransitionedObject {
|
||||
name: je.obj_name.clone(),
|
||||
version_id: je.version_id.clone(),
|
||||
tier: je.tier_name.clone(),
|
||||
status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
..Default::default()
|
||||
};
|
||||
let source = TierDeleteSourceIdentity::from_object_info("bucket", "object", &info, true, false);
|
||||
assert!(source.matches(&info));
|
||||
|
||||
let recreated = ObjectInfo {
|
||||
data_dir: Some(Uuid::from_u128(3)),
|
||||
..info
|
||||
};
|
||||
assert!(!source.matches(&recreated));
|
||||
assert!(object_info_references_tier_delete(&info, &je).expect("matching reference should be valid"));
|
||||
info.transitioned_object.version_id = "other-version".to_string();
|
||||
assert!(!object_info_references_tier_delete(&info, &je).expect("different exact version should be valid"));
|
||||
info.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
|
||||
assert!(
|
||||
object_info_references_tier_delete(&info, &je).expect("legacy unknown reference should fail closed"),
|
||||
"an unknown live source may still reference the journaled remote version"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -334,32 +334,6 @@ impl TierDeleteSourceIdentity {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn lookup_options(&self) -> crate::object_api::ObjectOptions {
|
||||
crate::object_api::ObjectOptions {
|
||||
version_id: self.version_id.clone(),
|
||||
versioned: self.versioned,
|
||||
version_suspended: self.version_suspended,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn matches(&self, info: &ObjectInfo) -> bool {
|
||||
if self.bucket != info.bucket {
|
||||
return false;
|
||||
}
|
||||
if let Some(version_id) = &self.version_id {
|
||||
return info.version_id.map(|id| id.to_string()).as_deref() == Some(version_id.as_str())
|
||||
&& self.data_dir == info.data_dir.map(|id| id.to_string());
|
||||
}
|
||||
if self.data_dir.is_some() {
|
||||
return self.data_dir == info.data_dir.map(|id| id.to_string());
|
||||
}
|
||||
self.etag.is_some()
|
||||
&& self.etag == info.etag
|
||||
&& self.mod_time.is_some()
|
||||
&& self.mod_time == info.mod_time.map(|time| time.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn has_stable_identity(&self) -> bool {
|
||||
self.version_id.is_some() || self.data_dir.is_some() || (self.etag.is_some() && self.mod_time.is_some())
|
||||
}
|
||||
|
||||
@@ -1016,7 +1016,7 @@ pub async fn recover_transition_transaction_records(
|
||||
return Err(Error::other("transition transaction recovery limit must be greater than zero"));
|
||||
}
|
||||
|
||||
let list_limit = i32::try_from(limit).map_or(i32::MAX, |value| value);
|
||||
let list_limit = i32::try_from(limit).unwrap_or(i32::MAX);
|
||||
let list = api
|
||||
.clone()
|
||||
.list_objects_v2(
|
||||
|
||||
@@ -23,6 +23,8 @@ use super::replication_queue_boundary::DeletedObjectReplicationInfo;
|
||||
use super::replication_storage_boundary::{
|
||||
DeletedObject, ObjectInfo, ObjectOptions, ObjectToDelete, deleted_object_for_replication,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
@@ -32,6 +34,9 @@ pub(crate) type ReplicationLifecycleConfig = ReplicationConfig;
|
||||
|
||||
pub(crate) struct ReplicationLifecycleBridge;
|
||||
|
||||
#[cfg(test)]
|
||||
static SCHEDULED_DELETE_OBJECTS: Mutex<Vec<DeletedObject>> = Mutex::new(Vec::new());
|
||||
|
||||
impl ReplicationLifecycleBridge {
|
||||
#[allow(
|
||||
dead_code,
|
||||
@@ -85,6 +90,13 @@ impl ReplicationLifecycleBridge {
|
||||
}
|
||||
|
||||
pub(crate) async fn schedule_delete(bucket: String, delete_object: DeletedObject) {
|
||||
#[cfg(test)]
|
||||
{
|
||||
SCHEDULED_DELETE_OBJECTS
|
||||
.lock()
|
||||
.expect("scheduled delete test hook lock should not poison")
|
||||
.push(delete_object.clone());
|
||||
}
|
||||
super::replication_pool::schedule_replication_delete(DeletedObjectReplicationInfo {
|
||||
delete_object: deleted_object_for_replication(delete_object),
|
||||
bucket,
|
||||
@@ -93,6 +105,15 @@ impl ReplicationLifecycleBridge {
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn take_scheduled_deletes_for_test() -> Vec<DeletedObject> {
|
||||
std::mem::take(
|
||||
&mut *SCHEDULED_DELETE_OBJECTS
|
||||
.lock()
|
||||
.expect("scheduled delete test hook lock should not poison"),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -52,8 +52,8 @@ use super::replication_storage_boundary::{
|
||||
ReplicationObjectIO, ReplicationStorage, StatObjectOptions, StorageObjectInfoOrErr, WalkOptions,
|
||||
};
|
||||
use super::replication_target_boundary::{
|
||||
ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, PutObjectOptions, PutObjectPartOptions, ReplicationTargetStore,
|
||||
SsecPassthroughCapability, SsecPassthroughGate, TargetClient, is_replication_target_offline_error,
|
||||
ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions,
|
||||
ReplicationTargetStore, SsecPassthroughCapability, SsecPassthroughGate, TargetClient, is_replication_target_offline_error,
|
||||
replication_action_for_target_head, replication_complete_multipart_options, replication_delete_marker_purge_remove_options,
|
||||
replication_delete_remove_options, replication_force_delete_remove_options, replication_object_is_ssec_encrypted,
|
||||
replication_put_object_header_size, replication_put_object_options, replication_target_head_is_newer_null_version,
|
||||
@@ -214,7 +214,7 @@ async fn head_object_for_worker(
|
||||
target_bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<String>,
|
||||
) -> std::result::Result<HeadObjectOutput, SdkError<HeadObjectError>> {
|
||||
) -> std::result::Result<HeadObjectOutput, HeadObjectSdkError> {
|
||||
target_client.head_object(target_bucket, object, version_id).await
|
||||
}
|
||||
|
||||
@@ -233,7 +233,7 @@ async fn mark_replication_target_offline_if_needed(target_client: &Arc<TargetCli
|
||||
async fn head_object_fallback(
|
||||
tgt_client: &TargetClient,
|
||||
object: &str,
|
||||
) -> std::result::Result<Option<HeadObjectOutput>, SdkError<HeadObjectError>> {
|
||||
) -> std::result::Result<Option<HeadObjectOutput>, HeadObjectSdkError> {
|
||||
match head_object_for_worker(tgt_client, &tgt_client.bucket, object, None).await {
|
||||
Ok(oi) => Ok(Some(oi)),
|
||||
Err(e) if e.as_service_error().is_some_and(|se| se.is_not_found()) || has_raw_status(&e, 404) => Ok(None),
|
||||
@@ -1152,11 +1152,11 @@ fn spawn_resync_walk_task<S: ReplicationStorage>(
|
||||
/// updating the per-object status counters and returning the accounted size
|
||||
/// together with any verification error.
|
||||
async fn verify_resync_head_result(
|
||||
head_result: std::result::Result<HeadObjectOutput, SdkError<HeadObjectError>>,
|
||||
head_result: std::result::Result<HeadObjectOutput, HeadObjectSdkError>,
|
||||
roi: &ReplicateObjectInfo,
|
||||
st: &mut TargetReplicationResyncStatus,
|
||||
target_client: &Arc<TargetClient>,
|
||||
) -> (i64, Option<SdkError<HeadObjectError>>) {
|
||||
) -> (i64, Option<HeadObjectSdkError>) {
|
||||
match head_result {
|
||||
Ok(_) => {
|
||||
st.replicated_count += 1;
|
||||
@@ -1275,7 +1275,7 @@ async fn resync_worker_process_object<S: ReplicationStorage>(
|
||||
"Processed resync object"
|
||||
);
|
||||
}
|
||||
st.error = err.as_ref().and_then(resync_target_error_detail);
|
||||
st.error = err.as_ref().and_then(|err| resync_target_error_detail(err.as_ref()));
|
||||
|
||||
st
|
||||
}
|
||||
@@ -2467,7 +2467,7 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
let non_retryable = matches!(
|
||||
&e,
|
||||
e.as_ref(),
|
||||
SdkError::ServiceError(service_err)
|
||||
if is_retryable_delete_replication_head_error(
|
||||
service_err.err().is_not_found(),
|
||||
|
||||
@@ -36,7 +36,8 @@ use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
pub(crate) use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient, resolve_read_api_version_id,
|
||||
AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
|
||||
resolve_read_api_version_id,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::target::BucketTarget;
|
||||
|
||||
@@ -41,6 +41,10 @@ use bytes::Bytes;
|
||||
use futures::lock::Mutex;
|
||||
use metrics::counter;
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE,
|
||||
INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
|
||||
};
|
||||
use rustfs_protos::ChannelClass;
|
||||
use rustfs_protos::evict_failed_connection;
|
||||
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
|
||||
@@ -64,7 +68,7 @@ use std::{
|
||||
atomic::{AtomicBool, AtomicU32, Ordering},
|
||||
},
|
||||
task::{Context, Poll},
|
||||
time::Duration,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tokio::time;
|
||||
use tokio::{
|
||||
@@ -1790,6 +1794,16 @@ fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str, value_
|
||||
}
|
||||
}
|
||||
|
||||
fn read_version_stage_timer(attribution_enabled: bool) -> Option<Instant> {
|
||||
attribution_enabled.then(Instant::now)
|
||||
}
|
||||
|
||||
fn record_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
|
||||
if let Some(started_at) = started_at {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_stage(stage, started_at.elapsed());
|
||||
}
|
||||
}
|
||||
|
||||
/// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads
|
||||
/// and falling back to the JSON compatibility strings. Used to size the RPC for the payload
|
||||
/// histogram / large-payload alerting (grpc-optimization P0 instrumentation).
|
||||
@@ -2705,8 +2719,11 @@ impl DiskAPI for RemoteDisk {
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
let opts_str = compat_json(opts)?;
|
||||
let opts_bin = encode_msgpack(opts)?;
|
||||
let read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
let encode_started = read_version_stage_timer(read_version_attribution_enabled);
|
||||
let encoded_opts = compat_json(opts).and_then(|opts_str| encode_msgpack(opts).map(|opts_bin| (opts_str, opts_bin)));
|
||||
record_read_version_stage(INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, encode_started);
|
||||
let (opts_str, opts_bin) = encoded_opts?;
|
||||
|
||||
// Idempotent version read: eligible for the bounded transient-network retry so a single
|
||||
// reset-by-peer during the read-after-write window does not erode the metadata read
|
||||
@@ -2722,6 +2739,14 @@ impl DiskAPI for RemoteDisk {
|
||||
.get_client()
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
let request_payload_bytes = read_version_attribution_enabled.then(|| {
|
||||
disk.len()
|
||||
.saturating_add(volume.len())
|
||||
.saturating_add(path.len())
|
||||
.saturating_add(version_id.len())
|
||||
.saturating_add(opts_str.len())
|
||||
.saturating_add(opts_bin.len())
|
||||
});
|
||||
let request = Request::new(ReadVersionRequest {
|
||||
disk,
|
||||
volume: volume.to_string(),
|
||||
@@ -2731,14 +2756,47 @@ impl DiskAPI for RemoteDisk {
|
||||
opts_bin: opts_bin.into(),
|
||||
});
|
||||
|
||||
let response = client.read_version(request).await?.into_inner();
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_request();
|
||||
if let Some(request_payload_bytes) = request_payload_bytes {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_sent_bytes(request_payload_bytes);
|
||||
}
|
||||
let rpc_started = read_version_stage_timer(read_version_attribution_enabled);
|
||||
let response = match client.read_version(request).await {
|
||||
Ok(response) => {
|
||||
record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
|
||||
response.into_inner()
|
||||
}
|
||||
Err(err) => {
|
||||
record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error();
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
|
||||
if !response.success {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error();
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
|
||||
let file_info = decode_msgpack_or_json::<FileInfo>(&response.file_info_bin, &response.file_info, "FileInfo")?;
|
||||
validate_decoded_file_info(&file_info)?;
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_recv_bytes(
|
||||
response.file_info.len().saturating_add(response.file_info_bin.len()),
|
||||
);
|
||||
let decode_started = read_version_stage_timer(read_version_attribution_enabled);
|
||||
let file_info = match decode_msgpack_or_json::<FileInfo>(&response.file_info_bin, &response.file_info, "FileInfo")
|
||||
.and_then(|file_info| {
|
||||
validate_decoded_file_info(&file_info)?;
|
||||
Ok(file_info)
|
||||
}) {
|
||||
Ok(file_info) => {
|
||||
record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, decode_started);
|
||||
file_info
|
||||
}
|
||||
Err(err) => {
|
||||
record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, decode_started);
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error();
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
Ok(file_info)
|
||||
},
|
||||
@@ -7931,12 +7989,17 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn read_version_uses_the_metadata_timeout_on_a_stalled_peer() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let Some((base_addr, accept_task)) = spawn_stalled_grpc_peer().await else {
|
||||
return;
|
||||
};
|
||||
let remote_disk = remote_disk_for_addr(&base_addr).await;
|
||||
let metrics = rustfs_io_metrics::internode_metrics::global_internode_metrics();
|
||||
let previous_stage_metrics = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
metrics.reset_for_test();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
@@ -7960,6 +8023,18 @@ mod tests {
|
||||
)
|
||||
.await;
|
||||
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(previous_stage_metrics);
|
||||
let snapshot = metrics.snapshot();
|
||||
assert!(
|
||||
snapshot.outgoing_requests_total >= 1,
|
||||
"ReadVersion call site should record outgoing attempts when attribution is enabled"
|
||||
);
|
||||
assert!(
|
||||
snapshot.sent_bytes_total > 0,
|
||||
"ReadVersion call site should record request payload bytes when attribution is enabled"
|
||||
);
|
||||
metrics.reset_for_test();
|
||||
|
||||
remote_disk.cancel_token.cancel();
|
||||
accept_task.abort();
|
||||
}
|
||||
|
||||
@@ -14,10 +14,11 @@
|
||||
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE,
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_WRITE_ALL,
|
||||
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
#[cfg(test)]
|
||||
use rustfs_io_metrics::internode_metrics::InternodeMetricsSnapshot;
|
||||
@@ -82,6 +83,59 @@ pub(crate) fn record_remote_disk_grpc_read_all_request() {
|
||||
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_version_request() {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_outgoing_request_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_version_error() {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics()
|
||||
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_version_sent_bytes(bytes: usize) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
bytes,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_version_recv_bytes(bytes: usize) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
bytes,
|
||||
);
|
||||
record_grpc_payload_size(INTERNODE_OPERATION_GRPC_READ_VERSION, bytes);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_version_stage(stage: &'static str, duration: Duration) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_stage_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
stage,
|
||||
duration,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_all_recv_bytes(bytes: usize) {
|
||||
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL,
|
||||
|
||||
@@ -1041,7 +1041,13 @@ fn should_count_decommission_version_complete(ignore: bool, cleanup_ignored: boo
|
||||
fn is_decommission_copy_cleanup_safe_error(err: &Error) -> bool {
|
||||
// DataMovementOverwriteErr only means source and destination pool resolved to
|
||||
// the same pool. Without a target equivalence check it is not cleanup-safe.
|
||||
is_err_object_not_found(err) || is_err_version_not_found(err)
|
||||
if is_err_object_not_found(err) || is_err_version_not_found(err) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// A not-found surfacing from inside a data-movement stage is the same
|
||||
// condition once the wrapper is unwrapped (backlog#1827 T2).
|
||||
crate::data_movement::data_movement_stage_source(err).is_some_and(is_decommission_copy_cleanup_safe_error)
|
||||
}
|
||||
|
||||
fn is_decommission_target_capacity_error(err: &Error) -> bool {
|
||||
@@ -1049,6 +1055,13 @@ fn is_decommission_target_capacity_error(err: &Error) -> bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
// A stage failure keeps the error it wrapped, so classify by type rather
|
||||
// than by the rendered message (backlog#1827 T2). The substring fallback
|
||||
// stays for errors that reached here through some other wrapper.
|
||||
if let Some(source) = crate::data_movement::data_movement_stage_source(err) {
|
||||
return is_decommission_target_capacity_error(source);
|
||||
}
|
||||
|
||||
let message = err.to_string();
|
||||
let disk_full = Error::DiskFull.to_string();
|
||||
let storage_full = Error::StorageFull.to_string();
|
||||
@@ -4427,6 +4440,36 @@ mod tests {
|
||||
assert!(is_decommission_target_capacity_error(&Error::StorageFull));
|
||||
}
|
||||
|
||||
/// The decommission loop classifies errors that came back through a
|
||||
/// data-movement stage wrapper. Before backlog#1827 T2 the wrapper flattened
|
||||
/// everything into `Error::other(String)`, so these two classifiers had to
|
||||
/// match on rendered text; now the wrapped error is recoverable by type.
|
||||
#[test]
|
||||
fn decommission_classifiers_see_through_a_stage_wrapper() {
|
||||
let wrap = |inner: Error| {
|
||||
crate::data_movement::data_movement_stage_error_for_test(
|
||||
"decommission_object",
|
||||
"put_object",
|
||||
"bucket-a",
|
||||
"object-a",
|
||||
inner,
|
||||
)
|
||||
};
|
||||
|
||||
// Capacity: the target pool filling up must still stop the loop.
|
||||
assert!(is_decommission_target_capacity_error(&wrap(Error::DiskFull)));
|
||||
assert!(is_decommission_target_capacity_error(&wrap(Error::StorageFull)));
|
||||
assert!(!is_decommission_target_capacity_error(&wrap(Error::SlowDown)));
|
||||
|
||||
// Cleanup safety: a not-found surfacing from inside a stage is the same
|
||||
// condition as one surfacing directly, so the source entry stays
|
||||
// eligible for cleanup.
|
||||
let not_found = Error::ObjectNotFound("bucket-a".to_string(), "object-a".to_string());
|
||||
assert!(is_decommission_copy_cleanup_safe_error(¬_found));
|
||||
assert!(is_decommission_copy_cleanup_safe_error(&wrap(not_found)));
|
||||
assert!(!is_decommission_copy_cleanup_safe_error(&wrap(Error::SlowDown)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
|
||||
let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
|
||||
|
||||
@@ -471,8 +471,60 @@ fn resolve_data_movement_abort_result(
|
||||
))
|
||||
}
|
||||
|
||||
fn data_movement_stage_error(op_label: &str, stage: &str, bucket: &str, object: &str, err: impl std::fmt::Display) -> Error {
|
||||
Error::other(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"))
|
||||
/// A data-movement stage failure that keeps the error it wrapped.
|
||||
///
|
||||
/// The rendered message is byte-identical to the `format!` this replaced, so
|
||||
/// logs and any message-matching callers are unaffected. What changes is that
|
||||
/// the original error stays reachable through `source()`, which is what lets
|
||||
/// the decommission loop classify by type instead of by substring
|
||||
/// (backlog#1827 T2).
|
||||
#[derive(Debug)]
|
||||
struct DataMovementStageError {
|
||||
rendered: String,
|
||||
source: Box<dyn std::error::Error + Send + Sync>,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for DataMovementStageError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.rendered)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for DataMovementStageError {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
Some(self.source.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, object: &str, err: E) -> Error
|
||||
where
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
let rendered = format!("{op_label}: {stage} failed for {bucket}/{object}: {err}");
|
||||
Error::other(DataMovementStageError {
|
||||
rendered,
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn data_movement_stage_error_for_test(op_label: &str, stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
data_movement_stage_error(op_label, stage, bucket, object, err)
|
||||
}
|
||||
|
||||
/// Recover the error a [`data_movement_stage_error`] wrapped, if this is one.
|
||||
///
|
||||
/// `Error::other` boxes through `std::io::Error`, so the chain is
|
||||
/// `StorageError::Io` -> `DataMovementStageError` -> the original error.
|
||||
pub(crate) fn data_movement_stage_source(err: &Error) -> Option<&Error> {
|
||||
let Error::Io(io_err) = err else {
|
||||
return None;
|
||||
};
|
||||
io_err
|
||||
.get_ref()?
|
||||
.downcast_ref::<DataMovementStageError>()?
|
||||
.source
|
||||
.downcast_ref::<Error>()
|
||||
}
|
||||
|
||||
fn schedule_data_movement_multipart_abort_cleanup(
|
||||
@@ -1865,6 +1917,40 @@ mod tests {
|
||||
assert!(message.contains(Error::SlowDown.to_string().as_str()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage_error_renders_exactly_as_the_format_it_replaced() {
|
||||
// The wrapper gained a source; its message must not have moved, or log
|
||||
// scrapers and any message-matching caller would break (backlog#1827 T2).
|
||||
// `Error::other` renders through `StorageError::Io`, which prefixes
|
||||
// "Io error: " — that was true of the `format!` this replaced too, so
|
||||
// the full string is what must stay stable.
|
||||
let err = data_movement_stage_error("rebalance_object", "put_object", "bucket-a", "object-a", Error::SlowDown);
|
||||
assert_eq!(
|
||||
err.to_string(),
|
||||
format!("Io error: rebalance_object: put_object failed for bucket-a/object-a: {}", Error::SlowDown)
|
||||
);
|
||||
assert_eq!(
|
||||
err.to_string(),
|
||||
Error::other(format!("rebalance_object: put_object failed for bucket-a/object-a: {}", Error::SlowDown)).to_string()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage_error_keeps_the_wrapped_error_recoverable() {
|
||||
for original in [Error::DiskFull, Error::StorageFull, Error::FileNotFound, Error::SlowDown] {
|
||||
let wrapped =
|
||||
data_movement_stage_error("decommission_object", "put_object", "bucket-a", "object-a", original.clone());
|
||||
let recovered = data_movement_stage_source(&wrapped).expect("the wrapped error must be recoverable");
|
||||
assert_eq!(recovered.to_string(), original.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage_source_ignores_errors_it_did_not_wrap() {
|
||||
assert!(data_movement_stage_source(&Error::DiskFull).is_none());
|
||||
assert!(data_movement_stage_source(&Error::other("plain io error")).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_movement_part_stage_error_includes_stage_object_and_part() {
|
||||
let err =
|
||||
|
||||
@@ -23,6 +23,7 @@ use std::{
|
||||
io,
|
||||
path::{Component, Path, PathBuf},
|
||||
sync::{Arc, LazyLock, Weak},
|
||||
time::Instant,
|
||||
};
|
||||
use tokio::fs;
|
||||
use tokio::sync::{
|
||||
@@ -325,6 +326,8 @@ pub async fn fsync_dir(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
|
||||
const ENV_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: &str = "RUSTFS_EXPERIMENTAL_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE";
|
||||
const DEFAULT_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE: bool = false;
|
||||
const ENV_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: &str = "RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE";
|
||||
const DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE: bool = false;
|
||||
#[cfg(not(test))]
|
||||
const MAX_DST_DIR_FSYNC_GROUPS: usize = 1024;
|
||||
#[cfg(test)]
|
||||
@@ -333,9 +336,24 @@ const MAX_DST_DIR_FSYNC_GROUPS: usize = 4;
|
||||
const MAX_DST_DIR_FSYNC_WAITERS: usize = 8192;
|
||||
#[cfg(test)]
|
||||
const MAX_DST_DIR_FSYNC_WAITERS: usize = 8;
|
||||
#[cfg(not(test))]
|
||||
const MAX_FILE_FDATASYNC_GROUPS: usize = 1024;
|
||||
#[cfg(test)]
|
||||
const MAX_FILE_FDATASYNC_GROUPS: usize = 4;
|
||||
#[cfg(not(test))]
|
||||
const MAX_FILE_FDATASYNC_WAITERS: usize = 8192;
|
||||
#[cfg(test)]
|
||||
const MAX_FILE_FDATASYNC_WAITERS: usize = 8;
|
||||
#[cfg(not(test))]
|
||||
const MAX_FILE_FDATASYNC_BATCH_FILES: usize = 1024;
|
||||
#[cfg(test)]
|
||||
const MAX_FILE_FDATASYNC_BATCH_FILES: usize = 8;
|
||||
static DST_DIR_FSYNC_GROUP_COMMIT_ENABLED: LazyLock<bool> = LazyLock::new(|| {
|
||||
rustfs_utils::get_env_bool(ENV_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE, DEFAULT_DST_DIR_FSYNC_GROUP_COMMIT_ENABLE)
|
||||
});
|
||||
static FILE_FDATASYNC_GROUP_COMMIT_ENABLED: LazyLock<bool> = LazyLock::new(|| {
|
||||
rustfs_utils::get_env_bool(ENV_FILE_FDATASYNC_GROUP_COMMIT_ENABLE, DEFAULT_FILE_FDATASYNC_GROUP_COMMIT_ENABLE)
|
||||
});
|
||||
|
||||
#[cfg(test)]
|
||||
mod dst_dir_fsync_group_commit_override {
|
||||
@@ -379,6 +397,48 @@ fn dst_dir_fsync_group_commit_enabled() -> bool {
|
||||
*DST_DIR_FSYNC_GROUP_COMMIT_ENABLED
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod file_fdatasync_group_commit_override {
|
||||
use std::sync::{Mutex, MutexGuard, PoisonError, RwLock};
|
||||
|
||||
static OVERRIDE: RwLock<Option<bool>> = RwLock::new(None);
|
||||
static SERIAL: Mutex<()> = Mutex::new(());
|
||||
|
||||
pub(crate) fn get() -> Option<bool> {
|
||||
*OVERRIDE.read().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
pub(crate) struct OverrideGuard {
|
||||
_serial: MutexGuard<'static, ()>,
|
||||
}
|
||||
|
||||
impl Drop for OverrideGuard {
|
||||
fn drop(&mut self) {
|
||||
*OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = None;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn set(enabled: bool) -> OverrideGuard {
|
||||
let serial = SERIAL.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
*OVERRIDE.write().unwrap_or_else(PoisonError::into_inner) = Some(enabled);
|
||||
OverrideGuard { _serial: serial }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_file_fdatasync_group_commit_for_test(enabled: bool) -> file_fdatasync_group_commit_override::OverrideGuard {
|
||||
file_fdatasync_group_commit_override::set(enabled)
|
||||
}
|
||||
|
||||
fn file_fdatasync_group_commit_enabled() -> bool {
|
||||
#[cfg(test)]
|
||||
if let Some(enabled) = file_fdatasync_group_commit_override::get() {
|
||||
return enabled;
|
||||
}
|
||||
|
||||
*FILE_FDATASYNC_GROUP_COMMIT_ENABLED
|
||||
}
|
||||
|
||||
#[derive(Clone, Eq, Hash, PartialEq)]
|
||||
struct DstDirFsyncGroupKey {
|
||||
canonical_path: PathBuf,
|
||||
@@ -703,6 +763,269 @@ fn clear_dst_dir_fsync_group_commit_for_test() {
|
||||
DST_DIR_FSYNC_GROUP_COMMIT.clear_for_test();
|
||||
}
|
||||
|
||||
type FileFdatasyncGroupKey = usize;
|
||||
|
||||
struct FileFdatasyncWaiter {
|
||||
files: Vec<PathBuf>,
|
||||
enqueued_at: Option<Instant>,
|
||||
wait_role: &'static str,
|
||||
result_tx: oneshot::Sender<SharedFileFdatasyncResult>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct SharedFileFdatasyncError {
|
||||
kind: io::ErrorKind,
|
||||
message: Arc<str>,
|
||||
}
|
||||
|
||||
impl SharedFileFdatasyncError {
|
||||
fn from_error(err: io::Error) -> Self {
|
||||
Self {
|
||||
kind: err.kind(),
|
||||
message: Arc::from(err.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn into_error(self) -> io::Error {
|
||||
io::Error::new(self.kind, self.message.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
type SharedFileFdatasyncResult = std::result::Result<(), SharedFileFdatasyncError>;
|
||||
|
||||
struct FileFdatasyncGroup {
|
||||
key: FileFdatasyncGroupKey,
|
||||
disk_permits: Weak<Semaphore>,
|
||||
inner: Mutex<FileFdatasyncGroupInner>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FileFdatasyncGroupInner {
|
||||
worker_running: bool,
|
||||
pending_files: usize,
|
||||
pending: VecDeque<FileFdatasyncWaiter>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FileFdatasyncGroupCommit {
|
||||
inner: Mutex<FileFdatasyncGroupCommitInner>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FileFdatasyncGroupCommitInner {
|
||||
groups: HashMap<FileFdatasyncGroupKey, Arc<FileFdatasyncGroup>>,
|
||||
total_waiters: usize,
|
||||
total_files: usize,
|
||||
}
|
||||
|
||||
static FILE_FDATASYNC_GROUP_COMMIT: LazyLock<FileFdatasyncGroupCommit> = LazyLock::new(FileFdatasyncGroupCommit::default);
|
||||
|
||||
impl FileFdatasyncGroupCommit {
|
||||
// Lock order: registry first, then per-group state. No path may hold a
|
||||
// group lock while acquiring the registry lock.
|
||||
fn enqueue(
|
||||
&self,
|
||||
disk_permits: Arc<Semaphore>,
|
||||
files: Vec<PathBuf>,
|
||||
) -> io::Result<(oneshot::Receiver<SharedFileFdatasyncResult>, Option<Arc<FileFdatasyncGroup>>)> {
|
||||
if files.is_empty() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"file fdatasync group commit needs at least one file",
|
||||
));
|
||||
}
|
||||
let (result_tx, result_rx) = oneshot::channel();
|
||||
let key = Arc::as_ptr(&disk_permits) as FileFdatasyncGroupKey;
|
||||
let mut registry = self.inner.lock();
|
||||
registry.groups.retain(|_, group| group.disk_permits.strong_count() > 0);
|
||||
if registry.total_waiters >= MAX_FILE_FDATASYNC_WAITERS {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::WouldBlock,
|
||||
"file fdatasync group commit waiter limit reached",
|
||||
));
|
||||
}
|
||||
if registry.total_files.saturating_add(files.len()) > MAX_FILE_FDATASYNC_BATCH_FILES {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::WouldBlock,
|
||||
"file fdatasync group commit file limit reached",
|
||||
));
|
||||
}
|
||||
let group = if let Some(group) = registry.groups.get(&key) {
|
||||
group.clone()
|
||||
} else {
|
||||
if registry.groups.len() >= MAX_FILE_FDATASYNC_GROUPS {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::WouldBlock,
|
||||
"file fdatasync group commit active group limit reached",
|
||||
));
|
||||
}
|
||||
let group = Arc::new(FileFdatasyncGroup {
|
||||
key,
|
||||
disk_permits: Arc::downgrade(&disk_permits),
|
||||
inner: Mutex::new(FileFdatasyncGroupInner::default()),
|
||||
});
|
||||
registry.groups.insert(key, group.clone());
|
||||
group
|
||||
};
|
||||
let file_count = files.len();
|
||||
let mut group_state = group.inner.lock();
|
||||
let start_worker = !group_state.worker_running;
|
||||
let wait_role = if start_worker {
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER
|
||||
} else {
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER
|
||||
};
|
||||
group_state.pending.push_back(FileFdatasyncWaiter {
|
||||
files,
|
||||
enqueued_at: rustfs_io_metrics::put_stage_timer(),
|
||||
wait_role,
|
||||
result_tx,
|
||||
});
|
||||
group_state.pending_files += file_count;
|
||||
if start_worker {
|
||||
group_state.worker_running = true;
|
||||
}
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_group_outstanding(
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_WAITERS,
|
||||
group_state.pending.len(),
|
||||
);
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_group_outstanding(
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_FILES,
|
||||
group_state.pending_files,
|
||||
);
|
||||
registry.total_waiters += 1;
|
||||
registry.total_files += file_count;
|
||||
drop(group_state);
|
||||
drop(registry);
|
||||
Ok((result_rx, start_worker.then_some(group)))
|
||||
}
|
||||
|
||||
fn complete_batch(&self, waiters: usize, files: usize) {
|
||||
let mut registry = self.inner.lock();
|
||||
registry.total_waiters = registry.total_waiters.saturating_sub(waiters);
|
||||
registry.total_files = registry.total_files.saturating_sub(files);
|
||||
}
|
||||
|
||||
fn remove_idle_group(&self, group: &Arc<FileFdatasyncGroup>) {
|
||||
let mut registry = self.inner.lock();
|
||||
let group_state = group.inner.lock();
|
||||
if !group_state.worker_running && group_state.pending.is_empty() {
|
||||
registry.groups.remove(&group.key);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn counts_for_test(&self) -> (usize, usize, usize) {
|
||||
let registry = self.inner.lock();
|
||||
(registry.groups.len(), registry.total_waiters, registry.total_files)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn clear_for_test(&self) {
|
||||
let mut registry = self.inner.lock();
|
||||
registry.groups.clear();
|
||||
registry.total_waiters = 0;
|
||||
registry.total_files = 0;
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_file_fdatasync_group_worker(group: Arc<FileFdatasyncGroup>) {
|
||||
loop {
|
||||
#[cfg(test)]
|
||||
file_sync_probe::run_before_group_batch();
|
||||
tokio::task::yield_now().await;
|
||||
let (batch, batch_file_count): (Vec<FileFdatasyncWaiter>, usize) = {
|
||||
let mut group_state = group.inner.lock();
|
||||
let batch_file_count = group_state.pending_files;
|
||||
group_state.pending_files = 0;
|
||||
(group_state.pending.drain(..).collect(), batch_file_count)
|
||||
};
|
||||
if batch.is_empty() {
|
||||
let mut group_state = group.inner.lock();
|
||||
group_state.worker_running = false;
|
||||
drop(group_state);
|
||||
FILE_FDATASYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
return;
|
||||
}
|
||||
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_group_outstanding(
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_BATCH_WAITERS,
|
||||
batch.len(),
|
||||
);
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_group_outstanding(
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_BATCH_FILES,
|
||||
batch_file_count,
|
||||
);
|
||||
for waiter in &batch {
|
||||
if let Some(enqueued_at) = waiter.enqueued_at {
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_group_wait(
|
||||
waiter.wait_role,
|
||||
enqueued_at.elapsed().as_secs_f64() * 1000.0,
|
||||
);
|
||||
}
|
||||
}
|
||||
let batch_files: Vec<PathBuf> = batch.iter().flat_map(|waiter| waiter.files.iter().cloned()).collect();
|
||||
#[cfg(test)]
|
||||
file_sync_probe::record_group_batch(batch_file_count);
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_batch(
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL,
|
||||
batch_file_count,
|
||||
);
|
||||
let result = if let Some(disk_permits) = group.disk_permits.upgrade() {
|
||||
run_file_sync_blocking(disk_permits, move || sync_files(&batch_files))
|
||||
.await
|
||||
.map_err(SharedFileFdatasyncError::from_error)
|
||||
} else {
|
||||
Err(SharedFileFdatasyncError::from_error(io::Error::other(
|
||||
"file fdatasync group commit limiter dropped",
|
||||
)))
|
||||
};
|
||||
FILE_FDATASYNC_GROUP_COMMIT.complete_batch(batch.len(), batch_file_count);
|
||||
|
||||
let should_stop = {
|
||||
let mut group_state = group.inner.lock();
|
||||
if group_state.pending.is_empty() {
|
||||
group_state.worker_running = false;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
if should_stop {
|
||||
FILE_FDATASYNC_GROUP_COMMIT.remove_idle_group(&group);
|
||||
}
|
||||
for waiter in batch {
|
||||
let _ = waiter.result_tx.send(result.clone());
|
||||
}
|
||||
if should_stop {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn sync_files_group_commit(files: Vec<PathBuf>, disk_permits: Arc<Semaphore>) -> io::Result<()> {
|
||||
let (result_rx, worker) = FILE_FDATASYNC_GROUP_COMMIT.enqueue(disk_permits, files)?;
|
||||
if let Some(group) = worker {
|
||||
tokio::spawn(run_file_fdatasync_group_worker(group));
|
||||
}
|
||||
|
||||
match result_rx.await {
|
||||
Ok(Ok(())) => Ok(()),
|
||||
Ok(Err(err)) => Err(err.into_error()),
|
||||
Err(_) => Err(io::Error::other("file fdatasync group worker dropped the waiter")),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn file_fdatasync_group_commit_counts_for_test() -> (usize, usize, usize) {
|
||||
FILE_FDATASYNC_GROUP_COMMIT.counts_for_test()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn clear_file_fdatasync_group_commit_for_test() {
|
||||
FILE_FDATASYNC_GROUP_COMMIT.clear_for_test();
|
||||
}
|
||||
|
||||
// Small object directories are cheaper to flush in one blocking task. Multipart
|
||||
// directories fan out only once enough files can amortize per-task scheduling.
|
||||
const PARALLEL_FILE_SYNC_THRESHOLD: usize = 16;
|
||||
@@ -880,6 +1203,8 @@ pub(crate) mod file_sync_probe {
|
||||
static ATTEMPTS: AtomicUsize = AtomicUsize::new(0);
|
||||
static FAIL_ON_ATTEMPT: AtomicUsize = AtomicUsize::new(usize::MAX);
|
||||
static BLOCK: AtomicBool = AtomicBool::new(false);
|
||||
static GROUP_BATCHES: Mutex<Vec<usize>> = Mutex::new(Vec::new());
|
||||
static BEFORE_GROUP_BATCH: Mutex<Option<Box<dyn FnOnce() + Send>>> = Mutex::new(None);
|
||||
const WAIT_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
pub(crate) struct ProbeGuard;
|
||||
@@ -905,6 +1230,8 @@ pub(crate) mod file_sync_probe {
|
||||
fn drop(&mut self) {
|
||||
release();
|
||||
FAIL_ON_ATTEMPT.store(usize::MAX, Ordering::SeqCst);
|
||||
GROUP_BATCHES.lock().expect("file sync group batch recorder poisoned").clear();
|
||||
BEFORE_GROUP_BATCH.lock().expect("file sync group batch hook poisoned").take();
|
||||
ROOTS.write().expect("file sync probe lock poisoned").clear();
|
||||
}
|
||||
}
|
||||
@@ -914,6 +1241,8 @@ pub(crate) mod file_sync_probe {
|
||||
PEAK.store(0, Ordering::SeqCst);
|
||||
ATTEMPTS.store(0, Ordering::SeqCst);
|
||||
FAIL_ON_ATTEMPT.store(fail_on_attempt.unwrap_or(usize::MAX), Ordering::SeqCst);
|
||||
GROUP_BATCHES.lock().expect("file sync group batch recorder poisoned").clear();
|
||||
BEFORE_GROUP_BATCH.lock().expect("file sync group batch hook poisoned").take();
|
||||
{
|
||||
let _guard = BLOCK_MUTEX.lock().expect("file sync probe blocker poisoned");
|
||||
BLOCK.store(block, Ordering::SeqCst);
|
||||
@@ -1025,6 +1354,27 @@ pub(crate) mod file_sync_probe {
|
||||
BLOCK.store(false, Ordering::SeqCst);
|
||||
BLOCK_CONDVAR.notify_all();
|
||||
}
|
||||
|
||||
pub(super) fn record_group_batch(batch_len: usize) {
|
||||
GROUP_BATCHES
|
||||
.lock()
|
||||
.expect("file sync group batch recorder poisoned")
|
||||
.push(batch_len);
|
||||
}
|
||||
|
||||
pub(crate) fn group_batches() -> Vec<usize> {
|
||||
GROUP_BATCHES.lock().expect("file sync group batch recorder poisoned").clone()
|
||||
}
|
||||
|
||||
pub(crate) fn set_before_group_batch(hook: impl FnOnce() + Send + 'static) {
|
||||
*BEFORE_GROUP_BATCH.lock().expect("file sync group batch hook poisoned") = Some(Box::new(hook));
|
||||
}
|
||||
|
||||
pub(super) fn run_before_group_batch() {
|
||||
if let Some(hook) = BEFORE_GROUP_BATCH.lock().expect("file sync group batch hook poisoned").take() {
|
||||
hook();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn sync_file(path: &Path) -> io::Result<()> {
|
||||
@@ -1095,9 +1445,17 @@ pub async fn sync_dir_files(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
pub(crate) async fn sync_dir_files_with_limiter(dir: impl AsRef<Path>, disk_permits: Arc<Semaphore>) -> io::Result<()> {
|
||||
let dir = dir.as_ref().to_path_buf();
|
||||
let scan_dir = dir.clone();
|
||||
let group_file_fdatasync = file_fdatasync_group_commit_enabled();
|
||||
let files = run_file_sync_blocking(disk_permits.clone(), move || {
|
||||
let files = regular_files(&scan_dir)?;
|
||||
if files.len() < PARALLEL_FILE_SYNC_THRESHOLD {
|
||||
if group_file_fdatasync && !files.is_empty() {
|
||||
return Ok(Some(files));
|
||||
}
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_batch(
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL,
|
||||
files.len(),
|
||||
);
|
||||
sync_files(&files)?;
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = fsync_dir_std(scan_dir);
|
||||
@@ -1115,6 +1473,23 @@ pub(crate) async fn sync_dir_files_with_limiter(dir: impl AsRef<Path>, disk_perm
|
||||
let Some(files) = files else {
|
||||
return Ok(());
|
||||
};
|
||||
if group_file_fdatasync && files.len() < PARALLEL_FILE_SYNC_THRESHOLD {
|
||||
sync_files_group_commit(files, disk_permits.clone()).await?;
|
||||
return run_file_sync_blocking(disk_permits, move || {
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = fsync_dir_std(dir);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_SRC_DIR_FSYNC,
|
||||
fsync_started,
|
||||
);
|
||||
result
|
||||
})
|
||||
.await;
|
||||
}
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_batch(
|
||||
rustfs_io_metrics::PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL,
|
||||
files.len(),
|
||||
);
|
||||
futures::stream::iter(files.into_iter().map(Ok::<_, io::Error>))
|
||||
.try_for_each_concurrent(MAX_PARALLEL_FILE_SYNCS, |path| {
|
||||
let disk_permits = disk_permits.clone();
|
||||
@@ -5667,6 +6042,247 @@ mod tests {
|
||||
.expect("sequential file sync must succeed");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
async fn file_fdatasync_group_commit_default_off_keeps_small_directory_serial() {
|
||||
let _group_commit = set_file_fdatasync_group_commit_for_test(false);
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
std::fs::write(temp_dir.path().join("part.1"), b"shard").expect("write part");
|
||||
let _probe = file_sync_probe::set_blocking(temp_dir.path());
|
||||
let path = temp_dir.path().to_path_buf();
|
||||
let task = tokio::spawn(async move { sync_dir_files_with_limiter(path, file_sync_limiter()).await });
|
||||
file_sync_probe::wait_for_active(1).await;
|
||||
|
||||
assert_eq!(
|
||||
file_sync_probe::group_batches(),
|
||||
Vec::<usize>::new(),
|
||||
"default-off small directory sync must not enter the file fdatasync group coordinator"
|
||||
);
|
||||
file_sync_probe::release();
|
||||
task.await
|
||||
.expect("join default-off file sync")
|
||||
.expect("default-off file sync must succeed");
|
||||
assert!(
|
||||
fsync_dir_recorder::was_fsynced(temp_dir.path()),
|
||||
"default-off successful sync must fsync the source directory"
|
||||
);
|
||||
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
async fn file_fdatasync_group_commit_batches_same_disk_small_directories() {
|
||||
use std::sync::mpsc;
|
||||
|
||||
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let first_dir = temp_dir.path().join("first");
|
||||
let second_dir = temp_dir.path().join("second");
|
||||
std::fs::create_dir(&first_dir).expect("create first dir");
|
||||
std::fs::create_dir(&second_dir).expect("create second dir");
|
||||
std::fs::write(first_dir.join("part.1"), b"first").expect("write first part");
|
||||
std::fs::write(second_dir.join("part.1"), b"second").expect("write second part");
|
||||
let _probe = file_sync_probe::set_blocking(temp_dir.path());
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_batch_tx, release_batch_rx) = mpsc::channel();
|
||||
file_sync_probe::set_before_group_batch(move || {
|
||||
entered_tx.send(()).expect("signal first file fdatasync group worker");
|
||||
release_batch_rx.recv().expect("wait until second waiter is queued");
|
||||
});
|
||||
|
||||
let limiter = file_sync_limiter();
|
||||
let first_limiter = limiter.clone();
|
||||
let first_path = first_dir.clone();
|
||||
let first = tokio::spawn(async move { sync_dir_files_with_limiter(first_path, first_limiter).await });
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("group worker hook waiter should run")
|
||||
.expect("first file fdatasync group worker should start");
|
||||
|
||||
let second_limiter = limiter.clone();
|
||||
let second_path = second_dir.clone();
|
||||
let second = tokio::spawn(async move { sync_dir_files_with_limiter(second_path, second_limiter).await });
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
if file_fdatasync_group_commit_counts_for_test().1 == 2 {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second waiter should enqueue before releasing the grouped batch");
|
||||
release_batch_tx.send(()).expect("release group batch hook");
|
||||
file_sync_probe::wait_for_active(1).await;
|
||||
|
||||
assert_eq!(
|
||||
file_sync_probe::group_batches(),
|
||||
vec![2],
|
||||
"same-disk small directory fdatasync waiters must share one observable batch"
|
||||
);
|
||||
file_sync_probe::release();
|
||||
first
|
||||
.await
|
||||
.expect("join first grouped file sync")
|
||||
.expect("first grouped file sync must succeed");
|
||||
second
|
||||
.await
|
||||
.expect("join second grouped file sync")
|
||||
.expect("second grouped file sync must succeed");
|
||||
assert!(
|
||||
fsync_dir_recorder::was_fsynced(&first_dir),
|
||||
"first source directory must still be fsynced"
|
||||
);
|
||||
assert!(
|
||||
fsync_dir_recorder::was_fsynced(&second_dir),
|
||||
"second source directory must still be fsynced"
|
||||
);
|
||||
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
async fn file_fdatasync_group_commit_failure_fails_all_waiters_before_dir_fsync() {
|
||||
use std::sync::mpsc;
|
||||
|
||||
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let first_dir = temp_dir.path().join("first");
|
||||
let second_dir = temp_dir.path().join("second");
|
||||
std::fs::create_dir(&first_dir).expect("create first dir");
|
||||
std::fs::create_dir(&second_dir).expect("create second dir");
|
||||
std::fs::write(first_dir.join("part.1"), b"first").expect("write first part");
|
||||
std::fs::write(second_dir.join("part.1"), b"second").expect("write second part");
|
||||
let _probe = file_sync_probe::set_failing_blocking(temp_dir.path());
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (release_batch_tx, release_batch_rx) = mpsc::channel();
|
||||
file_sync_probe::set_before_group_batch(move || {
|
||||
entered_tx.send(()).expect("signal first file fdatasync group worker");
|
||||
release_batch_rx.recv().expect("wait until second waiter is queued");
|
||||
});
|
||||
|
||||
let limiter = file_sync_limiter();
|
||||
let first_limiter = limiter.clone();
|
||||
let first_path = first_dir.clone();
|
||||
let first = tokio::spawn(async move { sync_dir_files_with_limiter(first_path, first_limiter).await });
|
||||
tokio::task::spawn_blocking(move || entered_rx.recv_timeout(Duration::from_secs(30)))
|
||||
.await
|
||||
.expect("group worker hook waiter should run")
|
||||
.expect("first file fdatasync group worker should start");
|
||||
|
||||
let second_limiter = limiter.clone();
|
||||
let second_path = second_dir.clone();
|
||||
let second = tokio::spawn(async move { sync_dir_files_with_limiter(second_path, second_limiter).await });
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
if file_fdatasync_group_commit_counts_for_test().1 == 2 {
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second waiter should enqueue before releasing the grouped batch");
|
||||
release_batch_tx.send(()).expect("release group batch hook");
|
||||
|
||||
let first_err = first
|
||||
.await
|
||||
.expect("join first grouped file sync")
|
||||
.expect_err("first grouped waiter must fail closed");
|
||||
let second_err = second
|
||||
.await
|
||||
.expect("join second grouped file sync")
|
||||
.expect_err("second grouped waiter must fail closed");
|
||||
|
||||
assert_eq!(first_err.kind(), io::ErrorKind::Other);
|
||||
assert_eq!(second_err.kind(), io::ErrorKind::Other);
|
||||
assert_eq!(file_sync_probe::group_batches(), vec![2]);
|
||||
assert!(
|
||||
!fsync_dir_recorder::was_fsynced(&first_dir) && !fsync_dir_recorder::was_fsynced(&second_dir),
|
||||
"source directories must not be fsynced after grouped file fdatasync failure"
|
||||
);
|
||||
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||
file_sync_probe::release();
|
||||
file_sync_probe::wait_for_idle().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
fn file_fdatasync_group_commit_rejects_active_group_overflow() {
|
||||
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
let mut receivers = Vec::new();
|
||||
let mut limiters = Vec::new();
|
||||
for index in 0..MAX_FILE_FDATASYNC_GROUPS {
|
||||
let limiter = Arc::new(Semaphore::new(1));
|
||||
let (result_rx, _worker) = FILE_FDATASYNC_GROUP_COMMIT
|
||||
.enqueue(limiter.clone(), vec![PathBuf::from(format!("part-{index}"))])
|
||||
.expect("group below cap should enqueue");
|
||||
limiters.push(limiter);
|
||||
receivers.push(result_rx);
|
||||
}
|
||||
|
||||
let overflow_limiter = Arc::new(Semaphore::new(1));
|
||||
let err = match FILE_FDATASYNC_GROUP_COMMIT.enqueue(overflow_limiter, vec![PathBuf::from("overflow")]) {
|
||||
Ok(_) => panic!("active group max+1 must fail closed"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::WouldBlock);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||
drop(receivers);
|
||||
drop(limiters);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
fn file_fdatasync_group_commit_rejects_waiter_and_file_overflow() {
|
||||
let _group_commit = set_file_fdatasync_group_commit_for_test(true);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
let limiter = Arc::new(Semaphore::new(1));
|
||||
let mut receivers = Vec::new();
|
||||
for index in 0..MAX_FILE_FDATASYNC_WAITERS {
|
||||
let (result_rx, _worker) = FILE_FDATASYNC_GROUP_COMMIT
|
||||
.enqueue(limiter.clone(), vec![PathBuf::from(format!("part-{index}"))])
|
||||
.expect("waiter below cap should enqueue");
|
||||
receivers.push(result_rx);
|
||||
}
|
||||
|
||||
let waiter_err = match FILE_FDATASYNC_GROUP_COMMIT.enqueue(limiter, vec![PathBuf::from("overflow-waiter")]) {
|
||||
Ok(_) => panic!("waiter max+1 must fail closed"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(waiter_err.kind(), io::ErrorKind::WouldBlock);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
drop(receivers);
|
||||
|
||||
let mut receivers = Vec::new();
|
||||
let file_limit_limiter = Arc::new(Semaphore::new(1));
|
||||
let (result_rx, _worker) = FILE_FDATASYNC_GROUP_COMMIT
|
||||
.enqueue(
|
||||
file_limit_limiter.clone(),
|
||||
(0..MAX_FILE_FDATASYNC_BATCH_FILES)
|
||||
.map(|index| PathBuf::from(format!("part-{index}")))
|
||||
.collect(),
|
||||
)
|
||||
.expect("file count up to cap should enqueue");
|
||||
receivers.push(result_rx);
|
||||
|
||||
let file_err = match FILE_FDATASYNC_GROUP_COMMIT.enqueue(file_limit_limiter, vec![PathBuf::from("overflow-file")]) {
|
||||
Ok(_) => panic!("file max+1 must fail closed"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert_eq!(file_err.kind(), io::ErrorKind::WouldBlock);
|
||||
clear_file_fdatasync_group_commit_for_test();
|
||||
assert_eq!(file_fdatasync_group_commit_counts_for_test(), (0, 0, 0));
|
||||
drop(receivers);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(file_sync_probe)]
|
||||
async fn sync_dir_files_bounds_concurrency_across_directories() {
|
||||
|
||||
@@ -218,6 +218,63 @@ pub struct QuotaAdmission {
|
||||
quota_limit: u64,
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct LifecycleDeleteAllRequest {
|
||||
pub(crate) version_id: Option<Uuid>,
|
||||
pub(crate) delete_marker: bool,
|
||||
pub(crate) action: rustfs_common::metrics::IlmAction,
|
||||
pub(crate) rule_id: String,
|
||||
pub(crate) phase: LifecycleDeleteAllPhase,
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum LifecycleDeleteAllPhase {
|
||||
Preflight,
|
||||
History,
|
||||
FinalPreflight,
|
||||
Trigger,
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
#[derive(Default)]
|
||||
pub struct LifecycleDeleteAllJournalState {
|
||||
prepared: HashMap<String, crate::bucket::lifecycle::tier_sweeper::Jentry>,
|
||||
mutation_started: bool,
|
||||
}
|
||||
|
||||
impl Debug for LifecycleDeleteAllJournalState {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("LifecycleDeleteAllJournalState")
|
||||
.field("prepared_count", &self.prepared.len())
|
||||
.field("mutation_started", &self.mutation_started)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl LifecycleDeleteAllJournalState {
|
||||
pub(crate) fn contains(&self, name: &str) -> bool {
|
||||
self.prepared.contains_key(name)
|
||||
}
|
||||
|
||||
pub(crate) fn insert(&mut self, name: String, entry: crate::bucket::lifecycle::tier_sweeper::Jentry) {
|
||||
self.prepared.insert(name, entry);
|
||||
}
|
||||
|
||||
pub(crate) fn prepared_entries(&self) -> Vec<crate::bucket::lifecycle::tier_sweeper::Jentry> {
|
||||
self.prepared.values().cloned().collect()
|
||||
}
|
||||
|
||||
pub(crate) fn mark_mutation_started(&mut self) {
|
||||
self.mutation_started = true;
|
||||
}
|
||||
|
||||
pub(crate) fn mutation_started(&self) -> bool {
|
||||
self.mutation_started
|
||||
}
|
||||
}
|
||||
|
||||
impl QuotaAdmission {
|
||||
pub(crate) fn current_usage(self) -> u64 {
|
||||
self.current_usage
|
||||
@@ -242,6 +299,11 @@ pub struct ObjectOptions {
|
||||
pub delete_prefix: bool,
|
||||
pub delete_prefix_object: bool,
|
||||
pub version_id: Option<String>,
|
||||
/// Lifecycle-only staged purge request checked under the object write lock.
|
||||
#[doc(hidden)]
|
||||
pub lifecycle_delete_all: Option<LifecycleDeleteAllRequest>,
|
||||
#[doc(hidden)]
|
||||
pub lifecycle_delete_all_journal: Option<Arc<parking_lot::Mutex<LifecycleDeleteAllJournalState>>>,
|
||||
/// RustFS-only compare-and-set condition checked under the object write lock.
|
||||
pub expected_current_version_id: Option<String>,
|
||||
/// Persisted bucket incarnation observed before authorization.
|
||||
@@ -349,6 +411,15 @@ impl ObjectOptions {
|
||||
self.namespace_lock_fence.get_or_insert_with(NamespaceLockFence::new);
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_lifecycle_delete_all_journal(&mut self) {
|
||||
self.lifecycle_delete_all_journal
|
||||
.get_or_insert_with(|| Arc::new(parking_lot::Mutex::new(LifecycleDeleteAllJournalState::default())));
|
||||
}
|
||||
|
||||
pub(crate) fn lifecycle_delete_all_journal(&self) -> Option<&Arc<parking_lot::Mutex<LifecycleDeleteAllJournalState>>> {
|
||||
self.lifecycle_delete_all_journal.as_ref()
|
||||
}
|
||||
|
||||
pub fn add_namespace_lock_guard(&mut self, guard: &rustfs_lock::NamespaceLockGuard) {
|
||||
if let Some(signal) = guard.lock_lost_signal() {
|
||||
self.add_namespace_lock_lost_signal(signal);
|
||||
@@ -648,14 +719,23 @@ impl ObjectInfo {
|
||||
}
|
||||
|
||||
pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
|
||||
let name = decode_dir_object(object);
|
||||
|
||||
let mut version_id = fi.version_id;
|
||||
|
||||
if versioned && version_id.is_none() {
|
||||
version_id = Some(Uuid::nil())
|
||||
}
|
||||
|
||||
Self::from_file_info_with_version_id(fi, bucket, object, version_id)
|
||||
}
|
||||
|
||||
pub(crate) fn from_file_info_with_version_id(
|
||||
fi: &FileInfo,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<Uuid>,
|
||||
) -> ObjectInfo {
|
||||
let name = decode_dir_object(object);
|
||||
|
||||
// etag
|
||||
let (content_type, content_encoding, etag) = {
|
||||
let content_type = fi.metadata.get("content-type").cloned();
|
||||
@@ -1569,6 +1649,18 @@ mod tests {
|
||||
assert_eq!(info.replication_decision, "arn=true;false;arn:replication::1:dest;rule-id");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_file_info_with_version_id_keeps_normalized_absent_version() {
|
||||
let fi = FileInfo {
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let info = ObjectInfo::from_file_info_with_version_id(&fi, "bucket", "object", None);
|
||||
|
||||
assert_eq!(info.version_id, None, "a normalized absent version must not be rewritten to nil");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_file_info_reports_effective_storage_class_for_legacy_metadata() {
|
||||
for legacy_label in [
|
||||
@@ -1890,4 +1982,13 @@ mod tests {
|
||||
assert!(default_cloned.user_tags.is_empty());
|
||||
assert!(default_cloned.parts.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_options_default_does_not_allocate_lifecycle_delete_all_journal() {
|
||||
let mut opts = ObjectOptions::default();
|
||||
|
||||
assert!(opts.lifecycle_delete_all_journal().is_none());
|
||||
opts.ensure_lifecycle_delete_all_journal();
|
||||
assert!(opts.lifecycle_delete_all_journal().is_some());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -573,9 +573,7 @@ pub(crate) async fn initialize_local_disk_maps(
|
||||
pub(crate) async fn init_tier_config_mgr(store: Arc<ECStore>) -> Result<()> {
|
||||
let handle = get_global_tier_config_mgr();
|
||||
TierConfigMgr::reload_handle(&handle, store.clone()).await?;
|
||||
if setup_is_dist_erasure().await {
|
||||
tokio::spawn(TierConfigMgr::refresh_tier_config_handle(handle, store));
|
||||
}
|
||||
tokio::spawn(TierConfigMgr::refresh_tier_config_handle(handle, store));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -657,7 +657,7 @@ where
|
||||
prefix,
|
||||
marker,
|
||||
None,
|
||||
i32::try_from(limit).map_or(i32::MAX, |value| value),
|
||||
i32::try_from(limit).unwrap_or(i32::MAX),
|
||||
false,
|
||||
None,
|
||||
false,
|
||||
|
||||
@@ -17,7 +17,7 @@ use std::sync::Arc;
|
||||
use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::tier::{TierConfigMgr, tier_config_etag_matches};
|
||||
use super::tier::{TierConfigMgr, tier_config_abort_matches, tier_config_commit_matches, tier_config_etag_matches};
|
||||
use super::tier_mutation_intent::{
|
||||
MAX_TIER_MUTATION_INTENT_SIZE, TierMutationIntent, TierMutationIntentState, advance_tier_mutation_intent_record_idempotent,
|
||||
load_tier_mutation_intent_record, save_tier_mutation_intent_record_if_absent,
|
||||
@@ -53,6 +53,10 @@ pub enum TierMutationPeerError {
|
||||
InvalidPayload(String),
|
||||
#[error("tier mutation peer intent conflicts with existing record")]
|
||||
ConflictingIntent,
|
||||
#[error("tier mutation peer commit proof does not match the persisted tier configuration")]
|
||||
CommitProofMismatch,
|
||||
#[error("tier mutation peer abort proof does not match the persisted tier configuration")]
|
||||
AbortProofMismatch,
|
||||
#[error("tier mutation peer runtime error: {0}")]
|
||||
Runtime(#[source] AdminError),
|
||||
#[error("tier mutation peer store error: {0}")]
|
||||
@@ -120,8 +124,13 @@ async fn handle_prepare(
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
}
|
||||
TierMutationIntentState::Committed | TierMutationIntentState::Aborted => {
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id).await;
|
||||
TierMutationIntentState::Committed => {
|
||||
TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &existing)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
}
|
||||
TierMutationIntentState::Aborted => {
|
||||
TierConfigMgr::request_committed_mutation_refresh(&tier_config_mgr).await;
|
||||
}
|
||||
}
|
||||
Ok(TierMutationPeerOutcome {
|
||||
@@ -140,6 +149,18 @@ async fn handle_commit(
|
||||
) -> TierMutationPeerResult<TierMutationPeerOutcome> {
|
||||
let committed_config_etag = parse_commit_etag(canonical_payload)?;
|
||||
let tier_config_mgr = api.tier_config_mgr();
|
||||
match load_tier_mutation_intent_record(api.clone(), mutation_id).await {
|
||||
Ok(intent) if intent.state == TierMutationIntentState::Prepared => {
|
||||
let proof_matches = tier_config_commit_matches(api.clone(), &committed_config_etag, intent.candidate_digest)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
if !proof_matches {
|
||||
return Err(TierMutationPeerError::CommitProofMismatch);
|
||||
}
|
||||
}
|
||||
Ok(_) | Err(Error::ConfigNotFound) => {}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
let (intent, applied) = match advance_tier_mutation_intent_record_idempotent(
|
||||
api.clone(),
|
||||
mutation_id,
|
||||
@@ -154,6 +175,9 @@ async fn handle_commit(
|
||||
.await
|
||||
.map_err(Error::other)? =>
|
||||
{
|
||||
TierConfigMgr::promote_prepared_mutation_intent_block(&tier_config_mgr, mutation_id)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
return Ok(TierMutationPeerOutcome {
|
||||
state: TierMutationPeerState::Committed,
|
||||
applied: false,
|
||||
@@ -162,7 +186,9 @@ async fn handle_commit(
|
||||
Err(err) => return Err(err.into()),
|
||||
};
|
||||
if intent.state == TierMutationIntentState::Committed {
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id).await;
|
||||
TierConfigMgr::apply_committed_mutation_intent_block(&tier_config_mgr, &intent)
|
||||
.await
|
||||
.map_err(TierMutationPeerError::Runtime)?;
|
||||
}
|
||||
Ok(TierMutationPeerOutcome {
|
||||
state: peer_state_from_intent(intent.state),
|
||||
@@ -178,11 +204,18 @@ async fn handle_abort(
|
||||
if !canonical_payload.is_empty() {
|
||||
return Err(TierMutationPeerError::InvalidPayload("abort payload must be empty".to_string()));
|
||||
}
|
||||
let tier_config_mgr = api.tier_config_mgr();
|
||||
let existing = load_tier_mutation_intent_record(api.clone(), mutation_id).await?;
|
||||
if existing.state == TierMutationIntentState::Prepared
|
||||
&& !tier_config_abort_matches(api.clone(), &existing)
|
||||
.await
|
||||
.map_err(Error::other)?
|
||||
{
|
||||
return Err(TierMutationPeerError::AbortProofMismatch);
|
||||
}
|
||||
let (intent, applied) =
|
||||
advance_tier_mutation_intent_record_idempotent(api, mutation_id, TierMutationIntentState::Aborted, None).await?;
|
||||
advance_tier_mutation_intent_record_idempotent(api.clone(), mutation_id, TierMutationIntentState::Aborted, None).await?;
|
||||
if intent.state == TierMutationIntentState::Aborted {
|
||||
TierConfigMgr::clear_prepared_mutation_intent_block(&tier_config_mgr, mutation_id).await;
|
||||
TierConfigMgr::request_committed_mutation_refresh(&api.tier_config_mgr()).await;
|
||||
}
|
||||
Ok(TierMutationPeerOutcome {
|
||||
state: peer_state_from_intent(intent.state),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -272,6 +272,7 @@ const MULTIPART_WRITE_QUORUM_RENAME_PART: &str = "rename_part";
|
||||
const EVENT_SET_DISK_WRITE: &str = "set_disk_write";
|
||||
const EVENT_SET_DISK_HEAL: &str = "set_disk_heal";
|
||||
const EVENT_SET_DISK_COMMIT_TAIL_SLOW: &str = "set_disk_commit_tail_slow";
|
||||
const EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED: &str = "set_disk_rename_tail_drain_failed";
|
||||
const EVENT_SET_DISK_PUT_OBJECT_STAGE_SUMMARY: &str = "set_disk_put_object_stage_summary";
|
||||
const SET_DISK_COMMIT_TAIL_WARN_THRESHOLD_MS: u128 = 5_000;
|
||||
const ENV_RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES: &str = "RUSTFS_PUT_LARGE_BATCH_MIN_SIZE_BYTES";
|
||||
@@ -1500,6 +1501,102 @@ pub fn get_lock_acquire_timeout() -> Duration {
|
||||
}
|
||||
}
|
||||
|
||||
fn get_put_object_commit_lock_acquire_timeout_override_ms() -> u64 {
|
||||
#[cfg(test)]
|
||||
{
|
||||
rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
|
||||
rustfs_config::DEFAULT_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
|
||||
)
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
static CACHED: OnceLock<u64> = OnceLock::new();
|
||||
*CACHED.get_or_init(|| {
|
||||
rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
|
||||
rustfs_config::DEFAULT_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS,
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn get_put_object_commit_lock_acquire_timeout(op: &'static str) -> Duration {
|
||||
let default_timeout = get_lock_acquire_timeout();
|
||||
if op != "put_object_commit" {
|
||||
return default_timeout;
|
||||
}
|
||||
|
||||
let timeout_ms = get_put_object_commit_lock_acquire_timeout_override_ms();
|
||||
if timeout_ms == 0 {
|
||||
default_timeout
|
||||
} else {
|
||||
Duration::from_millis(timeout_ms)
|
||||
}
|
||||
}
|
||||
|
||||
fn put_object_commit_lock_timeout_override_enabled(op: &'static str) -> bool {
|
||||
op == "put_object_commit" && get_put_object_commit_lock_acquire_timeout_override_ms() != 0
|
||||
}
|
||||
|
||||
fn put_object_commit_lock_admission_budget_label() -> &'static str {
|
||||
match get_put_object_commit_lock_acquire_timeout_override_ms() {
|
||||
0 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED,
|
||||
1..=250 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
|
||||
251..=500 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS,
|
||||
501..=1000 => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS,
|
||||
_ => rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_GT_1000MS,
|
||||
}
|
||||
}
|
||||
|
||||
fn record_put_object_commit_lock_admission(op: &'static str, outcome: &'static str) {
|
||||
if op != "put_object_commit" || !rustfs_io_metrics::put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_commit_lock_admission(put_object_commit_lock_admission_budget_label(), outcome);
|
||||
}
|
||||
|
||||
fn put_object_commit_lock_acquire_error_outcome(op: &'static str, err: &rustfs_lock::error::LockError) -> &'static str {
|
||||
if put_object_commit_lock_timeout_override_enabled(op) && matches!(err, rustfs_lock::error::LockError::Timeout { .. }) {
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN
|
||||
} else {
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_put_object_commit_lock_acquire_result(
|
||||
set: &SetDisks,
|
||||
op: &'static str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
result: std::result::Result<rustfs_lock::namespace::NamespaceLockGuard, rustfs_lock::error::LockError>,
|
||||
) -> Result<rustfs_lock::namespace::NamespaceLockGuard> {
|
||||
match result {
|
||||
Ok(guard) => {
|
||||
record_put_object_commit_lock_admission(op, rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED);
|
||||
Ok(guard)
|
||||
}
|
||||
Err(err) => {
|
||||
record_put_object_commit_lock_admission(op, put_object_commit_lock_acquire_error_outcome(op, &err));
|
||||
Err(map_put_object_commit_lock_acquire_error(set, op, bucket, object, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn map_put_object_commit_lock_acquire_error(
|
||||
set: &SetDisks,
|
||||
op: &'static str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
err: rustfs_lock::error::LockError,
|
||||
) -> StorageError {
|
||||
if put_object_commit_lock_timeout_override_enabled(op) && matches!(err, rustfs_lock::error::LockError::Timeout { .. }) {
|
||||
StorageError::SlowDown
|
||||
} else {
|
||||
set.map_namespace_lock_error(bucket, object, "write", err)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_object_lock_diag_enabled() -> bool {
|
||||
*OBJECT_LOCK_DIAG_ENABLED.get_or_init(|| {
|
||||
let enabled = rustfs_utils::get_env_bool(
|
||||
@@ -3263,6 +3360,17 @@ impl SetDisks {
|
||||
self.get_object_metadata_cache.invalidate_all();
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn record_put_object_commit_namespace_lock_wait(op: &'static str, acquire_start: Instant) {
|
||||
if op != "put_object_commit" || !rustfs_io_metrics::put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT,
|
||||
Some(acquire_start),
|
||||
);
|
||||
}
|
||||
|
||||
async fn acquire_read_lock_diag(&self, op: &'static str, bucket: &str, object: &str) -> Result<ObjectLockDiagGuard> {
|
||||
crate::hp_guard!("SetDisks::acquire_read_lock");
|
||||
let diag_enabled = is_object_lock_diag_enabled();
|
||||
@@ -3290,10 +3398,15 @@ impl SetDisks {
|
||||
let diag_enabled = is_object_lock_diag_enabled();
|
||||
let ns_lock = self.new_ns_lock(bucket, object).await?;
|
||||
let acquire_start = Instant::now();
|
||||
let guard = ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?;
|
||||
let acquire_timeout = get_put_object_commit_lock_acquire_timeout(op);
|
||||
let guard = resolve_put_object_commit_lock_acquire_result(
|
||||
self,
|
||||
op,
|
||||
bucket,
|
||||
object,
|
||||
ns_lock.get_write_lock(acquire_timeout).await,
|
||||
)?;
|
||||
Self::record_put_object_commit_namespace_lock_wait(op, acquire_start);
|
||||
let owner = diag_enabled.then(|| ns_lock.owner().to_string());
|
||||
self.log_object_lock_acquire_if_slow(
|
||||
op,
|
||||
@@ -3327,20 +3440,27 @@ impl SetDisks {
|
||||
let diag_enabled = is_object_lock_diag_enabled();
|
||||
let ns_lock = self.new_ns_lock(bucket, object).await?;
|
||||
let acquire_start = Instant::now();
|
||||
let acquire = ns_lock.get_write_lock(get_lock_acquire_timeout());
|
||||
let acquire_timeout = get_put_object_commit_lock_acquire_timeout(op);
|
||||
let acquire = ns_lock.get_write_lock(acquire_timeout);
|
||||
tokio::pin!(acquire);
|
||||
let mut on_pending = Some(on_pending);
|
||||
let guard = futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
|
||||
std::task::Poll::Pending => {
|
||||
if let Some(on_pending) = on_pending.take() {
|
||||
on_pending();
|
||||
let guard = resolve_put_object_commit_lock_acquire_result(
|
||||
self,
|
||||
op,
|
||||
bucket,
|
||||
object,
|
||||
futures::future::poll_fn(|cx| match std::future::Future::poll(acquire.as_mut(), cx) {
|
||||
std::task::Poll::Pending => {
|
||||
if let Some(on_pending) = on_pending.take() {
|
||||
on_pending();
|
||||
}
|
||||
std::task::Poll::Pending
|
||||
}
|
||||
std::task::Poll::Pending
|
||||
}
|
||||
std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
|
||||
})
|
||||
.await
|
||||
.map_err(|e| self.map_namespace_lock_error(bucket, object, "write", e))?;
|
||||
std::task::Poll::Ready(result) => std::task::Poll::Ready(result),
|
||||
})
|
||||
.await,
|
||||
)?;
|
||||
Self::record_put_object_commit_namespace_lock_wait(op, acquire_start);
|
||||
let owner = diag_enabled.then(|| ns_lock.owner().to_string());
|
||||
self.log_object_lock_acquire_if_slow(
|
||||
op,
|
||||
@@ -4578,10 +4698,15 @@ impl SetDisks {
|
||||
)?;
|
||||
let fi = build_tiered_decommission_file_info(bucket, object, fi, layout);
|
||||
let write_quorum = layout.write_quorum;
|
||||
if opts
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
if _lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| opts
|
||||
.namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
|| opts
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
|| bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
@@ -5454,6 +5579,7 @@ mod tests {
|
||||
};
|
||||
use crate::store::init_format::save_format_file;
|
||||
use crate::store::list_objects::ListPathOptions;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use rustfs_filemeta::ErasureInfo;
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_filemeta::MetaCacheEntry;
|
||||
@@ -5689,6 +5815,448 @@ mod tests {
|
||||
assert_eq!(Arc::strong_count(&set.set_lock_namespace), before);
|
||||
}
|
||||
|
||||
fn put_object_commit_namespace_lock_wait_sample_count(snapshotter: &metrics_util::debugging::Snapshotter) -> usize {
|
||||
snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.filter(|(composite, _, _, _)| {
|
||||
composite.key().name() == "rustfs_s3_put_object_stage_duration_ms"
|
||||
&& composite.key().labels().any(|label| {
|
||||
label.key() == "stage"
|
||||
&& label.value() == rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT
|
||||
})
|
||||
})
|
||||
.map(|(_, _, _, value)| match value {
|
||||
DebugValue::Histogram(samples) => samples.len(),
|
||||
_ => 0,
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
fn put_object_commit_lock_admission_count(
|
||||
rows: &[(
|
||||
metrics_util::CompositeKey,
|
||||
Option<metrics::Unit>,
|
||||
Option<metrics::SharedString>,
|
||||
DebugValue,
|
||||
)],
|
||||
budget: &'static str,
|
||||
outcome: &'static str,
|
||||
) -> u64 {
|
||||
rows.iter()
|
||||
.filter(|(composite, _, _, _)| {
|
||||
composite.key().name() == "rustfs_s3_put_object_commit_namespace_lock_admission_total"
|
||||
&& composite
|
||||
.key()
|
||||
.labels()
|
||||
.any(|label| label.key() == "budget" && label.value() == budget)
|
||||
&& composite
|
||||
.key()
|
||||
.labels()
|
||||
.any(|label| label.key() == "outcome" && label.value() == outcome)
|
||||
})
|
||||
.map(|(_, _, _, value)| match value {
|
||||
DebugValue::Counter(count) => *count,
|
||||
_ => 0,
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_admission_budget_labels_are_bounded() {
|
||||
let cases = [
|
||||
("0", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED),
|
||||
("250", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS),
|
||||
("251", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS),
|
||||
("500", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS),
|
||||
("501", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS),
|
||||
("1000", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS),
|
||||
("1001", rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_GT_1000MS),
|
||||
];
|
||||
for (timeout_ms, expected) in cases {
|
||||
temp_env::with_vars(
|
||||
[(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some(timeout_ms))],
|
||||
|| {
|
||||
assert_eq!(put_object_commit_lock_admission_budget_label(), expected);
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_admission_error_outcomes_are_bounded() {
|
||||
let timeout = LockError::timeout("bucket/object", Duration::from_millis(1));
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
|
||||
assert_eq!(
|
||||
put_object_commit_lock_acquire_error_outcome("put_object_commit", &timeout),
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN
|
||||
);
|
||||
assert_eq!(
|
||||
put_object_commit_lock_acquire_error_outcome("complete_multipart_upload_commit", &timeout),
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR
|
||||
);
|
||||
});
|
||||
|
||||
let internal = LockError::internal("simulated lock manager error");
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
|
||||
assert_eq!(
|
||||
put_object_commit_lock_acquire_error_outcome("put_object_commit", &internal),
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_namespace_lock_wait_metric_is_wired_to_both_write_lock_paths() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should start");
|
||||
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
let bucket = "bucket";
|
||||
let object = "object";
|
||||
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
let guard = set
|
||||
.acquire_write_lock_diag("put_object_commit", bucket, object)
|
||||
.await
|
||||
.expect("disabled metrics acquire should succeed");
|
||||
drop(guard);
|
||||
assert_eq!(put_object_commit_namespace_lock_wait_sample_count(&snapshotter), 0);
|
||||
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
let guard = set
|
||||
.acquire_write_lock_diag("put_object_commit", bucket, object)
|
||||
.await
|
||||
.expect("normal PUT commit acquire should succeed");
|
||||
drop(guard);
|
||||
assert_eq!(put_object_commit_namespace_lock_wait_sample_count(&snapshotter), 1);
|
||||
|
||||
let guard = set
|
||||
.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
|
||||
.await
|
||||
.expect("non-PUT commit acquire should succeed");
|
||||
drop(guard);
|
||||
assert_eq!(put_object_commit_namespace_lock_wait_sample_count(&snapshotter), 0);
|
||||
|
||||
let held_guard = set
|
||||
.acquire_write_lock_diag("put_object_commit", bucket, object)
|
||||
.await
|
||||
.expect("holder acquire should succeed");
|
||||
assert_eq!(put_object_commit_namespace_lock_wait_sample_count(&snapshotter), 1);
|
||||
|
||||
let (pending_tx, pending_rx) = tokio::sync::oneshot::channel();
|
||||
let pending_acquire =
|
||||
set.acquire_write_lock_diag_with_pending_hook("put_object_commit", bucket, object, move || {
|
||||
let _ = pending_tx.send(());
|
||||
});
|
||||
let release_holder = async {
|
||||
pending_rx.await.expect("pending hook should fire");
|
||||
drop(held_guard);
|
||||
};
|
||||
let (pending_guard, ()) = tokio::join!(pending_acquire, release_holder);
|
||||
drop(pending_guard.expect("pending-hook PUT commit acquire should succeed"));
|
||||
assert_eq!(put_object_commit_namespace_lock_wait_sample_count(&snapshotter), 1);
|
||||
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_timeout_override_only_applies_to_put_commit() {
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("17"))], || {
|
||||
assert_eq!(get_put_object_commit_lock_acquire_timeout("put_object_commit"), Duration::from_millis(17));
|
||||
assert_eq!(
|
||||
get_put_object_commit_lock_acquire_timeout("complete_multipart_upload_commit"),
|
||||
get_lock_acquire_timeout()
|
||||
);
|
||||
});
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("0"))], || {
|
||||
assert_eq!(
|
||||
get_put_object_commit_lock_acquire_timeout("put_object_commit"),
|
||||
get_lock_acquire_timeout()
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_timeout_override_bounds_contention_wait() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should start");
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
|
||||
runtime.block_on(async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
let bucket = "bucket";
|
||||
let object = "object";
|
||||
|
||||
let held_guard = set
|
||||
.acquire_write_lock_diag("put_object_commit", bucket, object)
|
||||
.await
|
||||
.expect("holder acquire should succeed");
|
||||
let started = Instant::now();
|
||||
let err = match set.acquire_write_lock_diag("put_object_commit", bucket, object).await {
|
||||
Ok(_) => panic!("contended PUT commit lock should honor the short timeout"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(1),
|
||||
"short PUT commit lock timeout should not wait for the global timeout"
|
||||
);
|
||||
assert!(matches!(err, StorageError::SlowDown));
|
||||
|
||||
drop(held_guard);
|
||||
set.acquire_write_lock_diag("put_object_commit", bucket, object)
|
||||
.await
|
||||
.expect("permit should not leak after timeout");
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_admission_records_acquired_and_timeout() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should start");
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("1"))], || {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
runtime.block_on(async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
let held_guard = set
|
||||
.acquire_write_lock_diag("put_object_commit", "bucket", "object")
|
||||
.await
|
||||
.expect("holder acquire should succeed");
|
||||
let err = match set.acquire_write_lock_diag("put_object_commit", "bucket", "object").await {
|
||||
Ok(_) => panic!("contended PUT commit acquire should return SlowDown"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(matches!(err, StorageError::SlowDown));
|
||||
drop(held_guard);
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(
|
||||
put_object_commit_lock_admission_count(
|
||||
&rows,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
|
||||
),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
put_object_commit_lock_admission_count(
|
||||
&rows,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
|
||||
),
|
||||
1
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_admission_records_disabled_budget_acquired() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should start");
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("0"))], || {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
runtime.block_on(async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
let guard = set
|
||||
.acquire_write_lock_diag("put_object_commit", "bucket", "object")
|
||||
.await
|
||||
.expect("PUT commit acquire should succeed with default timeout");
|
||||
drop(guard);
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(
|
||||
put_object_commit_lock_admission_count(
|
||||
&rows,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
|
||||
),
|
||||
1
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_admission_skips_non_put_commit_ops() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should start");
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("250"))], || {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
runtime.block_on(async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
let guard = set
|
||||
.acquire_write_lock_diag("complete_multipart_upload_commit", "bucket", "object")
|
||||
.await
|
||||
.expect("non-PUT commit acquire should succeed");
|
||||
drop(guard);
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(
|
||||
rows.iter()
|
||||
.filter(|(composite, _, _, _)| {
|
||||
composite.key().name() == "rustfs_s3_put_object_commit_namespace_lock_admission_total"
|
||||
})
|
||||
.count(),
|
||||
0
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_admission_records_lock_error() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should start");
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("250"))], || {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
runtime.block_on(async {
|
||||
let healthy: Arc<dyn LockClient> =
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let failing: Arc<dyn LockClient> = Arc::new(FailingClient);
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
let set = make_test_set_disks_with_ctx(vec![healthy, failing], ctx).await;
|
||||
assert!(
|
||||
set.acquire_write_lock_diag("put_object_commit", "bucket", "object")
|
||||
.await
|
||||
.is_err(),
|
||||
"one healthy locker must not satisfy the PUT commit write quorum"
|
||||
);
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(
|
||||
put_object_commit_lock_admission_count(
|
||||
&rows,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR,
|
||||
),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
put_object_commit_lock_admission_count(
|
||||
&rows,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
|
||||
),
|
||||
0
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn put_object_commit_lock_admission_records_pending_hook_acquired() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should start");
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_PUT_COMMIT_NAMESPACE_LOCK_ACQUIRE_TIMEOUT_MS, Some("500"))], || {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
runtime.block_on(async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
let held_guard = set
|
||||
.acquire_write_lock_diag("put_object_commit", "bucket", "object")
|
||||
.await
|
||||
.expect("holder acquire should succeed");
|
||||
let (pending_tx, pending_rx) = tokio::sync::oneshot::channel();
|
||||
let pending_acquire =
|
||||
set.acquire_write_lock_diag_with_pending_hook("put_object_commit", "bucket", "object", move || {
|
||||
let _ = pending_tx.send(());
|
||||
});
|
||||
let release_holder = async {
|
||||
pending_rx.await.expect("pending hook should fire");
|
||||
drop(held_guard);
|
||||
};
|
||||
let (pending_guard, ()) = tokio::join!(pending_acquire, release_holder);
|
||||
drop(pending_guard.expect("pending-hook PUT commit acquire should succeed"));
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(
|
||||
put_object_commit_lock_admission_count(
|
||||
&rows,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS,
|
||||
rustfs_io_metrics::PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
|
||||
),
|
||||
2
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_ns_lock_shares_clients_without_changing_quorum() {
|
||||
let healthy: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
|
||||
@@ -124,14 +124,7 @@ impl HealWalkCollector {
|
||||
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
|
||||
let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
|
||||
let lifecycle_object_info = if self.include_lifecycle_object_info {
|
||||
let mut lifecycle_fi = fi.clone();
|
||||
lifecycle_fi.version_id = version_uuid;
|
||||
Some(ObjectInfo::from_file_info(
|
||||
&lifecycle_fi,
|
||||
&self.bucket,
|
||||
&entry.name,
|
||||
version_uuid.is_some(),
|
||||
))
|
||||
Some(ObjectInfo::from_file_info_with_version_id(fi, &self.bucket, &entry.name, version_uuid))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -198,14 +191,7 @@ impl HealWalkCollector {
|
||||
let vid = version_uuid.map(|u| u.to_string());
|
||||
if seen.insert(vid.clone()) {
|
||||
let lifecycle_object_info = if self.include_lifecycle_object_info {
|
||||
let mut lifecycle_fi = fi.clone();
|
||||
lifecycle_fi.version_id = version_uuid;
|
||||
Some(ObjectInfo::from_file_info(
|
||||
&lifecycle_fi,
|
||||
&self.bucket,
|
||||
&entry.name,
|
||||
version_uuid.is_some(),
|
||||
))
|
||||
Some(ObjectInfo::from_file_info_with_version_id(fi, &self.bucket, &entry.name, version_uuid))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
@@ -2433,8 +2433,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let commit_object_lock_guard = object_lock_guard.take();
|
||||
let detach_commit_owner = commit_object_lock_guard.is_some() || upload_guard.is_some() || quota_mutation_fence;
|
||||
let commit = async move {
|
||||
let _object_lock_guard = commit_object_lock_guard;
|
||||
let _upload_guard = upload_guard;
|
||||
let mut _object_lock_guard = commit_object_lock_guard;
|
||||
let mut _upload_guard = upload_guard;
|
||||
let mut quota_reservation = quota_reservation;
|
||||
let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
|
||||
@@ -2570,6 +2570,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let op_old_dir = rename_commit.data_dir;
|
||||
let cleanup_disks = rename_commit.cleanup_disks;
|
||||
let committed_file_info = rename_commit.committed_file_info;
|
||||
let rename_tail_drain = rename_commit.tail_drain;
|
||||
|
||||
// Detach admission before any post-commit await: client cancellation
|
||||
// must not couple durable convergence repair to cleanup work.
|
||||
@@ -2628,7 +2629,30 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.invalidate_get_object_metadata_cache(&commit_bucket, &commit_object)
|
||||
.await;
|
||||
|
||||
drop(_object_lock_guard); // release the object lock before multipart cleanup tail IO.
|
||||
if let Some(rename_tail_drain) = rename_tail_drain {
|
||||
let object_lock_guard = _object_lock_guard.take();
|
||||
let upload_guard = _upload_guard.take();
|
||||
let tail_bucket = commit_bucket.clone();
|
||||
let tail_object = commit_object.clone();
|
||||
tokio::spawn(async move {
|
||||
let _object_lock_guard = object_lock_guard;
|
||||
let _upload_guard = upload_guard;
|
||||
if let Err(err) = rename_tail_drain.await {
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
state = "failed",
|
||||
bucket = %tail_bucket,
|
||||
object = %tail_object,
|
||||
error = %err,
|
||||
"rename tail drain failed"
|
||||
);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
drop(_object_lock_guard.take()); // release the object lock before multipart cleanup tail IO.
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::AfterObjectPublication).await;
|
||||
@@ -2685,7 +2709,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
drop(_upload_guard);
|
||||
drop(_upload_guard.take());
|
||||
|
||||
Ok(ObjectInfo::from_file_info(&fi, &commit_bucket, &commit_object, commit_is_versioned))
|
||||
};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1077,23 +1077,23 @@ impl SetDisks {
|
||||
"Recoverable decode error triggered read repair"
|
||||
);
|
||||
let version_id = fi.version_id.as_ref().map(ToString::to_string);
|
||||
// MRF journal intent: keeps a durable Urgent ECDecode
|
||||
// request alive across restarts even when the in-memory
|
||||
// read-repair request is dropped or lost (HS-01).
|
||||
rustfs_common::mrf_channel::try_send_mrf_intent(
|
||||
rustfs_common::mrf_channel::MrfKind::DecodeFailure,
|
||||
bucket,
|
||||
object,
|
||||
fi.version_id,
|
||||
);
|
||||
submit_read_repair_heal(
|
||||
bucket,
|
||||
object,
|
||||
version_id.as_deref(),
|
||||
pool_index,
|
||||
set_index,
|
||||
Some(part_number),
|
||||
"decode_error",
|
||||
// Single-flight (backlog#1894 axis A): the durable
|
||||
// MRF intent (Urgent ECDecode across restarts, HS-01)
|
||||
// is bound to the read-repair reservation, so only the
|
||||
// first sighting within the dedup TTL books a journal
|
||||
// record instead of one per retried read.
|
||||
submit_read_repair_heal_with_submitter(
|
||||
ReadRepairHealSubmission {
|
||||
bucket,
|
||||
object,
|
||||
version_id: version_id.as_deref(),
|
||||
pool_index,
|
||||
set_index,
|
||||
part_number: Some(part_number),
|
||||
reason: "decode_error",
|
||||
mrf_intent: Some((rustfs_common::mrf_channel::MrfKind::DecodeFailure, fi.version_id)),
|
||||
},
|
||||
send_read_repair_heal_request,
|
||||
)
|
||||
.await;
|
||||
has_err = false;
|
||||
@@ -2577,6 +2577,7 @@ mod metadata_cache_tests {
|
||||
set_index: 0,
|
||||
part_number: Some(1),
|
||||
reason: "missing_shards",
|
||||
mrf_intent: None,
|
||||
},
|
||||
slow_read_repair_submitter,
|
||||
)
|
||||
@@ -2611,6 +2612,7 @@ mod metadata_cache_tests {
|
||||
set_index: 0,
|
||||
part_number: Some(1),
|
||||
reason: "missing_shards",
|
||||
mrf_intent: None,
|
||||
},
|
||||
dropped_read_repair_submitter,
|
||||
)
|
||||
@@ -2647,6 +2649,7 @@ mod metadata_cache_tests {
|
||||
set_index: 0,
|
||||
part_number: Some(1),
|
||||
reason: "missing_shards",
|
||||
mrf_intent: None,
|
||||
},
|
||||
capture_read_repair_submitter,
|
||||
)
|
||||
|
||||
+1117
-39
File diff suppressed because it is too large
Load Diff
@@ -151,6 +151,7 @@ pub(crate) mod init_format;
|
||||
pub(crate) mod list_objects;
|
||||
mod multipart;
|
||||
mod object;
|
||||
pub(crate) use object::ObjectLockDiagGuard;
|
||||
pub use object::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
SnapshotConsistencyError,
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::{
|
||||
bucket_lifecycle_ops::eval_action_from_lifecycle,
|
||||
get_expiry_configs,
|
||||
tier_delete_journal::{
|
||||
abort_prepared_tier_delete_journal_entry as abort_prepared_journal_entry_if_current, commit_tier_delete_journal_entry,
|
||||
enqueue_committed_tier_delete_journal_entry, persist_tier_delete_journal_entry,
|
||||
@@ -211,7 +213,8 @@ async fn delete_prefix_with_tier_delete_journal(
|
||||
opts: &ObjectOptions,
|
||||
tier_journal_api: Option<&Arc<ECStore>>,
|
||||
) -> Result<()> {
|
||||
let journal_entry = if let Some(api) = tier_journal_api {
|
||||
let lifecycle_delete_all = opts.lifecycle_delete_all.is_some();
|
||||
let journal_entry = if !lifecycle_delete_all && let Some(api) = tier_journal_api {
|
||||
Some(prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?)
|
||||
} else {
|
||||
None
|
||||
@@ -220,14 +223,34 @@ async fn delete_prefix_with_tier_delete_journal(
|
||||
let result = store.delete_prefix(bucket, object, opts).await;
|
||||
match result {
|
||||
Ok(()) => {
|
||||
if let (Some(api), Some(entries)) = (tier_journal_api, journal_entry.as_ref()) {
|
||||
let lifecycle_entries = if lifecycle_delete_all {
|
||||
opts.lifecycle_delete_all_journal()
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.lock()
|
||||
.prepared_entries()
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let entries = journal_entry.as_deref().unwrap_or(&lifecycle_entries);
|
||||
if let Some(api) = tier_journal_api {
|
||||
commit_prepared_tier_delete_journal_entries(api, entries).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(err) => {
|
||||
if let (Some(api), Some(entries)) = (tier_journal_api, journal_entry.as_ref()) {
|
||||
abort_prepared_tier_delete_journal_entries(api, entries).await;
|
||||
if let Some(api) = tier_journal_api {
|
||||
if lifecycle_delete_all {
|
||||
let (abort, entries) = {
|
||||
let journal = opts.lifecycle_delete_all_journal().ok_or(StorageError::PreconditionFailed)?;
|
||||
let state = journal.lock();
|
||||
(!state.mutation_started(), state.prepared_entries())
|
||||
};
|
||||
if abort {
|
||||
abort_prepared_tier_delete_journal_entries(api, &entries).await;
|
||||
}
|
||||
} else if let Some(entries) = journal_entry.as_ref() {
|
||||
abort_prepared_tier_delete_journal_entries(api, entries).await;
|
||||
}
|
||||
}
|
||||
Err(err)
|
||||
}
|
||||
@@ -327,7 +350,7 @@ impl fmt::Display for ObjectLockDiagMode {
|
||||
}
|
||||
}
|
||||
|
||||
struct ObjectLockDiagGuard {
|
||||
pub(crate) struct ObjectLockDiagGuard {
|
||||
guard: rustfs_lock::NamespaceLockGuard,
|
||||
enabled: bool,
|
||||
op: &'static str,
|
||||
@@ -360,14 +383,14 @@ impl ObjectLockDiagGuard {
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
|
||||
pub(crate) fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
|
||||
match &self.guard {
|
||||
rustfs_lock::NamespaceLockGuard::Standard(guard) => Some(guard.lock_lost()),
|
||||
rustfs_lock::NamespaceLockGuard::Fast(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_lock_lost(&self) -> bool {
|
||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||
self.guard.is_lock_lost()
|
||||
}
|
||||
}
|
||||
@@ -1109,6 +1132,26 @@ fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo,
|
||||
&& source_actual_size == target_actual_size
|
||||
}
|
||||
|
||||
fn tiered_data_movement_source_matches(
|
||||
expected: &rustfs_filemeta::FileInfo,
|
||||
current: &rustfs_filemeta::FileInfo,
|
||||
) -> Result<bool> {
|
||||
let expected_backend = crate::services::tier::tier::tier_destination_id_from_metadata(&expected.metadata)?;
|
||||
let current_backend = crate::services::tier::tier::tier_destination_id_from_metadata(¤t.metadata)?;
|
||||
Ok(expected.version_id == current.version_id
|
||||
&& expected.data_dir == current.data_dir
|
||||
&& expected.mod_time == current.mod_time
|
||||
&& expected.size == current.size
|
||||
&& expected.get_etag() == current.get_etag()
|
||||
&& expected.transition_status == current.transition_status
|
||||
&& expected.transitioned_objname == current.transitioned_objname
|
||||
&& expected.transition_tier == current.transition_tier
|
||||
&& expected.transition_version_id == current.transition_version_id
|
||||
&& expected.transition_version == current.transition_version
|
||||
&& expected.transition_version_state == current.transition_version_state
|
||||
&& expected_backend == current_backend)
|
||||
}
|
||||
|
||||
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
|
||||
target_pool_idx != src_pool_idx
|
||||
}
|
||||
@@ -1247,7 +1290,9 @@ impl ECStore {
|
||||
let mut opts = opts.clone();
|
||||
opts.no_lock = false;
|
||||
opts.metadata_cache_safe = false;
|
||||
let read_lock_guards = self.acquire_select_object_read_locks(bucket, &object, &mut opts).await?;
|
||||
let read_lock_guards = self
|
||||
.acquire_all_object_read_locks("select_object", bucket, &object, &mut opts)
|
||||
.await?;
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(SnapshotConsistencyError::LockingDisabled.into());
|
||||
}
|
||||
@@ -1488,8 +1533,9 @@ impl ECStore {
|
||||
)))
|
||||
}
|
||||
|
||||
async fn acquire_select_object_read_locks(
|
||||
pub(crate) async fn acquire_all_object_read_locks(
|
||||
&self,
|
||||
op: &'static str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &mut ObjectOptions,
|
||||
@@ -1501,10 +1547,7 @@ impl ECStore {
|
||||
// for each object's hashed set. DELETE and same-key CopyObject use the
|
||||
// fixed domain, while PUT commits and data movement use the hashed set.
|
||||
let distributed = self.ctx.is_dist_erasure().await;
|
||||
if let Some(guard) = self
|
||||
.acquire_object_read_lock_if_needed("select_object", bucket, object, opts)
|
||||
.await?
|
||||
{
|
||||
if let Some(guard) = self.acquire_object_read_lock_if_needed(op, bucket, object, opts).await? {
|
||||
guards.push(guard);
|
||||
}
|
||||
let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
|
||||
@@ -1527,7 +1570,7 @@ impl ECStore {
|
||||
.map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
|
||||
let owner = diag_enabled.then(|| ns_lock.owner().to_string());
|
||||
log_object_lock_acquire_if_slow(
|
||||
"select_object",
|
||||
op,
|
||||
bucket,
|
||||
object,
|
||||
owner.as_deref(),
|
||||
@@ -1538,7 +1581,7 @@ impl ECStore {
|
||||
guards.push(ObjectLockDiagGuard::new(
|
||||
guard,
|
||||
diag_enabled,
|
||||
"select_object",
|
||||
op,
|
||||
diag_enabled.then(|| bucket.to_string()),
|
||||
diag_enabled.then(|| object.to_string()),
|
||||
owner,
|
||||
@@ -1549,6 +1592,77 @@ impl ECStore {
|
||||
Ok(guards)
|
||||
}
|
||||
|
||||
async fn acquire_data_movement_object_write_locks(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
source_pool_idx: usize,
|
||||
target_pool_idx: usize,
|
||||
opts: &mut ObjectOptions,
|
||||
) -> Result<Vec<ObjectLockDiagGuard>> {
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(Error::other("tiered data movement requires namespace locking"));
|
||||
}
|
||||
let distributed = self.ctx.is_dist_erasure().await;
|
||||
let diag_enabled = is_object_lock_diag_enabled();
|
||||
let mut pool_indices = [source_pool_idx, target_pool_idx];
|
||||
pool_indices.sort_unstable();
|
||||
let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
|
||||
let mut locked_sets = vec![fixed_set];
|
||||
let mut guards = Vec::with_capacity(3);
|
||||
|
||||
// Lock order matches journal recovery: fixed store domain first, then
|
||||
// hashed domains by ascending pool index. This also serializes source
|
||||
// revalidation and target publication against ordinary object deletes.
|
||||
guards.push(self.acquire_object_write_lock("tiered_data_movement", bucket, object).await?);
|
||||
for pool_idx in pool_indices {
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_idx)
|
||||
.ok_or_else(|| Error::other(format!("invalid tiered data movement pool {pool_idx}")))?;
|
||||
let set = pool.get_disks_by_key(object);
|
||||
let lock_domain_already_held = !distributed
|
||||
|| locked_sets.iter().any(|locked_set: &Arc<crate::set_disk::SetDisks>| {
|
||||
same_distributed_lock_domain(&locked_set.lockers, &set.lockers)
|
||||
});
|
||||
if lock_domain_already_held {
|
||||
continue;
|
||||
}
|
||||
let ns_lock = set.new_ns_lock(bucket, object).await?;
|
||||
let acquire_start = Instant::now();
|
||||
let guard = ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
|
||||
let owner = diag_enabled.then(|| ns_lock.owner().to_string());
|
||||
log_object_lock_acquire_if_slow(
|
||||
"tiered_data_movement",
|
||||
bucket,
|
||||
object,
|
||||
owner.as_deref(),
|
||||
ObjectLockDiagMode::Write,
|
||||
acquire_start.elapsed(),
|
||||
diag_enabled,
|
||||
);
|
||||
guards.push(ObjectLockDiagGuard::new(
|
||||
guard,
|
||||
diag_enabled,
|
||||
"tiered_data_movement",
|
||||
diag_enabled.then(|| bucket.to_string()),
|
||||
diag_enabled.then(|| object.to_string()),
|
||||
owner,
|
||||
ObjectLockDiagMode::Write,
|
||||
));
|
||||
locked_sets.push(set);
|
||||
}
|
||||
opts.no_lock = true;
|
||||
for signal in guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal) {
|
||||
opts.add_namespace_lock_lost_signal(signal);
|
||||
}
|
||||
opts.ensure_namespace_lock_fence();
|
||||
Ok(guards)
|
||||
}
|
||||
|
||||
fn attach_read_lock_guard(mut reader: GetObjectReader, guard: Option<ObjectLockDiagGuard>) -> GetObjectReader {
|
||||
if is_lock_optimization_enabled() || reader.buffered_body.is_some() {
|
||||
return reader;
|
||||
@@ -1686,13 +1800,8 @@ impl ECStore {
|
||||
Some(guard)
|
||||
};
|
||||
|
||||
let mut fi = fi.clone();
|
||||
if opts.data_movement {
|
||||
crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?;
|
||||
}
|
||||
|
||||
let object = encode_dir_object(object);
|
||||
|
||||
let logical_object = object;
|
||||
let object = encode_dir_object(logical_object);
|
||||
if self.single_pool() {
|
||||
return Self::resolve_decommission_tiered_object_result(
|
||||
Err(Error::other("single pool deployments cannot decommission tiered objects")),
|
||||
@@ -1715,6 +1824,33 @@ impl ECStore {
|
||||
&object,
|
||||
)?
|
||||
};
|
||||
let _object_guards = self
|
||||
.acquire_data_movement_object_write_locks(bucket, &object, opts.src_pool_idx, idx, &mut opts)
|
||||
.await?;
|
||||
let source_pool = self
|
||||
.pools
|
||||
.get(opts.src_pool_idx)
|
||||
.ok_or_else(|| Error::other(format!("invalid tiered data movement source pool {}", opts.src_pool_idx)))?;
|
||||
let source_versions = source_pool
|
||||
.get_disks_by_key(&object)
|
||||
.load_file_info_versions_exact(bucket, logical_object)
|
||||
.await?;
|
||||
let current_source = source_versions
|
||||
.as_ref()
|
||||
.and_then(|versions| {
|
||||
versions
|
||||
.versions
|
||||
.iter()
|
||||
.find(|current| current.version_id == fi.version_id && !current.tier_free_version())
|
||||
})
|
||||
.ok_or_else(|| to_object_err(StorageError::FileNotFound, vec![bucket, object.as_str()]))?;
|
||||
if !tiered_data_movement_source_matches(fi, current_source)? {
|
||||
return Err(to_object_err(StorageError::FileNotFound, vec![bucket, object.as_str()]));
|
||||
}
|
||||
let mut fi = current_source.clone();
|
||||
if opts.data_movement {
|
||||
crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?;
|
||||
}
|
||||
if opts.data_movement && idx == opts.src_pool_idx {
|
||||
let resume_target_pool_idx = self
|
||||
.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
|
||||
@@ -2190,6 +2326,10 @@ impl ECStore {
|
||||
) -> Result<ObjectInfo> {
|
||||
check_del_obj_args(bucket, object)?;
|
||||
|
||||
if opts.lifecycle_delete_all.is_some() && self.ctx.lock_manager().is_disabled() {
|
||||
return Err(Error::other("lifecycle delete-all requires namespace locking"));
|
||||
}
|
||||
|
||||
let _bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
|
||||
None
|
||||
} else if opts.delete_prefix {
|
||||
@@ -2204,6 +2344,11 @@ impl ECStore {
|
||||
};
|
||||
let object = object.as_str();
|
||||
let mut opts = opts;
|
||||
let delete_all_configs = if opts.lifecycle_delete_all.is_some() {
|
||||
Some(get_expiry_configs(self, bucket).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
opts.tier_delete_journal_api = tier_journal_api.clone();
|
||||
if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
|
||||
opts.add_bucket_lifecycle_lock_guard(guard);
|
||||
@@ -2301,6 +2446,34 @@ impl ECStore {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(trigger) = opts.lifecycle_delete_all.as_ref() {
|
||||
let configs = delete_all_configs.as_ref().ok_or(StorageError::PreconditionFailed)?;
|
||||
let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
|
||||
if configs.table_bucket_enabled || configs.bucket_incarnation_id != expected_bucket_incarnation_id {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
let lifecycle = configs.lifecycle.as_ref().ok_or(StorageError::PreconditionFailed)?;
|
||||
let (mut current, _) = self
|
||||
.get_latest_object_info_with_idx(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
metadata_cache_safe: false,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let current_version_id = current.version_id.filter(|version_id| !version_id.is_nil());
|
||||
if current_version_id != trigger.version_id || current.delete_marker != trigger.delete_marker {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
current.name = decode_dir_object(¤t.name);
|
||||
let current_event = eval_action_from_lifecycle(lifecycle, configs.object_lock.as_deref(), ¤t).await;
|
||||
if current_event.action != trigger.action || current_event.rule_id != trigger.rule_id {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
}
|
||||
if opts.delete_prefix {
|
||||
delete_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
|
||||
return Ok(ObjectInfo::default());
|
||||
@@ -3828,6 +4001,24 @@ mod tests {
|
||||
assert!(is_equivalent_data_movement_tiered_object(&source, &target));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiered_data_movement_source_match_rejects_transition_identity_changes() {
|
||||
let source = tiered_equivalence_source();
|
||||
assert!(tiered_data_movement_source_matches(&source, &source).expect("matching source metadata should parse"));
|
||||
|
||||
let mut changed_remote = source.clone();
|
||||
changed_remote.transitioned_objname = "remote/replaced".to_string();
|
||||
assert!(!tiered_data_movement_source_matches(&source, &changed_remote).expect("changed remote metadata should parse"));
|
||||
|
||||
let mut changed_backend = source.clone();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut changed_backend.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
|
||||
rustfs_utils::crypto::hex([9; 32]),
|
||||
);
|
||||
assert!(!tiered_data_movement_source_matches(&source, &changed_backend).expect("backend metadata should parse"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn equivalent_data_movement_tiered_object_uses_logical_compressed_and_encrypted_sizes() {
|
||||
let mut compressed = tiered_equivalence_source();
|
||||
|
||||
@@ -202,10 +202,76 @@ impl ECStore {
|
||||
}
|
||||
|
||||
pub(super) async fn delete_prefix(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
|
||||
if opts.lifecycle_delete_all.is_some() {
|
||||
let mut preflight_opts = opts.clone();
|
||||
preflight_opts
|
||||
.lifecycle_delete_all
|
||||
.as_mut()
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.phase = crate::object_api::LifecycleDeleteAllPhase::Preflight;
|
||||
for pool in &self.pools {
|
||||
#[cfg(test)]
|
||||
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Preflight, pool.pool_idx)?;
|
||||
pool.delete_object(bucket, object, preflight_opts.clone()).await?;
|
||||
}
|
||||
|
||||
opts.lifecycle_delete_all_journal()
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.lock()
|
||||
.mark_mutation_started();
|
||||
let mut non_trigger_opts = opts.clone();
|
||||
non_trigger_opts
|
||||
.lifecycle_delete_all
|
||||
.as_mut()
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.phase = crate::object_api::LifecycleDeleteAllPhase::History;
|
||||
for pool in &self.pools {
|
||||
#[cfg(test)]
|
||||
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::History, pool.pool_idx)?;
|
||||
let mut pool_opts = non_trigger_opts.clone();
|
||||
pool_opts.delete_prefix = true;
|
||||
pool.delete_object(bucket, object, pool_opts).await?;
|
||||
}
|
||||
|
||||
let mut final_preflight_opts = opts.clone();
|
||||
final_preflight_opts
|
||||
.lifecycle_delete_all
|
||||
.as_mut()
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.phase = crate::object_api::LifecycleDeleteAllPhase::FinalPreflight;
|
||||
let mut trigger_pools = Vec::new();
|
||||
for (pool_index, pool) in self.pools.iter().enumerate() {
|
||||
#[cfg(test)]
|
||||
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::FinalPreflight, pool.pool_idx)?;
|
||||
let result = pool.delete_object(bucket, object, final_preflight_opts.clone()).await?;
|
||||
if !result.name.is_empty() {
|
||||
trigger_pools.push(pool_index);
|
||||
}
|
||||
}
|
||||
if trigger_pools.is_empty() {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
|
||||
let mut trigger_opts = opts.clone();
|
||||
trigger_opts
|
||||
.lifecycle_delete_all
|
||||
.as_mut()
|
||||
.ok_or(StorageError::PreconditionFailed)?
|
||||
.phase = crate::object_api::LifecycleDeleteAllPhase::Trigger;
|
||||
for pool_index in trigger_pools {
|
||||
#[cfg(test)]
|
||||
lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Trigger, pool_index)?;
|
||||
let mut pool_opts = trigger_opts.clone();
|
||||
pool_opts.delete_prefix = true;
|
||||
self.pools[pool_index].delete_object(bucket, object, pool_opts).await?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut first_error = None;
|
||||
let mut first_volume_error = None;
|
||||
let mut has_success = false;
|
||||
for pool in self.pools.iter() {
|
||||
for pool in &self.pools {
|
||||
let mut opts = opts.clone();
|
||||
opts.delete_prefix = true;
|
||||
match pool.delete_object(bucket, object, opts).await {
|
||||
@@ -774,6 +840,22 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static LIFECYCLE_DELETE_ALL_TEST_FAILURE: std::sync::Mutex<Option<(crate::object_api::LifecycleDeleteAllPhase, usize)>> =
|
||||
std::sync::Mutex::new(None);
|
||||
|
||||
#[cfg(test)]
|
||||
fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAllPhase, pool_index: usize) -> Result<()> {
|
||||
if LIFECYCLE_DELETE_ALL_TEST_FAILURE
|
||||
.lock()
|
||||
.expect("lifecycle delete-all failure hook should not poison")
|
||||
.is_some_and(|failure| failure == (phase, pool_index))
|
||||
{
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -781,23 +863,28 @@ mod tests {
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::layout::endpoint::Endpoint;
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::object_api::ObjectLockConfigSnapshot;
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use arc_swap::ArcSwap;
|
||||
use rustfs_config::server_config::KVS;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use std::sync::Arc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_attempts_later_pools_after_an_earlier_pool_error() {
|
||||
let temp_dir = tempfile::tempdir().expect("multi-pool delete test directory should be created");
|
||||
let mut pools = Vec::with_capacity(2);
|
||||
for (pool_index, drives_per_set) in [2, 4].into_iter().enumerate() {
|
||||
async fn setup_multi_pool_test_store(
|
||||
name: &str,
|
||||
drives_per_pool: &[usize],
|
||||
) -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
|
||||
let temp_dir = tempfile::tempdir().expect("multi-pool test directory should be created");
|
||||
let mut pools = Vec::with_capacity(drives_per_pool.len());
|
||||
for (pool_index, drives_per_set) in drives_per_pool.iter().copied().enumerate() {
|
||||
let mut endpoints = Vec::with_capacity(drives_per_set);
|
||||
for disk_index in 0..drives_per_set {
|
||||
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
|
||||
tokio::fs::create_dir_all(&disk_path)
|
||||
.await
|
||||
.expect("multi-pool delete test disk should be created");
|
||||
.expect("multi-pool test disk should be created");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
@@ -810,7 +897,7 @@ mod tests {
|
||||
set_count: 1,
|
||||
drives_per_set,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("delete-prefix-pool-{pool_index}"),
|
||||
cmd_line: format!("{name}-pool-{pool_index}"),
|
||||
platform: "test".to_string(),
|
||||
});
|
||||
}
|
||||
@@ -830,6 +917,92 @@ mod tests {
|
||||
.await
|
||||
.expect("multi-pool store should initialize");
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
(temp_dir, store, shutdown)
|
||||
}
|
||||
|
||||
struct LifecycleDeleteAllFailureGuard;
|
||||
|
||||
impl Drop for LifecycleDeleteAllFailureGuard {
|
||||
fn drop(&mut self) {
|
||||
*LIFECYCLE_DELETE_ALL_TEST_FAILURE
|
||||
.lock()
|
||||
.expect("lifecycle delete-all failure hook should not poison") = None;
|
||||
}
|
||||
}
|
||||
|
||||
async fn seed_multi_pool_delete_all(store: &Arc<ECStore>, bucket: &str, object: &str) -> ObjectOptions {
|
||||
let trigger_id = Uuid::new_v4();
|
||||
for (pool_index, pool) in store.pools.iter().enumerate() {
|
||||
let mut history_reader = PutObjReader::from_vec(format!("{object}-history-{pool_index}").into_bytes());
|
||||
pool.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut history_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("history should be stored");
|
||||
let mut trigger_reader = PutObjReader::from_vec(format!("{object}-trigger-{pool_index}").into_bytes());
|
||||
pool.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut trigger_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(trigger_id.to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("shared trigger should be stored");
|
||||
}
|
||||
let mut opts = ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
versioned: true,
|
||||
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
|
||||
version_id: Some(trigger_id),
|
||||
delete_marker: false,
|
||||
action: rustfs_common::metrics::IlmAction::DeleteAllVersionsAction,
|
||||
rule_id: "rule".to_string(),
|
||||
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
||||
}),
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
delete_replication_config_snapshot: Some(Arc::new(
|
||||
crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
|
||||
)),
|
||||
..Default::default()
|
||||
};
|
||||
opts.ensure_lifecycle_delete_all_journal();
|
||||
opts
|
||||
}
|
||||
|
||||
async fn ordinary_version_count(store: &ECStore, pool_index: usize, bucket: &str, object: &str) -> usize {
|
||||
store.pools[pool_index].disk_set[0]
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("pool metadata should load")
|
||||
.map(|versions| {
|
||||
versions
|
||||
.versions
|
||||
.iter()
|
||||
.filter(|version| !version.tier_free_version())
|
||||
.count()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_prefix_attempts_later_pools_after_an_earlier_pool_error() {
|
||||
let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("delete-prefix", &[2, 4]).await;
|
||||
let bucket = format!("delete-prefix-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
@@ -921,6 +1094,171 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn lifecycle_delete_all_history_failure_preserves_trigger_and_retry_converges() {
|
||||
let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("lifecycle-delete-all", &[4, 4]).await;
|
||||
let bucket = format!("lifecycle-delete-all-{}", Uuid::new_v4().simple());
|
||||
let object = "object";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created in both pools");
|
||||
|
||||
for pool_index in 0..2 {
|
||||
let mut reader = PutObjReader::from_vec(format!("pool-{pool_index}-history").into_bytes());
|
||||
store.pools[pool_index]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("historical version should be stored");
|
||||
}
|
||||
let marker = store.pools[0]
|
||||
.delete_object(
|
||||
&bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("trigger marker should be stored in the first pool");
|
||||
let marker_id = marker.version_id.expect("trigger marker should have a version id");
|
||||
let mut opts = ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
versioned: true,
|
||||
lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
|
||||
version_id: Some(marker_id),
|
||||
delete_marker: true,
|
||||
action: rustfs_common::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
|
||||
rule_id: "rule".to_string(),
|
||||
phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
|
||||
}),
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
delete_replication_config_snapshot: Some(Arc::new(
|
||||
crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
|
||||
)),
|
||||
..Default::default()
|
||||
};
|
||||
opts.ensure_lifecycle_delete_all_journal();
|
||||
|
||||
let _failure_guard = LifecycleDeleteAllFailureGuard;
|
||||
*LIFECYCLE_DELETE_ALL_TEST_FAILURE
|
||||
.lock()
|
||||
.expect("lifecycle delete-all failure hook should not poison") =
|
||||
Some((crate::object_api::LifecycleDeleteAllPhase::History, 1));
|
||||
let err = store
|
||||
.delete_prefix(&bucket, object, &opts)
|
||||
.await
|
||||
.expect_err("a later pool history failure must stop before trigger deletion");
|
||||
assert_eq!(err, StorageError::PreconditionFailed);
|
||||
assert!(
|
||||
opts.lifecycle_delete_all_journal()
|
||||
.expect("delete-all journal should be initialized")
|
||||
.lock()
|
||||
.mutation_started()
|
||||
);
|
||||
|
||||
let first_pool = store.pools[0].disk_set[0]
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("first pool metadata should load")
|
||||
.expect("the trigger should remain");
|
||||
let first_pool_ordinary: Vec<&FileInfo> = first_pool
|
||||
.versions
|
||||
.iter()
|
||||
.filter(|version| !version.tier_free_version())
|
||||
.collect();
|
||||
assert_eq!(first_pool_ordinary.len(), 1);
|
||||
assert_eq!(first_pool_ordinary[0].version_id, Some(marker_id));
|
||||
assert!(first_pool_ordinary[0].deleted);
|
||||
|
||||
*LIFECYCLE_DELETE_ALL_TEST_FAILURE
|
||||
.lock()
|
||||
.expect("lifecycle delete-all failure hook should not poison") = None;
|
||||
store
|
||||
.delete_prefix(&bucket, object, &opts)
|
||||
.await
|
||||
.expect("retry should delete remaining history and its trigger owner");
|
||||
for pool in &store.pools {
|
||||
assert!(
|
||||
pool.disk_set[0]
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("pool metadata should load after retry")
|
||||
.is_none(),
|
||||
"all ordinary versions should be removed after retry"
|
||||
);
|
||||
}
|
||||
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn lifecycle_delete_all_phase_failures_preserve_barriers_and_retry() {
|
||||
let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("lifecycle-delete-all-phases", &[4, 4]).await;
|
||||
let bucket = format!("lifecycle-delete-all-phases-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created in both pools");
|
||||
let _failure_guard = LifecycleDeleteAllFailureGuard;
|
||||
|
||||
for (object, phase, expected_counts, mutation_started) in [
|
||||
("preflight-failure", crate::object_api::LifecycleDeleteAllPhase::Preflight, [2, 2], false),
|
||||
(
|
||||
"final-preflight-failure",
|
||||
crate::object_api::LifecycleDeleteAllPhase::FinalPreflight,
|
||||
[1, 1],
|
||||
true,
|
||||
),
|
||||
("trigger-failure", crate::object_api::LifecycleDeleteAllPhase::Trigger, [0, 1], true),
|
||||
] {
|
||||
let opts = seed_multi_pool_delete_all(&store, &bucket, object).await;
|
||||
*LIFECYCLE_DELETE_ALL_TEST_FAILURE
|
||||
.lock()
|
||||
.expect("lifecycle delete-all failure hook should not poison") = Some((phase, 1));
|
||||
let err = store
|
||||
.delete_prefix(&bucket, object, &opts)
|
||||
.await
|
||||
.expect_err("injected phase failure should stop the transaction");
|
||||
assert_eq!(err, StorageError::PreconditionFailed);
|
||||
assert_eq!(
|
||||
opts.lifecycle_delete_all_journal()
|
||||
.expect("delete-all journal should be initialized")
|
||||
.lock()
|
||||
.mutation_started(),
|
||||
mutation_started
|
||||
);
|
||||
assert_eq!(ordinary_version_count(&store, 0, &bucket, object).await, expected_counts[0]);
|
||||
assert_eq!(ordinary_version_count(&store, 1, &bucket, object).await, expected_counts[1]);
|
||||
|
||||
*LIFECYCLE_DELETE_ALL_TEST_FAILURE
|
||||
.lock()
|
||||
.expect("lifecycle delete-all failure hook should not poison") = None;
|
||||
store
|
||||
.delete_prefix(&bucket, object, &opts)
|
||||
.await
|
||||
.expect("retry should converge after the injected failure is removed");
|
||||
assert_eq!(ordinary_version_count(&store, 0, &bucket, object).await, 0);
|
||||
assert_eq!(ordinary_version_count(&store, 1, &bucket, object).await, 0);
|
||||
}
|
||||
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
fn assert_backend_layout_empty(info: &rustfs_madmin::BackendInfo) {
|
||||
assert!(info.standard_sc_parities.is_empty());
|
||||
assert!(info.standard_sc_data.is_empty());
|
||||
|
||||
@@ -759,7 +759,7 @@ impl HealChannelProcessor {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::{DiskStore, Endpoint};
|
||||
use super::super::DiskStore;
|
||||
use super::*;
|
||||
use crate::heal::manager::HealConfig;
|
||||
use crate::heal::storage::{HealObjectInfo, HealStorageAPI};
|
||||
@@ -776,45 +776,18 @@ mod tests {
|
||||
async fn get_object_meta(&self, _bucket: &str, _object: &str) -> crate::Result<Option<HealObjectInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn get_object_data(&self, _bucket: &str, _object: &str) -> crate::Result<Option<Vec<u8>>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn delete_object(&self, _bucket: &str, _object: &str) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> crate::Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> crate::Result<Vec<u8>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
async fn get_disk_status(&self, _endpoint: &Endpoint) -> crate::Result<crate::heal::storage::DiskStatus> {
|
||||
Ok(crate::heal::storage::DiskStatus::Ok)
|
||||
}
|
||||
async fn format_disk(&self, _endpoint: &Endpoint) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn get_bucket_info(&self, _bucket: &str) -> crate::Result<Option<crate::heal::storage_api::status::BucketInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn heal_bucket_metadata(&self, _bucket: &str) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn list_buckets(&self) -> crate::Result<Vec<crate::heal::storage_api::status::BucketInfo>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
async fn object_exists(&self, _bucket: &str, _object: &str) -> crate::Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
async fn get_object_size(&self, _bucket: &str, _object: &str) -> crate::Result<Option<u64>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> crate::Result<Option<String>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn heal_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -837,13 +810,6 @@ mod tests {
|
||||
) -> crate::Result<(rustfs_madmin::heal_commands::HealResultItem, Option<crate::Error>)> {
|
||||
Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None))
|
||||
}
|
||||
async fn list_objects_for_heal(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
) -> crate::Result<Vec<crate::heal::storage::HealListItem>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
|
||||
@@ -1267,7 +1267,7 @@ mod resume_loop_tests {
|
||||
CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils,
|
||||
compose_key,
|
||||
};
|
||||
use crate::heal::storage::{DiskStatus, HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::storage::{HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI};
|
||||
use crate::heal::storage_api::status::BucketInfo;
|
||||
use crate::heal::{
|
||||
BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, RUSTFS_META_BUCKET, new_disk,
|
||||
@@ -1448,36 +1448,15 @@ mod resume_loop_tests {
|
||||
async fn get_object_meta(&self, _b: &str, _o: &str) -> Result<Option<HealObjectInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn get_object_data(&self, _b: &str, _o: &str) -> Result<Option<Vec<u8>>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn put_object_data(&self, _b: &str, _o: &str, _d: &[u8]) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn delete_object(&self, _b: &str, _o: &str) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn verify_object_integrity(&self, _b: &str, _o: &str) -> Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
async fn ec_decode_rebuild(&self, _b: &str, _o: &str) -> Result<Vec<u8>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
async fn get_disk_status(&self, _e: &Endpoint) -> Result<DiskStatus> {
|
||||
Ok(DiskStatus::Ok)
|
||||
}
|
||||
async fn format_disk(&self, _e: &Endpoint) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>> {
|
||||
Ok(Some(BucketInfo {
|
||||
name: bucket.to_string(),
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
async fn heal_bucket_metadata(&self, _b: &str) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
@@ -1485,12 +1464,6 @@ mod resume_loop_tests {
|
||||
// Must never be consulted: the resume loop always goes through heal_object.
|
||||
panic!("object_exists must not be called by the resume heal loop");
|
||||
}
|
||||
async fn get_object_size(&self, _b: &str, _o: &str) -> Result<Option<u64>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn get_object_checksum(&self, _b: &str, _o: &str) -> Result<Option<String>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
|
||||
Ok((!self.lifecycle_expired.lock().unwrap().is_empty()).then(HealLifecycleExpiryContext::test))
|
||||
}
|
||||
@@ -1556,9 +1529,6 @@ mod resume_loop_tests {
|
||||
ReplacementCommitEvidence::Error(message) => Err(Error::other(message)),
|
||||
}
|
||||
}
|
||||
async fn list_objects_for_heal(&self, _b: &str, _p: &str) -> Result<Vec<HealListItem>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
|
||||
@@ -1,683 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::heal::{HealOptions, HealPriority, HealRequest, HealType};
|
||||
use crate::{Error, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::SystemTime;
|
||||
|
||||
use super::Endpoint;
|
||||
|
||||
/// Corruption type
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub enum CorruptionType {
|
||||
/// Data corruption
|
||||
DataCorruption,
|
||||
/// Metadata corruption
|
||||
MetadataCorruption,
|
||||
/// Partial corruption
|
||||
PartialCorruption,
|
||||
/// Complete corruption
|
||||
CompleteCorruption,
|
||||
}
|
||||
|
||||
/// Severity level
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
pub enum Severity {
|
||||
/// Low severity
|
||||
Low = 0,
|
||||
/// Medium severity
|
||||
Medium = 1,
|
||||
/// High severity
|
||||
High = 2,
|
||||
/// Critical severity
|
||||
Critical = 3,
|
||||
}
|
||||
|
||||
/// Heal event
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum HealEvent {
|
||||
/// Object corruption event
|
||||
ObjectCorruption {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<String>,
|
||||
corruption_type: CorruptionType,
|
||||
severity: Severity,
|
||||
},
|
||||
/// Object missing event
|
||||
ObjectMissing {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<String>,
|
||||
expected_locations: Vec<usize>,
|
||||
available_locations: Vec<usize>,
|
||||
},
|
||||
/// Metadata corruption event
|
||||
MetadataCorruption {
|
||||
bucket: String,
|
||||
object: String,
|
||||
corruption_type: CorruptionType,
|
||||
},
|
||||
/// Disk status change event
|
||||
DiskStatusChange {
|
||||
endpoint: Endpoint,
|
||||
old_status: String,
|
||||
new_status: String,
|
||||
},
|
||||
/// EC decode failure event
|
||||
ECDecodeFailure {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<String>,
|
||||
missing_shards: Vec<usize>,
|
||||
available_shards: Vec<usize>,
|
||||
},
|
||||
/// Checksum mismatch event
|
||||
ChecksumMismatch {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<String>,
|
||||
expected_checksum: String,
|
||||
actual_checksum: String,
|
||||
},
|
||||
/// Bucket metadata corruption event
|
||||
BucketMetadataCorruption {
|
||||
bucket: String,
|
||||
corruption_type: CorruptionType,
|
||||
},
|
||||
/// MRF metadata corruption event
|
||||
MRFMetadataCorruption {
|
||||
meta_path: String,
|
||||
corruption_type: CorruptionType,
|
||||
},
|
||||
}
|
||||
|
||||
impl HealEvent {
|
||||
/// Convert HealEvent to HealRequest
|
||||
pub fn to_heal_request(&self) -> Result<HealRequest> {
|
||||
match self {
|
||||
HealEvent::ObjectCorruption {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
severity,
|
||||
..
|
||||
} => Ok(HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: bucket.clone(),
|
||||
object: object.clone(),
|
||||
version_id: version_id.clone(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
Self::severity_to_priority(severity),
|
||||
)),
|
||||
HealEvent::ObjectMissing {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
..
|
||||
} => Ok(HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: bucket.clone(),
|
||||
object: object.clone(),
|
||||
version_id: version_id.clone(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
)),
|
||||
HealEvent::MetadataCorruption { bucket, object, .. } => Ok(HealRequest::new(
|
||||
HealType::Metadata {
|
||||
bucket: bucket.clone(),
|
||||
object: object.clone(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
)),
|
||||
HealEvent::DiskStatusChange { endpoint, .. } => {
|
||||
// Convert disk status change to erasure set heal
|
||||
// Note: This requires access to storage to get bucket list, which is not available here
|
||||
// The actual bucket list will need to be provided by the caller or retrieved differently
|
||||
let set_disk_id = crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx)
|
||||
.ok_or_else(|| Error::InvalidHealType {
|
||||
heal_type: format!("erasure-set(pool={}, set={})", endpoint.pool_idx, endpoint.set_idx),
|
||||
})?;
|
||||
Ok(HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec![], // Empty bucket list - caller should populate this
|
||||
set_disk_id,
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
))
|
||||
}
|
||||
HealEvent::ECDecodeFailure {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
..
|
||||
} => Ok(HealRequest::new(
|
||||
HealType::ECDecode {
|
||||
bucket: bucket.clone(),
|
||||
object: object.clone(),
|
||||
version_id: version_id.clone(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Urgent,
|
||||
)),
|
||||
HealEvent::ChecksumMismatch {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
..
|
||||
} => Ok(HealRequest::new(
|
||||
HealType::Object {
|
||||
bucket: bucket.clone(),
|
||||
object: object.clone(),
|
||||
version_id: version_id.clone(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
)),
|
||||
HealEvent::BucketMetadataCorruption { bucket, .. } => Ok(HealRequest::new(
|
||||
HealType::Bucket { bucket: bucket.clone() },
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
)),
|
||||
HealEvent::MRFMetadataCorruption { meta_path, .. } => Ok(HealRequest::new(
|
||||
HealType::MRF {
|
||||
meta_path: meta_path.clone(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert severity to priority
|
||||
fn severity_to_priority(severity: &Severity) -> HealPriority {
|
||||
match severity {
|
||||
Severity::Low => HealPriority::Low,
|
||||
Severity::Medium => HealPriority::Normal,
|
||||
Severity::High => HealPriority::High,
|
||||
Severity::Critical => HealPriority::Urgent,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get event description
|
||||
pub fn description(&self) -> String {
|
||||
match self {
|
||||
HealEvent::ObjectCorruption {
|
||||
bucket,
|
||||
object,
|
||||
corruption_type,
|
||||
..
|
||||
} => {
|
||||
format!("Object corruption detected: {bucket}/{object} - {corruption_type:?}")
|
||||
}
|
||||
HealEvent::ObjectMissing { bucket, object, .. } => {
|
||||
format!("Object missing: {bucket}/{object}")
|
||||
}
|
||||
HealEvent::MetadataCorruption {
|
||||
bucket,
|
||||
object,
|
||||
corruption_type,
|
||||
..
|
||||
} => {
|
||||
format!("Metadata corruption: {bucket}/{object} - {corruption_type:?}")
|
||||
}
|
||||
HealEvent::DiskStatusChange {
|
||||
endpoint,
|
||||
old_status,
|
||||
new_status,
|
||||
..
|
||||
} => {
|
||||
format!("Disk status changed: {endpoint:?} {old_status} -> {new_status}")
|
||||
}
|
||||
HealEvent::ECDecodeFailure {
|
||||
bucket,
|
||||
object,
|
||||
missing_shards,
|
||||
..
|
||||
} => {
|
||||
format!("EC decode failure: {bucket}/{object} - missing shards: {missing_shards:?}")
|
||||
}
|
||||
HealEvent::ChecksumMismatch {
|
||||
bucket,
|
||||
object,
|
||||
expected_checksum,
|
||||
actual_checksum,
|
||||
..
|
||||
} => {
|
||||
format!("Checksum mismatch: {bucket}/{object} - expected: {expected_checksum}, actual: {actual_checksum}")
|
||||
}
|
||||
HealEvent::BucketMetadataCorruption {
|
||||
bucket, corruption_type, ..
|
||||
} => {
|
||||
format!("Bucket metadata corruption: {bucket} - {corruption_type:?}")
|
||||
}
|
||||
HealEvent::MRFMetadataCorruption {
|
||||
meta_path,
|
||||
corruption_type,
|
||||
..
|
||||
} => {
|
||||
format!("MRF metadata corruption: {meta_path} - {corruption_type:?}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Get event severity
|
||||
pub fn severity(&self) -> Severity {
|
||||
match self {
|
||||
HealEvent::ObjectCorruption { severity, .. } => severity.clone(),
|
||||
HealEvent::ObjectMissing { .. } => Severity::High,
|
||||
HealEvent::MetadataCorruption { .. } => Severity::High,
|
||||
HealEvent::DiskStatusChange { .. } => Severity::High,
|
||||
HealEvent::ECDecodeFailure { .. } => Severity::Critical,
|
||||
HealEvent::ChecksumMismatch { .. } => Severity::High,
|
||||
HealEvent::BucketMetadataCorruption { .. } => Severity::High,
|
||||
HealEvent::MRFMetadataCorruption { .. } => Severity::High,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get event timestamp
|
||||
pub fn timestamp(&self) -> SystemTime {
|
||||
SystemTime::now()
|
||||
}
|
||||
}
|
||||
|
||||
/// Heal event handler
|
||||
pub struct HealEventHandler {
|
||||
/// Event queue
|
||||
events: Vec<HealEvent>,
|
||||
/// Maximum number of events
|
||||
max_events: usize,
|
||||
}
|
||||
|
||||
impl HealEventHandler {
|
||||
pub fn new(max_events: usize) -> Self {
|
||||
Self {
|
||||
events: Vec::new(),
|
||||
max_events,
|
||||
}
|
||||
}
|
||||
|
||||
/// Add event
|
||||
pub fn add_event(&mut self, event: HealEvent) {
|
||||
if self.events.len() >= self.max_events {
|
||||
// Remove oldest event
|
||||
self.events.remove(0);
|
||||
}
|
||||
self.events.push(event);
|
||||
}
|
||||
|
||||
/// Get all events
|
||||
pub fn get_events(&self) -> &[HealEvent] {
|
||||
&self.events
|
||||
}
|
||||
|
||||
/// Clear events
|
||||
pub fn clear_events(&mut self) {
|
||||
self.events.clear();
|
||||
}
|
||||
|
||||
/// Get event count
|
||||
pub fn event_count(&self) -> usize {
|
||||
self.events.len()
|
||||
}
|
||||
|
||||
/// Filter events by severity
|
||||
pub fn filter_by_severity(&self, min_severity: Severity) -> Vec<&HealEvent> {
|
||||
self.events.iter().filter(|event| event.severity() >= min_severity).collect()
|
||||
}
|
||||
|
||||
/// Filter events by type
|
||||
pub fn filter_by_type(&self, event_type: &str) -> Vec<&HealEvent> {
|
||||
self.events
|
||||
.iter()
|
||||
.filter(|event| match event {
|
||||
HealEvent::ObjectCorruption { .. } => event_type == "ObjectCorruption",
|
||||
HealEvent::ObjectMissing { .. } => event_type == "ObjectMissing",
|
||||
HealEvent::MetadataCorruption { .. } => event_type == "MetadataCorruption",
|
||||
HealEvent::DiskStatusChange { .. } => event_type == "DiskStatusChange",
|
||||
HealEvent::ECDecodeFailure { .. } => event_type == "ECDecodeFailure",
|
||||
HealEvent::ChecksumMismatch { .. } => event_type == "ChecksumMismatch",
|
||||
HealEvent::BucketMetadataCorruption { .. } => event_type == "BucketMetadataCorruption",
|
||||
HealEvent::MRFMetadataCorruption { .. } => event_type == "MRFMetadataCorruption",
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for HealEventHandler {
|
||||
fn default() -> Self {
|
||||
Self::new(1000)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::heal::task::{HealPriority, HealType};
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_object_corruption_to_request() {
|
||||
let event = HealEvent::ObjectCorruption {
|
||||
bucket: "test-bucket".to_string(),
|
||||
object: "test-object".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
|
||||
let request = event.to_heal_request().unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::Object { .. }));
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_object_missing_to_request() {
|
||||
let event = HealEvent::ObjectMissing {
|
||||
bucket: "test-bucket".to_string(),
|
||||
object: "test-object".to_string(),
|
||||
version_id: Some("v1".to_string()),
|
||||
expected_locations: vec![0, 1],
|
||||
available_locations: vec![2, 3],
|
||||
};
|
||||
|
||||
let request = event.to_heal_request().unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::Object { .. }));
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_metadata_corruption_to_request() {
|
||||
let event = HealEvent::MetadataCorruption {
|
||||
bucket: "test-bucket".to_string(),
|
||||
object: "test-object".to_string(),
|
||||
corruption_type: CorruptionType::MetadataCorruption,
|
||||
};
|
||||
|
||||
let request = event.to_heal_request().unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::Metadata { .. }));
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_ec_decode_failure_to_request() {
|
||||
let event = HealEvent::ECDecodeFailure {
|
||||
bucket: "test-bucket".to_string(),
|
||||
object: "test-object".to_string(),
|
||||
version_id: None,
|
||||
missing_shards: vec![0, 1],
|
||||
available_shards: vec![2, 3, 4],
|
||||
};
|
||||
|
||||
let request = event.to_heal_request().unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::ECDecode { .. }));
|
||||
assert_eq!(request.priority, HealPriority::Urgent);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_checksum_mismatch_to_request() {
|
||||
let event = HealEvent::ChecksumMismatch {
|
||||
bucket: "test-bucket".to_string(),
|
||||
object: "test-object".to_string(),
|
||||
version_id: None,
|
||||
expected_checksum: "abc123".to_string(),
|
||||
actual_checksum: "def456".to_string(),
|
||||
};
|
||||
|
||||
let request = event.to_heal_request().unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::Object { .. }));
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_bucket_metadata_corruption_to_request() {
|
||||
let event = HealEvent::BucketMetadataCorruption {
|
||||
bucket: "test-bucket".to_string(),
|
||||
corruption_type: CorruptionType::MetadataCorruption,
|
||||
};
|
||||
|
||||
let request = event.to_heal_request().unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::Bucket { .. }));
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_mrf_metadata_corruption_to_request() {
|
||||
let event = HealEvent::MRFMetadataCorruption {
|
||||
meta_path: "test-bucket/test-object".to_string(),
|
||||
corruption_type: CorruptionType::MetadataCorruption,
|
||||
};
|
||||
|
||||
let request = event.to_heal_request().unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::MRF { .. }));
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_severity_to_priority() {
|
||||
let event_low = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::Low,
|
||||
};
|
||||
let request = event_low.to_heal_request().unwrap();
|
||||
assert_eq!(request.priority, HealPriority::Low);
|
||||
|
||||
let event_medium = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::Medium,
|
||||
};
|
||||
let request = event_medium.to_heal_request().unwrap();
|
||||
assert_eq!(request.priority, HealPriority::Normal);
|
||||
|
||||
let event_high = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
let request = event_high.to_heal_request().unwrap();
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
|
||||
let event_critical = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::Critical,
|
||||
};
|
||||
let request = event_critical.to_heal_request().unwrap();
|
||||
assert_eq!(request.priority, HealPriority::Urgent);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_description() {
|
||||
let event = HealEvent::ObjectCorruption {
|
||||
bucket: "test-bucket".to_string(),
|
||||
object: "test-object".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
|
||||
let desc = event.description();
|
||||
assert!(desc.contains("Object corruption detected"));
|
||||
assert!(desc.contains("test-bucket/test-object"));
|
||||
assert!(desc.contains("DataCorruption"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_severity() {
|
||||
let event = HealEvent::ECDecodeFailure {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
missing_shards: vec![],
|
||||
available_shards: vec![],
|
||||
};
|
||||
assert_eq!(event.severity(), Severity::Critical);
|
||||
|
||||
let event = HealEvent::ObjectMissing {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
expected_locations: vec![],
|
||||
available_locations: vec![],
|
||||
};
|
||||
assert_eq!(event.severity(), Severity::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_new() {
|
||||
let handler = HealEventHandler::new(10);
|
||||
assert_eq!(handler.event_count(), 0);
|
||||
assert_eq!(handler.max_events, 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_default() {
|
||||
let handler = HealEventHandler::default();
|
||||
assert_eq!(handler.max_events, 1000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_add_event() {
|
||||
let mut handler = HealEventHandler::new(3);
|
||||
let event = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
|
||||
handler.add_event(event.clone());
|
||||
assert_eq!(handler.event_count(), 1);
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event);
|
||||
assert_eq!(handler.event_count(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_max_events() {
|
||||
let mut handler = HealEventHandler::new(2);
|
||||
let event = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event); // Should remove oldest
|
||||
|
||||
assert_eq!(handler.event_count(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_get_events() {
|
||||
let mut handler = HealEventHandler::new(10);
|
||||
let event = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
|
||||
handler.add_event(event.clone());
|
||||
handler.add_event(event);
|
||||
|
||||
let events = handler.get_events();
|
||||
assert_eq!(events.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_clear_events() {
|
||||
let mut handler = HealEventHandler::new(10);
|
||||
let event = HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
|
||||
handler.add_event(event);
|
||||
assert_eq!(handler.event_count(), 1);
|
||||
|
||||
handler.clear_events();
|
||||
assert_eq!(handler.event_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_filter_by_severity() {
|
||||
let mut handler = HealEventHandler::new(10);
|
||||
handler.add_event(HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::Low,
|
||||
});
|
||||
handler.add_event(HealEvent::ECDecodeFailure {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
missing_shards: vec![],
|
||||
available_shards: vec![],
|
||||
});
|
||||
|
||||
let high_severity = handler.filter_by_severity(Severity::High);
|
||||
assert_eq!(high_severity.len(), 1); // Only ECDecodeFailure is Critical >= High
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_handler_filter_by_type() {
|
||||
let mut handler = HealEventHandler::new(10);
|
||||
handler.add_event(HealEvent::ObjectCorruption {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
});
|
||||
handler.add_event(HealEvent::ObjectMissing {
|
||||
bucket: "test".to_string(),
|
||||
object: "test".to_string(),
|
||||
version_id: None,
|
||||
expected_locations: vec![],
|
||||
available_locations: vec![],
|
||||
});
|
||||
|
||||
let corruption_events = handler.filter_by_type("ObjectCorruption");
|
||||
assert_eq!(corruption_events.len(), 1);
|
||||
|
||||
let missing_events = handler.filter_by_type("ObjectMissing");
|
||||
assert_eq!(missing_events.len(), 1);
|
||||
}
|
||||
}
|
||||
+267
-5002
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,562 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// The automatic disk scanner: replacement discovery and unformatted-disk enqueue.
|
||||
use super::*;
|
||||
|
||||
impl HealManager {
|
||||
/// Start background task to auto scan local disks and enqueue erasure set heal requests
|
||||
pub(super) async fn start_auto_disk_scanner(&self) -> Result<()> {
|
||||
let config = self.config.clone();
|
||||
let heal_queue = self.heal_queue.clone();
|
||||
let active_heals = self.active_heals.clone();
|
||||
let task_aliases = self.task_aliases.clone();
|
||||
let mrf_repair_notice_targets = self.mrf_repair_notice_targets.clone();
|
||||
let storage = self.storage.clone();
|
||||
let replacement_recovery_anchors = self.replacement_recovery_anchors.clone();
|
||||
let replacement_recovery_blocked_sets = self.replacement_recovery_blocked_sets.clone();
|
||||
let cancel_token = self.cancel_token.clone();
|
||||
let notify = self.notify.clone();
|
||||
let mut duration = {
|
||||
let config = config.read().await;
|
||||
config.heal_interval
|
||||
};
|
||||
if duration < Duration::from_secs(10) {
|
||||
duration = Duration::from_secs(10);
|
||||
}
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
state = "started",
|
||||
interval = ?duration,
|
||||
"Heal auto disk scanner started"
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval = interval(duration);
|
||||
|
||||
loop {
|
||||
let mut candidate_count = 0usize;
|
||||
let mut skipped_duplicate_count = 0usize;
|
||||
let mut skipped_invalid_count = 0usize;
|
||||
let mut enqueued_count = 0usize;
|
||||
let mut not_enqueued_count = 0usize;
|
||||
let mut dropped_count = 0usize;
|
||||
let mut full_count = 0usize;
|
||||
tokio::select! {
|
||||
_ = cancel_token.cancelled() => {
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
state = "shutdown",
|
||||
"Heal auto disk scanner stopped"
|
||||
);
|
||||
break;
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
// Build list of endpoints that need healing
|
||||
let mut endpoints = HashMap::<String, Vec<Endpoint>>::new();
|
||||
let mut durable_recoveries = HashMap::<String, (String, Vec<Endpoint>, Vec<String>, String)>::new();
|
||||
let mut conflicted_recovery_sets = HashSet::<String>::new();
|
||||
let mut deferred_replacement_endpoints = HashSet::<String>::new();
|
||||
let local_disks = {
|
||||
let local_disk_map = local_disk_map_read().await;
|
||||
local_disk_map.values().flatten().cloned().collect::<Vec<_>>()
|
||||
};
|
||||
let local_endpoints = local_disks.iter().map(|disk| disk.endpoint()).collect::<Vec<_>>();
|
||||
let blocked_sets = replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned")
|
||||
.clone();
|
||||
if !blocked_sets.is_empty() {
|
||||
let mut retry_succeeded = HashSet::new();
|
||||
let mut retry_failed = HashSet::new();
|
||||
for disk in &local_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
let Some(set_disk_id) =
|
||||
crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if !blocked_sets.contains(&set_disk_id) {
|
||||
continue;
|
||||
}
|
||||
match Self::validate_replacement_recovery_records(disk).await {
|
||||
Ok(()) => {
|
||||
retry_succeeded.insert(set_disk_id);
|
||||
}
|
||||
Err(error) => {
|
||||
retry_failed.insert(set_disk_id.clone());
|
||||
conflicted_recovery_sets.insert(set_disk_id);
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
error = %error,
|
||||
"Replacement recovery retry failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut blocked = replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned");
|
||||
unblock_replacement_recovery_sets_after_validation(&mut blocked, retry_succeeded, &retry_failed);
|
||||
}
|
||||
for disk in &local_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
let runtime_state = disk.runtime_state();
|
||||
let set_disk_id =
|
||||
crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx);
|
||||
if set_disk_id.as_ref().is_some_and(|set_disk_id| {
|
||||
replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned")
|
||||
.contains(set_disk_id)
|
||||
}) {
|
||||
skipped_invalid_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_DISK,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
set_disk_id = set_disk_id.as_deref().unwrap_or_default(),
|
||||
disk_state = "replacement_recovery_blocked",
|
||||
"Heal auto-scan replacement deferred because durable recovery is blocked"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// detect unformatted disk via get_disk_id()
|
||||
match disk.get_disk_id().await {
|
||||
Err(DiskError::UnformattedDisk) => {
|
||||
if !super::super::replacement_readiness::auto_replacement_target_ready(disk, &local_disks)
|
||||
.await
|
||||
{
|
||||
deferred_replacement_endpoints.insert(endpoint.to_string());
|
||||
skipped_invalid_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_DISK,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
disk_state = "replacement_path_unavailable",
|
||||
"Heal auto-scan replacement deferred"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let Some(set_disk_id) = set_disk_id else {
|
||||
skipped_invalid_count += 1;
|
||||
continue;
|
||||
};
|
||||
candidate_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_DISK,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
disk_state = "unformatted",
|
||||
"Heal auto-scan candidate detected"
|
||||
);
|
||||
endpoints.entry(set_disk_id).or_default().push(endpoint);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_DISK,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
disk_state = "check_failed",
|
||||
error = ?e,
|
||||
"Heal auto-scan disk inspection failed"
|
||||
);
|
||||
}
|
||||
Ok(_) => {
|
||||
if runtime_state.as_str() == "returning" && let Some(set_disk_id) = set_disk_id {
|
||||
candidate_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_DISK,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
set_disk_id,
|
||||
disk_state = "returning",
|
||||
"Heal auto-scan returning disk candidate detected"
|
||||
);
|
||||
endpoints.entry(set_disk_id).or_default().push(endpoint);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Once formatting succeeds a replacement is no longer
|
||||
// discoverable as UnformattedDisk. Re-admit exactly one
|
||||
// incomplete durable generation per set after bounded
|
||||
// scheduler retries are exhausted, or re-admit its
|
||||
// verified terminal cleanup. Multiple generations are a
|
||||
// durable conflict: leave every marker/state intact and
|
||||
// require reconciliation rather than choosing one.
|
||||
for disk in &local_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
let disk_set_disk_id =
|
||||
crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx);
|
||||
let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await {
|
||||
Ok(task_ids) => task_ids,
|
||||
Err(error) => {
|
||||
let endpoint_string = endpoint.to_string();
|
||||
if replacement_discovery_error_is_expected_for_deferred_endpoint(
|
||||
&error,
|
||||
&endpoint_string,
|
||||
&deferred_replacement_endpoints,
|
||||
) {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
disk_state = "replacement_path_unavailable",
|
||||
result = "recovery_records_unavailable",
|
||||
"Replacement recovery discovery skipped for deferred replacement"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if let Some(set_disk_id) = &disk_set_disk_id {
|
||||
conflicted_recovery_sets.insert(set_disk_id.clone());
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
error = %error,
|
||||
"Replacement recovery discovery failed"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for task_id in replacement_task_ids {
|
||||
let resume_manager = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await {
|
||||
Ok(resume_manager) => resume_manager,
|
||||
Err(error) => {
|
||||
if let Some(set_disk_id) = &disk_set_disk_id {
|
||||
conflicted_recovery_sets.insert(set_disk_id.clone());
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint = %endpoint,
|
||||
task_id,
|
||||
error = %error,
|
||||
"Replacement recovery intent load failed"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let state = resume_manager.get_state().await;
|
||||
if !durable_replacement_recovery_is_due(&state, &task_id) {
|
||||
continue;
|
||||
}
|
||||
if !matches!(state.replacement_phase, ReplacementPhase::CleanupPending) {
|
||||
let Ok(identities) = storage.replacement_target_identities(&state.replacement_targets).await else {
|
||||
continue;
|
||||
};
|
||||
if identities != state.replacement_target_identities {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let targets = state
|
||||
.replacement_targets
|
||||
.iter()
|
||||
.filter_map(|target| {
|
||||
local_endpoints
|
||||
.iter()
|
||||
.find(|endpoint| endpoint.to_string() == *target)
|
||||
.cloned()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if targets.len() != state.replacement_targets.len() {
|
||||
continue;
|
||||
}
|
||||
let Some(set_disk_id) = crate::heal::utils::format_set_disk_id_from_i32(
|
||||
targets[0].pool_idx,
|
||||
targets[0].set_idx,
|
||||
) else {
|
||||
continue;
|
||||
};
|
||||
if targets.iter().any(|target| {
|
||||
crate::heal::utils::format_set_disk_id_from_i32(target.pool_idx, target.set_idx)
|
||||
.as_deref()
|
||||
!= Some(set_disk_id.as_str())
|
||||
}) {
|
||||
continue;
|
||||
}
|
||||
let resume_endpoint = disk.endpoint().to_string();
|
||||
match durable_recoveries.get(&set_disk_id) {
|
||||
Some((existing_task_id, _, _, existing_anchor))
|
||||
if existing_task_id != &task_id || existing_anchor != &resume_endpoint => {
|
||||
replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned")
|
||||
.insert(set_disk_id.clone());
|
||||
conflicted_recovery_sets.insert(set_disk_id);
|
||||
}
|
||||
Some(_) => {}
|
||||
None => {
|
||||
durable_recoveries.insert(
|
||||
set_disk_id,
|
||||
(task_id, targets, state.replacement_buckets, resume_endpoint),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for set_disk_id in &conflicted_recovery_sets {
|
||||
durable_recoveries.remove(set_disk_id);
|
||||
endpoints.remove(set_disk_id);
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
set_disk_id,
|
||||
result = "durable_generation_conflict",
|
||||
"Replacement recovery deferred because multiple durable generations exist"
|
||||
);
|
||||
}
|
||||
|
||||
for (set_disk_id, (_, targets, _, _)) in &durable_recoveries {
|
||||
let expected = targets.iter().map(ToString::to_string).collect::<HashSet<_>>();
|
||||
let observed = endpoints
|
||||
.get(set_disk_id)
|
||||
.map(|endpoints| endpoints.iter().map(ToString::to_string).collect::<HashSet<_>>())
|
||||
.unwrap_or_default();
|
||||
if !observed.is_subset(&expected) {
|
||||
replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned")
|
||||
.insert(set_disk_id.clone());
|
||||
conflicted_recovery_sets.insert(set_disk_id.clone());
|
||||
continue;
|
||||
}
|
||||
endpoints.entry(set_disk_id.clone()).or_default().extend(targets.clone());
|
||||
}
|
||||
for set_disk_id in &conflicted_recovery_sets {
|
||||
durable_recoveries.remove(set_disk_id);
|
||||
endpoints.remove(set_disk_id);
|
||||
}
|
||||
|
||||
for target_endpoints in endpoints.values_mut() {
|
||||
target_endpoints.sort_by_key(ToString::to_string);
|
||||
target_endpoints.dedup_by(|left, right| left.to_string() == right.to_string());
|
||||
}
|
||||
|
||||
if endpoints.is_empty() {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
state = "idle",
|
||||
"Heal auto disk scanner idle"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Admit one set task with every ready replacement target. Queue deduplication is
|
||||
// set-scoped, so admitting endpoints independently would silently drop later targets.
|
||||
for (set_disk_id, endpoints) in endpoints {
|
||||
if replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned")
|
||||
.contains(&set_disk_id)
|
||||
{
|
||||
skipped_invalid_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
set_disk_id,
|
||||
result = "replacement_recovery_blocked",
|
||||
"Heal auto-scan replacement admission deferred because durable recovery is blocked"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
// skip if already queued or healing
|
||||
// Use consistent lock order: queue first, then active_heals to avoid deadlock
|
||||
let mut skip = false;
|
||||
{
|
||||
let queue = heal_queue.lock().await;
|
||||
if queue.contains_erasure_set(&set_disk_id) {
|
||||
skip = true;
|
||||
}
|
||||
}
|
||||
if !skip {
|
||||
let active = active_heals.lock().await;
|
||||
if active.values().any(|task| {
|
||||
matches!(
|
||||
&task.heal_type,
|
||||
crate::heal::task::HealType::ErasureSet { set_disk_id: active_id, .. }
|
||||
if active_id == &set_disk_id
|
||||
)
|
||||
}) {
|
||||
skip = true;
|
||||
}
|
||||
}
|
||||
|
||||
if skip {
|
||||
skipped_duplicate_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint_count = endpoints.len(),
|
||||
set_disk_id,
|
||||
result = "skipped_duplicate",
|
||||
"Heal auto-scan duplicate skipped"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// enqueue erasure set heal request for all ready replacements in this set
|
||||
let recovery = durable_recoveries.remove(&set_disk_id);
|
||||
let mut req = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: recovery
|
||||
.as_ref()
|
||||
.map(|(_, _, buckets, _)| buckets.clone())
|
||||
.unwrap_or_default(),
|
||||
set_disk_id: set_disk_id.clone(),
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: endpoints
|
||||
.first()
|
||||
.and_then(|endpoint| usize::try_from(endpoint.pool_idx).ok()),
|
||||
set_index: endpoints
|
||||
.first()
|
||||
.and_then(|endpoint| usize::try_from(endpoint.set_idx).ok()),
|
||||
timeout: None,
|
||||
..HealOptions::default()
|
||||
},
|
||||
HealPriority::Low,
|
||||
);
|
||||
let recovery_anchor = recovery.as_ref().map(|(_, _, _, anchor)| anchor.clone());
|
||||
if let Some((task_id, _, _, _)) = recovery {
|
||||
req.id = task_id;
|
||||
}
|
||||
req.source = HealRequestSource::AutoHeal;
|
||||
req.heal_endpoints = endpoints.iter().map(ToString::to_string).collect();
|
||||
let request_id = req.id.clone();
|
||||
let endpoint_count = req.heal_endpoints.len();
|
||||
let config = config.read().await;
|
||||
let mut queue = heal_queue.lock().await;
|
||||
let admission_decision = Self::admit_request_to_queue(&mut queue, req, &config, "auto_scan");
|
||||
let admission = admission_decision.result;
|
||||
let should_notify =
|
||||
matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable;
|
||||
if matches!(admission, HealAdmissionResult::Accepted)
|
||||
&& let Some(anchor) = recovery_anchor
|
||||
{
|
||||
replacement_recovery_anchors
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.insert(request_id, anchor);
|
||||
}
|
||||
drop(queue);
|
||||
drop(config);
|
||||
if let Some(displaced_task_id) = admission_decision.displaced_task_id {
|
||||
remove_task_aliases_for_task(&task_aliases, &displaced_task_id).await;
|
||||
lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id);
|
||||
}
|
||||
if matches!(admission, HealAdmissionResult::Accepted) {
|
||||
if should_notify {
|
||||
notify.notify_one();
|
||||
}
|
||||
enqueued_count += 1;
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint_count,
|
||||
set_disk_id,
|
||||
bucket_count = 0,
|
||||
result = "enqueued",
|
||||
"Heal auto-scan task enqueued"
|
||||
);
|
||||
} else {
|
||||
if matches!(admission, HealAdmissionResult::Merged) {
|
||||
skipped_duplicate_count += 1;
|
||||
} else {
|
||||
not_enqueued_count += 1;
|
||||
}
|
||||
if matches!(admission, HealAdmissionResult::Full) {
|
||||
full_count += 1;
|
||||
}
|
||||
if matches!(admission, HealAdmissionResult::Dropped(_)) {
|
||||
dropped_count += 1;
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_ENQUEUE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
endpoint_count,
|
||||
set_disk_id,
|
||||
bucket_count = 0,
|
||||
admission = admission.result_label(),
|
||||
reason = admission.reason_label(),
|
||||
result = "not_enqueued",
|
||||
"Heal auto-scan task not enqueued"
|
||||
);
|
||||
}
|
||||
}
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_AUTO_SCAN_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_SCANNER,
|
||||
state = "cycle_completed",
|
||||
candidate_count,
|
||||
enqueued_count,
|
||||
not_enqueued_count,
|
||||
dropped_count,
|
||||
full_count,
|
||||
skipped_duplicate_count,
|
||||
skipped_invalid_count,
|
||||
"Heal auto-scan cycle completed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,458 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// The priority heal queue and its per-key dedup index.
|
||||
use super::*;
|
||||
|
||||
/// Per-key bookkeeping for the queued-request dedup index: how many queued
|
||||
/// requests hold the key, and the id of the first request that opened it —
|
||||
/// the O(1) stand-in for the former heap scan when a merge receipt needs to
|
||||
/// name a queued representative.
|
||||
#[derive(Debug)]
|
||||
pub(super) struct DedupKeyEntry {
|
||||
pub(super) refcount: usize,
|
||||
pub(super) representative_request_id: String,
|
||||
}
|
||||
|
||||
/// Priority queue wrapper for heal requests
|
||||
/// Uses BinaryHeap for priority-based ordering while maintaining FIFO for same-priority items
|
||||
#[derive(Debug)]
|
||||
pub(super) struct PriorityHealQueue {
|
||||
/// Heap of (priority, sequence, request) tuples
|
||||
pub(super) heap: BinaryHeap<PriorityQueueItem>,
|
||||
/// Sequence counter for FIFO ordering within same priority
|
||||
pub(super) sequence: u64,
|
||||
/// Deduplication index for queued requests
|
||||
pub(super) dedup_keys: HashMap<String, DedupKeyEntry>,
|
||||
}
|
||||
|
||||
/// Wrapper for heap items to implement proper ordering
|
||||
#[derive(Debug)]
|
||||
pub(super) struct PriorityQueueItem {
|
||||
pub(super) priority: HealPriority,
|
||||
pub(super) sequence: u64,
|
||||
pub(super) dedup_key: String,
|
||||
pub(super) request: HealRequest,
|
||||
}
|
||||
|
||||
impl Eq for PriorityQueueItem {}
|
||||
|
||||
impl PartialEq for PriorityQueueItem {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.priority == other.priority && self.sequence == other.sequence
|
||||
}
|
||||
}
|
||||
|
||||
impl Ord for PriorityQueueItem {
|
||||
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||
// First compare by priority (higher priority first)
|
||||
match self.priority.cmp(&other.priority) {
|
||||
std::cmp::Ordering::Equal => {
|
||||
// If priorities are equal, use sequence for FIFO (lower sequence first)
|
||||
other.sequence.cmp(&self.sequence)
|
||||
}
|
||||
ordering => ordering,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialOrd for PriorityQueueItem {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum QueuePushOutcome {
|
||||
Accepted,
|
||||
Merged,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) struct ForegroundPressure {
|
||||
pub(super) class: WorkloadClass,
|
||||
pub(super) usage_pct: usize,
|
||||
pub(super) threshold_pct: usize,
|
||||
}
|
||||
|
||||
impl ForegroundPressure {
|
||||
pub(super) const fn reason(self) -> &'static str {
|
||||
match self.class {
|
||||
WorkloadClass::ForegroundRead => "foreground_read_pressure",
|
||||
WorkloadClass::ForegroundWrite => "foreground_write_pressure",
|
||||
_ => "foreground_pressure",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct CompletedHealStatus {
|
||||
pub(super) heal_type: HealType,
|
||||
pub(super) status: HealTaskStatus,
|
||||
pub(super) result_items_truncated: bool,
|
||||
pub(super) completed_at: SystemTime,
|
||||
/// Sequence-stamped retained window, archived with the completion so
|
||||
/// incremental consumers keep their cursor across the transition (HS-06).
|
||||
/// The un-stamped legacy view is derived from it on demand.
|
||||
pub(super) seqed_items: Vec<(u64, HealResultItem)>,
|
||||
pub(super) next_seq: u64,
|
||||
pub(super) min_seq: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct HealTaskAlias {
|
||||
pub(super) task_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct RetryingHeal {
|
||||
pub(super) request: HealRequest,
|
||||
pub(super) error: String,
|
||||
pub(super) cancel_token: CancellationToken,
|
||||
}
|
||||
|
||||
impl PriorityHealQueue {
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
heap: BinaryHeap::new(),
|
||||
sequence: 0,
|
||||
dedup_keys: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn len(&self) -> usize {
|
||||
self.heap.len()
|
||||
}
|
||||
|
||||
pub(super) fn pop_next(&mut self) -> Option<HealRequest> {
|
||||
self.heap.pop().map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
item.request
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn is_empty(&self) -> bool {
|
||||
self.heap.is_empty()
|
||||
}
|
||||
|
||||
pub(super) fn push(&mut self, request: HealRequest) -> QueuePushOutcome {
|
||||
let key = Self::make_dedup_key(&request);
|
||||
|
||||
// Check for duplicates unless the caller explicitly forces admission.
|
||||
if self.dedup_keys.contains_key(&key) && !request.force_start {
|
||||
return QueuePushOutcome::Merged;
|
||||
}
|
||||
// Track dedup keys for both normal and forced requests so queued forced work
|
||||
// also reserves the dedup key for later non-forced duplicates. The first
|
||||
// request that opens the key becomes the named representative for merge
|
||||
// receipts (taken before `request` moves into the heap).
|
||||
self.dedup_keys
|
||||
.entry(key.clone())
|
||||
.or_insert_with(|| DedupKeyEntry {
|
||||
refcount: 0,
|
||||
representative_request_id: request.id.clone(),
|
||||
})
|
||||
.refcount += 1;
|
||||
self.sequence += 1;
|
||||
self.heap.push(PriorityQueueItem {
|
||||
priority: request.priority,
|
||||
sequence: self.sequence,
|
||||
dedup_key: key,
|
||||
request,
|
||||
});
|
||||
QueuePushOutcome::Accepted
|
||||
}
|
||||
|
||||
pub(super) fn can_displace_lower_priority(&self, priority: HealPriority) -> bool {
|
||||
self.heap.iter().any(|item| item.priority < priority)
|
||||
}
|
||||
|
||||
pub(super) fn push_displacing_lower_priority(&mut self, request: HealRequest) -> Option<HealRequest> {
|
||||
let mut retained = BinaryHeap::new();
|
||||
let mut displaced: Option<PriorityQueueItem> = None;
|
||||
|
||||
while let Some(item) = self.heap.pop() {
|
||||
if item.priority < request.priority {
|
||||
let should_displace = displaced
|
||||
.as_ref()
|
||||
.map(|current| {
|
||||
item.priority < current.priority
|
||||
|| (item.priority == current.priority && item.sequence > current.sequence)
|
||||
})
|
||||
.unwrap_or(true);
|
||||
if should_displace {
|
||||
if let Some(current) = displaced.replace(item) {
|
||||
retained.push(current);
|
||||
}
|
||||
} else {
|
||||
retained.push(item);
|
||||
}
|
||||
} else {
|
||||
retained.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
self.heap = retained;
|
||||
|
||||
let displaced = displaced.map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
self.refresh_dedup_representative(&item.dedup_key);
|
||||
item.request
|
||||
});
|
||||
|
||||
if displaced.is_some() {
|
||||
// The enqueue side effect must run in ALL builds. Do NOT fold `self.push(request)`
|
||||
// into `debug_assert_eq!` — in release builds (`debug_assertions` off) the whole
|
||||
// macro, including its argument expression, is compiled out, which would silently
|
||||
// drop the new high-priority request after having already evicted a queued item.
|
||||
let outcome = self.push(request);
|
||||
debug_assert_eq!(outcome, QueuePushOutcome::Accepted);
|
||||
}
|
||||
|
||||
displaced
|
||||
}
|
||||
|
||||
/// Get statistics about queue contents by priority
|
||||
pub(super) fn get_priority_stats(&self) -> HashMap<HealPriority, usize> {
|
||||
let mut stats = HashMap::new();
|
||||
for item in &self.heap {
|
||||
*stats.entry(item.priority).or_insert(0) += 1;
|
||||
}
|
||||
stats
|
||||
}
|
||||
|
||||
pub(super) fn operation_counts(&self) -> (HealPriorityCounts, HealSourceCounts) {
|
||||
let mut priority = HealPriorityCounts::default();
|
||||
let mut source = HealSourceCounts::default();
|
||||
for item in &self.heap {
|
||||
priority.increment(item.request.priority);
|
||||
source.increment(item.request.source);
|
||||
}
|
||||
(priority, source)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn pop(&mut self) -> Option<HealRequest> {
|
||||
self.heap.pop().map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
item.request
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn pop_runnable_with_skips<F, G>(&mut self, can_run: F, skip_label: G) -> (Option<HealRequest>, Vec<String>)
|
||||
where
|
||||
F: Fn(&HealRequest) -> bool,
|
||||
G: Fn(&HealRequest) -> Option<String>,
|
||||
{
|
||||
let mut deferred = Vec::new();
|
||||
let mut selected = None;
|
||||
let mut skipped = Vec::new();
|
||||
|
||||
while let Some(item) = self.heap.pop() {
|
||||
if can_run(&item.request) {
|
||||
selected = Some(item);
|
||||
break;
|
||||
}
|
||||
if let Some(label) = skip_label(&item.request) {
|
||||
skipped.push(label);
|
||||
}
|
||||
deferred.push(item);
|
||||
}
|
||||
|
||||
self.restore_deferred_items(deferred);
|
||||
|
||||
(
|
||||
selected.map(|item| {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
item.request
|
||||
}),
|
||||
skipped,
|
||||
)
|
||||
}
|
||||
|
||||
fn restore_deferred_items(&mut self, deferred: Vec<PriorityQueueItem>) {
|
||||
if deferred.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
if deferred.len() > self.heap.len() / 2 {
|
||||
let mut items = std::mem::take(&mut self.heap).into_vec();
|
||||
items.reserve(deferred.len());
|
||||
items.extend(deferred);
|
||||
self.heap = BinaryHeap::from(items);
|
||||
} else {
|
||||
for item in deferred {
|
||||
self.heap.push(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a deduplication key from a heal request
|
||||
pub(super) fn make_dedup_key(request: &HealRequest) -> String {
|
||||
Self::make_dedup_key_for_type(&request.heal_type)
|
||||
}
|
||||
|
||||
pub(super) fn make_dedup_key_for_type(heal_type: &HealType) -> String {
|
||||
match heal_type {
|
||||
HealType::Cluster => "cluster".to_string(),
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
format!("object:{}:{}:{}", bucket, object, version_id.as_deref().unwrap_or(""))
|
||||
}
|
||||
HealType::Bucket { bucket } => {
|
||||
format!("bucket:{bucket}")
|
||||
}
|
||||
HealType::Prefix { bucket, prefix } => {
|
||||
format!("prefix:{bucket}/{prefix}")
|
||||
}
|
||||
HealType::ErasureSet { set_disk_id, .. } => {
|
||||
format!("erasure_set:{set_disk_id}")
|
||||
}
|
||||
HealType::Metadata { bucket, object } => {
|
||||
format!("metadata:{bucket}:{object}")
|
||||
}
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
format!("ecdecode:{}:{}:{}", bucket, object, version_id.as_deref().unwrap_or(""))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn decrement_or_remove_dedup_key(dedup_keys: &mut HashMap<String, DedupKeyEntry>, key: &str) {
|
||||
if let Some(entry) = dedup_keys.get_mut(key) {
|
||||
if entry.refcount <= 1 {
|
||||
dedup_keys.remove(key);
|
||||
} else {
|
||||
entry.refcount -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Check if an erasure set heal request for a specific set_disk_id exists
|
||||
pub(super) fn contains_erasure_set(&self, set_disk_id: &str) -> bool {
|
||||
let key = format!("erasure_set:{set_disk_id}");
|
||||
self.dedup_keys.contains_key(&key)
|
||||
}
|
||||
|
||||
/// Iterate queued requests (used by the admin overlap check).
|
||||
pub(super) fn requests(&self) -> impl Iterator<Item = &HealRequest> {
|
||||
self.heap.iter().map(|item| &item.request)
|
||||
}
|
||||
|
||||
pub(super) fn contains_request_id(&self, request_id: &str) -> bool {
|
||||
self.heap.iter().any(|item| item.request.id == request_id)
|
||||
}
|
||||
|
||||
pub(super) fn contains_request_id_matching_path(&self, request_id: &str, heal_path: &str) -> bool {
|
||||
self.heap
|
||||
.iter()
|
||||
.any(|item| item.request.id == request_id && heal_type_matches_path(&item.request.heal_type, heal_path))
|
||||
}
|
||||
|
||||
pub(super) fn queued_request_id_for_dedup_key(&self, key: &str) -> Option<&str> {
|
||||
self.dedup_keys.get(key).map(|entry| entry.representative_request_id.as_str())
|
||||
}
|
||||
|
||||
/// Re-elect the representative for `key` from the queue entries holding
|
||||
/// it. Needed after a holder leaves the queue *without* becoming active
|
||||
/// (canceled by id, or displaced): the former opener may be the request
|
||||
/// that just left, and a merge receipt must never name an id that
|
||||
/// resolves nowhere. The scheduler pop path does not need this — the
|
||||
/// popped request surfaces in `active_heals` under the same id and the
|
||||
/// duplicate pre-check consults active heals before the queue. No-op for
|
||||
/// released keys; the survivor scan only runs when a key still has
|
||||
/// holders, which under forced duplicates is the rare admin path.
|
||||
pub(super) fn refresh_dedup_representative(&mut self, key: &str) {
|
||||
if !self.dedup_keys.contains_key(key) {
|
||||
return;
|
||||
}
|
||||
if let Some(id) = self
|
||||
.heap
|
||||
.iter()
|
||||
.find(|item| item.dedup_key == key)
|
||||
.map(|item| item.request.id.clone())
|
||||
&& let Some(entry) = self.dedup_keys.get_mut(key)
|
||||
{
|
||||
entry.representative_request_id = id;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn contains_matching<F>(&self, mut matches: F) -> bool
|
||||
where
|
||||
F: FnMut(&HealRequest) -> bool,
|
||||
{
|
||||
self.heap.iter().any(|item| matches(&item.request))
|
||||
}
|
||||
|
||||
pub(super) fn remove_request_id(&mut self, request_id: &str) -> Option<HealRequest> {
|
||||
let mut retained = BinaryHeap::new();
|
||||
let mut removed = None;
|
||||
let mut affected_key = None;
|
||||
|
||||
while let Some(item) = self.heap.pop() {
|
||||
if removed.is_none() && item.request.id == request_id {
|
||||
let key = item.dedup_key.clone();
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key);
|
||||
affected_key = Some(key);
|
||||
removed = Some(item.request);
|
||||
} else {
|
||||
retained.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
self.heap = retained;
|
||||
if let Some(key) = affected_key.as_deref() {
|
||||
self.refresh_dedup_representative(key);
|
||||
}
|
||||
removed
|
||||
}
|
||||
|
||||
pub(super) fn remove_matching<F>(&mut self, mut should_remove: F) -> Vec<HealRequest>
|
||||
where
|
||||
F: FnMut(&HealRequest) -> bool,
|
||||
{
|
||||
let mut retained = BinaryHeap::new();
|
||||
let mut removed = Vec::new();
|
||||
let mut affected_keys = Vec::new();
|
||||
|
||||
while let Some(item) = self.heap.pop() {
|
||||
if should_remove(&item.request) {
|
||||
Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &item.dedup_key);
|
||||
affected_keys.push(item.dedup_key);
|
||||
removed.push(item.request);
|
||||
} else {
|
||||
retained.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
self.heap = retained;
|
||||
for key in &affected_keys {
|
||||
self.refresh_dedup_representative(key);
|
||||
}
|
||||
removed
|
||||
}
|
||||
}
|
||||
|
||||
impl RetryingHeal {
|
||||
pub(super) fn status(&self) -> HealTaskStatus {
|
||||
HealTaskStatus::Retrying {
|
||||
error: self.error.clone(),
|
||||
retry_attempt: self.request.retry_attempts,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,686 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// The heal scheduler: queue consumption loop and its skip/metric helpers.
|
||||
use super::*;
|
||||
|
||||
impl HealManager {
|
||||
/// Start scheduler
|
||||
pub(super) async fn start_scheduler(&self) -> Result<()> {
|
||||
let config = self.config.clone();
|
||||
let heal_queue = self.heal_queue.clone();
|
||||
let active_heals = self.active_heals.clone();
|
||||
let completed_heals = self.completed_heals.clone();
|
||||
let task_aliases = self.task_aliases.clone();
|
||||
let retrying_heals = self.retrying_heals.clone();
|
||||
let mrf_repair_notice_targets = self.mrf_repair_notice_targets.clone();
|
||||
let replacement_recovery_anchors = self.replacement_recovery_anchors.clone();
|
||||
let cancel_token = self.cancel_token.clone();
|
||||
let statistics = self.statistics.clone();
|
||||
let storage = self.storage.clone();
|
||||
let notify = self.notify.clone();
|
||||
let workload_provider = self.workload_provider.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval = interval(config.read().await.heal_interval);
|
||||
|
||||
loop {
|
||||
let event_driven_scheduler_enable = config.read().await.event_driven_scheduler_enable;
|
||||
tokio::select! {
|
||||
_ = cancel_token.cancelled() => {
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_SCHEDULER_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
state = "shutdown",
|
||||
"Heal scheduler stopped"
|
||||
);
|
||||
break;
|
||||
}
|
||||
_ = notify.notified(), if event_driven_scheduler_enable => {
|
||||
Self::process_heal_queue(HealQueueContext {
|
||||
heal_queue: &heal_queue,
|
||||
active_heals: &active_heals,
|
||||
completed_heals: &completed_heals,
|
||||
task_aliases: &task_aliases,
|
||||
retrying_heals: &retrying_heals,
|
||||
mrf_repair_notice_targets: &mrf_repair_notice_targets,
|
||||
replacement_recovery_anchors: &replacement_recovery_anchors,
|
||||
config: &config,
|
||||
statistics: &statistics,
|
||||
storage: &storage,
|
||||
notify: ¬ify,
|
||||
cancel_token: &cancel_token,
|
||||
workload_provider: &workload_provider,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
Self::process_heal_queue(HealQueueContext {
|
||||
heal_queue: &heal_queue,
|
||||
active_heals: &active_heals,
|
||||
completed_heals: &completed_heals,
|
||||
task_aliases: &task_aliases,
|
||||
retrying_heals: &retrying_heals,
|
||||
mrf_repair_notice_targets: &mrf_repair_notice_targets,
|
||||
replacement_recovery_anchors: &replacement_recovery_anchors,
|
||||
config: &config,
|
||||
statistics: &statistics,
|
||||
storage: &storage,
|
||||
notify: ¬ify,
|
||||
cancel_token: &cancel_token,
|
||||
workload_provider: &workload_provider,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Process heal queue
|
||||
/// Processes multiple tasks per cycle when capacity allows and queue has high-priority items
|
||||
pub(super) async fn process_heal_queue(context: HealQueueContext<'_>) {
|
||||
let HealQueueContext {
|
||||
heal_queue,
|
||||
active_heals,
|
||||
completed_heals,
|
||||
task_aliases,
|
||||
retrying_heals,
|
||||
mrf_repair_notice_targets,
|
||||
replacement_recovery_anchors,
|
||||
config,
|
||||
statistics,
|
||||
storage,
|
||||
notify,
|
||||
cancel_token,
|
||||
workload_provider,
|
||||
} = context;
|
||||
|
||||
let config = config.read().await;
|
||||
let mainline_pressure = Self::mainline_throttle_active(&config, workload_provider);
|
||||
let mut active_heals_guard = active_heals.lock().await;
|
||||
publish_active_heal_count(&active_heals_guard);
|
||||
|
||||
// Check if new heal tasks can be started
|
||||
let active_count = active_heals_guard.len();
|
||||
if active_count >= config.max_concurrent_heals {
|
||||
return;
|
||||
}
|
||||
|
||||
// Calculate how many tasks we can start this cycle
|
||||
let available_slots = config.max_concurrent_heals - active_count;
|
||||
|
||||
let mut queue = heal_queue.lock().await;
|
||||
let queue_len = queue.len();
|
||||
publish_heal_queue_length(&queue);
|
||||
|
||||
if queue_len == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut running_per_set = running_heal_set_counts(&active_heals_guard);
|
||||
let mut tasks_started = 0usize;
|
||||
let mut delayed_by_mainline_throttle = false;
|
||||
|
||||
for _ in 0..available_slots {
|
||||
let selected_request = if config.set_bulkhead_enable || mainline_pressure.is_some() {
|
||||
let max_concurrent_per_set = config.max_concurrent_per_set;
|
||||
let (selected_request, skipped_sets) = queue.pop_runnable_with_skips(
|
||||
|request| {
|
||||
let set_allowed = !config.set_bulkhead_enable
|
||||
|| can_schedule_request(request, &running_per_set, max_concurrent_per_set);
|
||||
let mainline_allowed = mainline_pressure.is_none() || Self::request_bypasses_mainline_throttle(request);
|
||||
set_allowed && mainline_allowed
|
||||
},
|
||||
|request| heal_request_set_key(request).map(|_| heal_request_set_metric_label(request)),
|
||||
);
|
||||
for skipped_set in skipped_sets {
|
||||
record_scheduler_skip(&skipped_set);
|
||||
}
|
||||
selected_request
|
||||
} else {
|
||||
queue.pop_next()
|
||||
};
|
||||
|
||||
if let Some(mut request) = selected_request {
|
||||
request.options.timeout.get_or_insert(config.task_timeout);
|
||||
let task_priority = request.priority;
|
||||
let task_type_label = heal_request_type_label(&request).to_string();
|
||||
let task_set_label = heal_request_set_metric_label(&request);
|
||||
if config.set_bulkhead_enable
|
||||
&& let Some(set_key) = heal_request_set_key(&request)
|
||||
{
|
||||
*running_per_set.entry(set_key).or_insert(0) += 1;
|
||||
}
|
||||
let replacement_resume_endpoint = replacement_recovery_anchors
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.get(&request.id)
|
||||
.cloned();
|
||||
let task = Arc::new(HealTask::from_replacement_recovery_request(
|
||||
request,
|
||||
storage.clone(),
|
||||
replacement_resume_endpoint,
|
||||
));
|
||||
let task_id = task.id.clone();
|
||||
active_heals_guard.insert(task_id.clone(), task.clone());
|
||||
publish_active_heal_count(&active_heals_guard);
|
||||
update_task_running_metric_for_task(&active_heals_guard, task.as_ref());
|
||||
let active_heals_clone = active_heals.clone();
|
||||
let heal_queue_clone = heal_queue.clone();
|
||||
let completed_heals_clone = completed_heals.clone();
|
||||
let task_aliases_clone = task_aliases.clone();
|
||||
let retrying_heals_clone = retrying_heals.clone();
|
||||
let mrf_repair_notice_targets_clone = mrf_repair_notice_targets.clone();
|
||||
let replacement_recovery_anchors_clone = replacement_recovery_anchors.clone();
|
||||
let statistics_clone = statistics.clone();
|
||||
let notify_clone = notify.clone();
|
||||
let manager_cancel_token = cancel_token.clone();
|
||||
let task_type_label_for_spawn = task_type_label.clone();
|
||||
let task_set_label_for_spawn = task_set_label.clone();
|
||||
let config_for_spawn = config.clone();
|
||||
|
||||
// start heal task
|
||||
tokio::spawn(async move {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_SCHEDULER_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
task_id,
|
||||
priority = ?task_priority,
|
||||
heal_type = %task_type_label_for_spawn,
|
||||
set = %task_set_label_for_spawn,
|
||||
state = "task_started",
|
||||
"Heal scheduler task started"
|
||||
);
|
||||
let result = task.execute().await;
|
||||
let retry_request = retry_request_for_result_with_budget(task.as_ref(), &result).await;
|
||||
match &result {
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_SCHEDULER_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
task_id,
|
||||
heal_type = %task_type_label_for_spawn,
|
||||
set = %task_set_label_for_spawn,
|
||||
state = "task_completed",
|
||||
"Heal scheduler task completed"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
let will_retry = retry_request.is_some();
|
||||
if will_retry {
|
||||
demote_to_debug_when!(task.heal_type.is_per_object(), warn, target: "rustfs::heal::manager", {
|
||||
event = EVENT_HEAL_SCHEDULER_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
task_id,
|
||||
heal_type = %task_type_label_for_spawn,
|
||||
set = %task_set_label_for_spawn,
|
||||
state = "task_retrying",
|
||||
retry_attempt = task.retry_attempts.saturating_add(1),
|
||||
error = %e,
|
||||
"Heal scheduler task retrying"
|
||||
});
|
||||
} else {
|
||||
error!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_SCHEDULER_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
task_id,
|
||||
heal_type = %task_type_label_for_spawn,
|
||||
set = %task_set_label_for_spawn,
|
||||
state = "task_failed",
|
||||
error = %e,
|
||||
"Heal scheduler task failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let retry_request_for_status = retry_request.as_ref().map(|(request, _, error)| HealTaskStatus::Retrying {
|
||||
error: error.clone(),
|
||||
retry_attempt: request.retry_attempts,
|
||||
});
|
||||
let retry_request_for_queue = retry_request;
|
||||
let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new());
|
||||
if retry_request_for_queue.is_none() {
|
||||
replacement_recovery_anchors_clone
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.remove(&task_id);
|
||||
}
|
||||
let mut active_heals_guard = active_heals_clone.lock().await;
|
||||
// Keep retry ownership continuous: status snapshots acquire
|
||||
// these locks in the same active -> retrying order.
|
||||
let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) =
|
||||
(retry_request_for_queue.as_ref(), retry_cancel_token.as_ref())
|
||||
{
|
||||
let mut retrying = retrying_heals_clone.lock().await;
|
||||
if active_heals_guard.contains_key(&task_id) {
|
||||
retrying.insert(
|
||||
request.id.clone(),
|
||||
RetryingHeal {
|
||||
request: request.clone(),
|
||||
error: error.clone(),
|
||||
cancel_token: cancel_token.clone(),
|
||||
},
|
||||
);
|
||||
#[cfg(test)]
|
||||
pause_retry_ownership_transition(&task_id, false).await;
|
||||
}
|
||||
Some(retrying)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let completed_task = active_heals_guard.remove(&task_id);
|
||||
if let Some(completed_task) = completed_task.as_ref() {
|
||||
publish_active_heal_count(&active_heals_guard);
|
||||
update_task_running_metric_for_task(&active_heals_guard, completed_task.as_ref());
|
||||
}
|
||||
let active_count = active_heals_guard.len();
|
||||
drop(retrying_heals_guard.take());
|
||||
drop(active_heals_guard);
|
||||
|
||||
if let Some(completed_task) = completed_task {
|
||||
let completed_status = if let Some(status) = retry_request_for_status {
|
||||
status
|
||||
} else {
|
||||
completed_task.get_status().await
|
||||
};
|
||||
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
|
||||
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
|
||||
let completed_progress = completed_task.get_progress().await;
|
||||
// Single snapshot of the retained window: the task is
|
||||
// finished and already off the active map, so there is
|
||||
// no concurrent writer to race with.
|
||||
let seqed_items = completed_task.get_seqed_result_items().await;
|
||||
let (next_seq, min_seq) = completed_task.result_seq_cursors();
|
||||
let completed_status_entry = CompletedHealStatus {
|
||||
heal_type: completed_task.heal_type.clone(),
|
||||
status: completed_status.clone(),
|
||||
result_items_truncated: completed_task.result_items_truncated(),
|
||||
completed_at: SystemTime::now(),
|
||||
seqed_items,
|
||||
next_seq,
|
||||
min_seq,
|
||||
};
|
||||
let mut completed_heals_guard = completed_heals_clone.lock().await;
|
||||
prune_completed_heal_statuses(&mut completed_heals_guard);
|
||||
completed_heals_guard.insert(task_id.clone(), Arc::new(completed_status_entry));
|
||||
drop(completed_heals_guard);
|
||||
// update statistics
|
||||
let mut stats = statistics_clone.write().await;
|
||||
match completed_status {
|
||||
HealTaskStatus::Completed => {
|
||||
stats.update_task_completion(true);
|
||||
stats.add_healed_objects(completed_progress.objects_healed, completed_progress.bytes_processed);
|
||||
}
|
||||
HealTaskStatus::Retrying { .. } => {}
|
||||
_ => {
|
||||
stats.update_task_completion(false);
|
||||
}
|
||||
}
|
||||
stats.update_running_tasks(usize_to_u64_saturated(active_count));
|
||||
drop(stats);
|
||||
if terminal_completion {
|
||||
let notice_targets = take_mrf_repair_notice_targets(&mrf_repair_notice_targets_clone, &task_id);
|
||||
if successful_completion {
|
||||
emit_mrf_repaired_events(notice_targets);
|
||||
}
|
||||
task_aliases_clone
|
||||
.lock()
|
||||
.await
|
||||
.retain(|alias_id, alias| alias_id != &task_id && alias.task_id != task_id);
|
||||
}
|
||||
}
|
||||
|
||||
if let (Some((retry_request, retry_delay, retry_error)), Some(retry_cancel_token)) =
|
||||
(retry_request_for_queue, retry_cancel_token)
|
||||
{
|
||||
let retry_request_id = retry_request.id.clone();
|
||||
let retry_attempt = retry_request.retry_attempts;
|
||||
let retry_key = PriorityHealQueue::make_dedup_key(&retry_request);
|
||||
let retry_priority = retry_request.priority;
|
||||
let retry_active_heals = active_heals_clone.clone();
|
||||
let retry_heal_queue = heal_queue_clone.clone();
|
||||
let retrying_heals_for_spawn = retrying_heals_clone.clone();
|
||||
let retry_task_aliases = task_aliases_clone.clone();
|
||||
let retry_mrf_repair_notice_targets = mrf_repair_notice_targets_clone.clone();
|
||||
let retry_completed_heals = completed_heals_clone.clone();
|
||||
let retry_notify = notify_clone.clone();
|
||||
let retry_manager_cancel_token = manager_cancel_token.clone();
|
||||
let retry_config = config_for_spawn.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = retry_cancel_token.cancelled() => {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %retry_request_id,
|
||||
priority = ?retry_priority,
|
||||
retry_attempt,
|
||||
result = "retry_cancelled",
|
||||
"Heal retry admission decided"
|
||||
);
|
||||
return;
|
||||
}
|
||||
_ = retry_manager_cancel_token.cancelled() => {
|
||||
retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
|
||||
return;
|
||||
}
|
||||
_ = sleep(retry_delay) => {}
|
||||
}
|
||||
|
||||
{
|
||||
let retrying_heals_guard = retrying_heals_for_spawn.lock().await;
|
||||
if !retrying_heals_guard.contains_key(&retry_request_id) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let active_duplicate_task_id = {
|
||||
let active_heals_guard = retry_active_heals.lock().await;
|
||||
active_heal_for_dedup_key(&active_heals_guard, &retry_key).map(|(task_id, _)| task_id)
|
||||
};
|
||||
if let Some(active_duplicate_task_id) = active_duplicate_task_id {
|
||||
retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
|
||||
move_mrf_repair_notice_targets(
|
||||
&retry_mrf_repair_notice_targets,
|
||||
&retry_request_id,
|
||||
&active_duplicate_task_id,
|
||||
);
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %retry_request_id,
|
||||
priority = ?retry_priority,
|
||||
retry_attempt,
|
||||
result = "retry_merged_active_duplicate",
|
||||
"Heal retry admission decided"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let mut queue = retry_heal_queue.lock().await;
|
||||
let admission_decision =
|
||||
Self::admit_request_to_queue(&mut queue, retry_request.clone(), &retry_config, "retry");
|
||||
let admission = admission_decision.result;
|
||||
let should_notify = matches!(admission, HealAdmissionResult::Accepted)
|
||||
&& retry_config.event_driven_scheduler_enable;
|
||||
match admission {
|
||||
HealAdmissionResult::Accepted => {
|
||||
// Transfer ownership while holding queue -> retrying,
|
||||
// matching operations_snapshot's lock order.
|
||||
#[cfg(test)]
|
||||
pause_retry_ownership_transition(&retry_request_id, true).await;
|
||||
retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
|
||||
let displaced_task_id = admission_decision.displaced_task_id;
|
||||
drop(queue);
|
||||
if let Some(displaced_task_id) = displaced_task_id {
|
||||
remove_task_aliases_for_task(&retry_task_aliases, &displaced_task_id).await;
|
||||
remove_mrf_repair_notice_targets(
|
||||
&retry_mrf_repair_notice_targets,
|
||||
&displaced_task_id,
|
||||
);
|
||||
}
|
||||
retry_completed_heals.lock().await.remove(&retry_request_id);
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %retry_request_id,
|
||||
priority = ?retry_priority,
|
||||
retry_attempt,
|
||||
retry_delay_ms = retry_delay.as_millis(),
|
||||
error = %retry_error,
|
||||
result = "retry_enqueued",
|
||||
"Heal retry admission decided"
|
||||
);
|
||||
if should_notify {
|
||||
retry_notify.notify_one();
|
||||
}
|
||||
return;
|
||||
}
|
||||
HealAdmissionResult::Merged => {
|
||||
let merged_task_id =
|
||||
queue.queued_request_id_for_dedup_key(&retry_key).map(ToOwned::to_owned);
|
||||
retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
|
||||
drop(queue);
|
||||
if let Some(merged_task_id) = merged_task_id {
|
||||
move_mrf_repair_notice_targets(
|
||||
&retry_mrf_repair_notice_targets,
|
||||
&retry_request_id,
|
||||
&merged_task_id,
|
||||
);
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %retry_request_id,
|
||||
priority = ?retry_priority,
|
||||
retry_attempt,
|
||||
result = "retry_merged_duplicate",
|
||||
"Heal retry admission decided"
|
||||
);
|
||||
return;
|
||||
}
|
||||
HealAdmissionResult::Full => {
|
||||
// admit_request_to_queue already logged the
|
||||
// rejection (context = "retry"); this repeats
|
||||
// every backoff cycle while the queue stays
|
||||
// full, so keep it at debug!.
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %retry_request_id,
|
||||
priority = ?retry_priority,
|
||||
retry_attempt,
|
||||
result = "retry_rejected_full",
|
||||
"Heal retry admission decided"
|
||||
);
|
||||
}
|
||||
HealAdmissionResult::Dropped(reason) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_ADMISSION,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
request_id = %retry_request_id,
|
||||
priority = ?retry_priority,
|
||||
retry_attempt,
|
||||
reason = reason.as_str(),
|
||||
result = "retry_dropped",
|
||||
"Heal retry admission decided"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
notify_clone.notify_one();
|
||||
});
|
||||
tasks_started += 1;
|
||||
} else {
|
||||
delayed_by_mainline_throttle = mainline_pressure.is_some();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Update statistics for all started tasks
|
||||
let mut stats = statistics.write().await;
|
||||
stats.total_tasks += tasks_started as u64;
|
||||
stats.update_running_tasks(active_heals_guard.len() as u64);
|
||||
publish_active_heal_count(&active_heals_guard);
|
||||
publish_heal_queue_length(&queue);
|
||||
|
||||
if delayed_by_mainline_throttle && let Some(pressure) = mainline_pressure {
|
||||
Self::record_mainline_throttle_delay(pressure, &config);
|
||||
Self::schedule_mainline_throttle_recheck(notify.clone(), config.mainline_max_sleep);
|
||||
}
|
||||
|
||||
// Log queue status if items remain
|
||||
if !queue.is_empty() {
|
||||
let remaining = queue.len();
|
||||
if remaining > 10 {
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_QUEUE_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
queue_len = remaining,
|
||||
active_tasks = active_heals_guard.len(),
|
||||
state = "backlog_high",
|
||||
"Heal queue backlog high"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
pub(super) fn heal_request_set_key(request: &HealRequest) -> Option<String> {
|
||||
match &request.heal_type {
|
||||
HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()),
|
||||
HealType::Object { .. } => request.options.set_key(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn heal_request_type_label(request: &HealRequest) -> &'static str {
|
||||
request.heal_type.kind_label()
|
||||
}
|
||||
|
||||
pub(super) fn heal_request_set_metric_label(request: &HealRequest) -> String {
|
||||
heal_request_set_key(request).unwrap_or_else(|| request.options.set_metric_label())
|
||||
}
|
||||
|
||||
pub(super) fn record_scheduler_skip(set_label: &str) {
|
||||
counter!(
|
||||
"rustfs_heal_scheduler_skip_total",
|
||||
"reason" => "set_limit".to_string(),
|
||||
"set" => set_label.to_string()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
pub(super) fn update_task_running_metric_for_task(active_heals: &HashMap<String, Arc<HealTask>>, task: &HealTask) {
|
||||
let type_label = task.metric_type_label();
|
||||
let set_label = task.metric_set_label();
|
||||
let count = active_heals
|
||||
.values()
|
||||
.filter(|active_task| active_task.metric_type_label() == type_label && active_task.metric_set_label() == set_label)
|
||||
.count();
|
||||
|
||||
gauge!(
|
||||
"rustfs_heal_task_running",
|
||||
"type" => type_label.to_string(),
|
||||
"set" => set_label
|
||||
)
|
||||
.set(count as f64);
|
||||
}
|
||||
|
||||
pub(super) fn running_heal_set_counts(active_heals: &HashMap<String, Arc<HealTask>>) -> HashMap<String, usize> {
|
||||
let mut running = HashMap::new();
|
||||
for task in active_heals.values() {
|
||||
if let Some(set_key) = heal_request_set_key_for_task(task) {
|
||||
*running.entry(set_key).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
running
|
||||
}
|
||||
|
||||
fn remove_mrf_repair_notice_targets(registry: &Arc<StdMutex<HashMap<String, Vec<MrfRepairNoticeTarget>>>>, task_id: &str) {
|
||||
lock_mrf_repair_notice_targets(registry).remove(task_id);
|
||||
}
|
||||
|
||||
fn take_mrf_repair_notice_targets(
|
||||
registry: &Arc<StdMutex<HashMap<String, Vec<MrfRepairNoticeTarget>>>>,
|
||||
task_id: &str,
|
||||
) -> Vec<MrfRepairNoticeTarget> {
|
||||
lock_mrf_repair_notice_targets(registry).remove(task_id).unwrap_or_default()
|
||||
}
|
||||
|
||||
fn move_mrf_repair_notice_targets(
|
||||
registry: &Arc<StdMutex<HashMap<String, Vec<MrfRepairNoticeTarget>>>>,
|
||||
from_task_id: &str,
|
||||
to_task_id: &str,
|
||||
) {
|
||||
if from_task_id == to_task_id {
|
||||
return;
|
||||
}
|
||||
let mut registry = lock_mrf_repair_notice_targets(registry);
|
||||
let Some(moving) = registry.remove(from_task_id) else {
|
||||
return;
|
||||
};
|
||||
let targets = registry.entry(to_task_id.to_string()).or_default();
|
||||
for target in moving {
|
||||
if !targets.contains(&target) {
|
||||
targets.push(target);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_mrf_repaired_events(targets: Vec<MrfRepairNoticeTarget>) {
|
||||
for target in targets {
|
||||
rustfs_common::mrf_channel::note_mrf_repaired(&target.bucket, &target.object, target.version_id);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn heal_request_set_key_for_task(task: &HealTask) -> Option<String> {
|
||||
match &task.heal_type {
|
||||
HealType::ErasureSet { set_disk_id, .. } => Some(set_disk_id.clone()),
|
||||
HealType::Object { .. } => task.options.set_key(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>) {
|
||||
let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else {
|
||||
return;
|
||||
};
|
||||
|
||||
completed_heals.retain(|_, completed| {
|
||||
completed
|
||||
.completed_at
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
.map(|completed_at| now.saturating_sub(completed_at) <= KEEP_HEAL_TASK_STATUS_DURATION)
|
||||
.unwrap_or(false)
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn can_schedule_request(
|
||||
request: &HealRequest,
|
||||
running_per_set: &HashMap<String, usize>,
|
||||
max_concurrent_per_set: usize,
|
||||
) -> bool {
|
||||
match heal_request_set_key(request) {
|
||||
Some(set_key) => running_per_set.get(&set_key).copied().unwrap_or(0) < max_concurrent_per_set,
|
||||
None => true,
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,390 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// Unclean-shutdown recovery: durable replacement-intent discovery and healing-marker rewrite.
|
||||
use super::*;
|
||||
|
||||
pub(super) fn durable_replacement_recovery_is_due(state: &ResumeState, task_id: &str) -> bool {
|
||||
state.replacement_generation.as_deref() == Some(task_id)
|
||||
&& !state.replacement_targets.is_empty()
|
||||
&& ((!state.completed
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding)
|
||||
&& state.retry_count >= state.max_retries)
|
||||
|| (state.completed
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending)))
|
||||
}
|
||||
|
||||
pub(super) fn replacement_discovery_error_is_expected_for_deferred_endpoint(
|
||||
error: &Error,
|
||||
endpoint: &str,
|
||||
deferred_replacement_endpoints: &HashSet<String>,
|
||||
) -> bool {
|
||||
matches!(error, Error::Disk(DiskError::UnformattedDisk)) && deferred_replacement_endpoints.contains(endpoint)
|
||||
}
|
||||
|
||||
pub(super) fn unblock_replacement_recovery_sets_after_validation(
|
||||
blocked_sets: &mut HashSet<String>,
|
||||
retry_succeeded: HashSet<String>,
|
||||
retry_failed: &HashSet<String>,
|
||||
) {
|
||||
for set_disk_id in retry_succeeded {
|
||||
if !retry_failed.contains(&set_disk_id) {
|
||||
blocked_sets.remove(&set_disk_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HealManager {
|
||||
/// Detect whether the previous run ended without a clean shutdown and, if so,
|
||||
/// enqueue a full erasure-set heal for every local set. Also (re)writes the
|
||||
/// marker for the current run; [`super::super::clear_unclean_shutdown_markers`]
|
||||
/// removes it again during graceful shutdown. Best-effort: failures only log.
|
||||
pub(super) async fn process_unclean_shutdown(&self) {
|
||||
let mut unclean = false;
|
||||
let mut set_disk_ids = HashSet::new();
|
||||
let mut replacement_intents = HashMap::<String, (String, Vec<String>, Vec<String>, String)>::new();
|
||||
let mut replacement_restarts = HashMap::<String, (String, Vec<String>)>::new();
|
||||
let mut conflicted_replacement_sets = HashSet::new();
|
||||
|
||||
{
|
||||
let local_disks = {
|
||||
let local_disk_map = local_disk_map_read().await;
|
||||
local_disk_map.values().flatten().cloned().collect::<Vec<_>>()
|
||||
};
|
||||
for disk in &local_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
match disk
|
||||
.read_all(super::super::RUSTFS_META_BUCKET, super::super::UNCLEAN_SHUTDOWN_MARKER_PATH)
|
||||
.await
|
||||
{
|
||||
Ok(_) => unclean = true,
|
||||
Err(DiskError::FileNotFound) | Err(DiskError::VolumeNotFound) => {}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
endpoint = %endpoint,
|
||||
error = ?err,
|
||||
"Unclean-shutdown marker check failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let marker = SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs().to_string())
|
||||
.unwrap_or_default();
|
||||
if let Err(err) = disk
|
||||
.write_all(
|
||||
super::super::RUSTFS_META_BUCKET,
|
||||
super::super::UNCLEAN_SHUTDOWN_MARKER_PATH,
|
||||
marker.into(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
endpoint = %endpoint,
|
||||
error = ?err,
|
||||
"Unclean-shutdown marker write failed"
|
||||
);
|
||||
}
|
||||
|
||||
let disk_set_disk_id = crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx);
|
||||
if let Some(set_disk_id) = &disk_set_disk_id {
|
||||
set_disk_ids.insert(set_disk_id.clone());
|
||||
}
|
||||
|
||||
// Legacy flat records are inspected only while starting. The
|
||||
// periodic scanner lists the dedicated replacement directory.
|
||||
if let Err(error) = ResumeUtils::migrate_legacy_replacement_records(disk).await {
|
||||
if let Some(set_disk_id) = &disk_set_disk_id {
|
||||
self.block_replacement_recovery_set(set_disk_id);
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
endpoint = %endpoint,
|
||||
error = %error,
|
||||
"Legacy replacement recovery migration failed"
|
||||
);
|
||||
}
|
||||
let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await {
|
||||
Ok(task_ids) => task_ids,
|
||||
Err(error) => {
|
||||
if let Some(set_disk_id) = &disk_set_disk_id {
|
||||
self.block_replacement_recovery_set(set_disk_id);
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
endpoint = %endpoint,
|
||||
error = %error,
|
||||
"Replacement recovery discovery failed"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for task_id in replacement_task_ids {
|
||||
let manager = match ResumeManager::load_replacement_intent(disk.clone(), &task_id).await {
|
||||
Ok(manager) => manager,
|
||||
Err(error) => {
|
||||
if let Some(set_disk_id) = &disk_set_disk_id {
|
||||
self.block_replacement_recovery_set(set_disk_id);
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
endpoint = %endpoint,
|
||||
task_id,
|
||||
error = %error,
|
||||
"Replacement recovery intent load failed"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let state = manager.get_state().await;
|
||||
let active_replacement = !state.completed
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding);
|
||||
let verified_replacement = state.completed
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending);
|
||||
if (active_replacement || verified_replacement)
|
||||
&& state.replacement_generation.as_deref() == Some(task_id.as_str())
|
||||
&& !state.replacement_targets.is_empty()
|
||||
{
|
||||
if matches!(state.replacement_phase, ReplacementPhase::CleanupPending) {
|
||||
replacement_intents.entry(task_id).or_insert((
|
||||
state.set_disk_id,
|
||||
state.replacement_targets,
|
||||
state.replacement_buckets,
|
||||
endpoint.to_string(),
|
||||
));
|
||||
continue;
|
||||
}
|
||||
match self.storage.replacement_target_identities(&state.replacement_targets).await {
|
||||
Ok(identities) if identities == state.replacement_target_identities => {
|
||||
let resume_endpoint = endpoint.to_string();
|
||||
match replacement_intents.entry(task_id) {
|
||||
std::collections::hash_map::Entry::Vacant(entry) => {
|
||||
entry.insert((
|
||||
state.set_disk_id,
|
||||
state.replacement_targets,
|
||||
state.replacement_buckets,
|
||||
resume_endpoint,
|
||||
));
|
||||
}
|
||||
std::collections::hash_map::Entry::Occupied(entry) => {
|
||||
let (existing_set_disk_id, existing_targets, existing_buckets, existing_anchor) =
|
||||
entry.get();
|
||||
if existing_set_disk_id != &state.set_disk_id
|
||||
|| existing_targets != &state.replacement_targets
|
||||
|| existing_buckets != &state.replacement_buckets
|
||||
|| existing_anchor != &resume_endpoint
|
||||
{
|
||||
conflicted_replacement_sets.insert(state.set_disk_id.clone());
|
||||
self.block_replacement_recovery_set(&state.set_disk_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(_) => {
|
||||
if manager.abandon_replacement_intent().await.is_ok() {
|
||||
replacement_restarts
|
||||
.entry(task_id)
|
||||
.or_insert((state.set_disk_id, state.replacement_targets));
|
||||
}
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !unclean && replacement_intents.is_empty() && replacement_restarts.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut recovery_by_set = HashMap::<String, Vec<(Option<String>, Vec<String>, Vec<String>, Option<String>)>>::new();
|
||||
for (task_id, (set_disk_id, heal_endpoints, buckets, resume_endpoint)) in replacement_intents {
|
||||
recovery_by_set
|
||||
.entry(set_disk_id)
|
||||
.or_default()
|
||||
.push((Some(task_id), heal_endpoints, buckets, Some(resume_endpoint)));
|
||||
}
|
||||
for (_abandoned_task_id, (set_disk_id, heal_endpoints)) in replacement_restarts {
|
||||
recovery_by_set
|
||||
.entry(set_disk_id)
|
||||
.or_default()
|
||||
.push((None, heal_endpoints, Vec::new(), None));
|
||||
}
|
||||
|
||||
for (set_disk_id, mut recoveries) in recovery_by_set {
|
||||
let Ok((pool_index, set_index)) = crate::heal::utils::parse_set_disk_id(&set_disk_id) else {
|
||||
continue;
|
||||
};
|
||||
if self.replacement_recovery_set_is_blocked(&set_disk_id) {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
set_disk_id,
|
||||
recovery_count = recoveries.len(),
|
||||
"Replacement recovery deferred because durable recovery validation is blocked"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if conflicted_replacement_sets.contains(&set_disk_id) || recoveries.len() != 1 {
|
||||
self.block_replacement_recovery_set(&set_disk_id);
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
set_disk_id,
|
||||
recovery_count = recoveries.len(),
|
||||
"Replacement recovery deferred because multiple durable generations exist"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let reuse_single_generation = recoveries.len() == 1 && recoveries[0].0.is_some();
|
||||
let mut heal_endpoints = recoveries
|
||||
.iter_mut()
|
||||
.flat_map(|(_, targets, _, _)| std::mem::take(targets))
|
||||
.collect::<Vec<_>>();
|
||||
heal_endpoints.sort_unstable();
|
||||
heal_endpoints.dedup();
|
||||
let buckets = if reuse_single_generation {
|
||||
std::mem::take(&mut recoveries[0].2)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let mut req = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets,
|
||||
set_disk_id: set_disk_id.clone(),
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: Some(pool_index),
|
||||
set_index: Some(set_index),
|
||||
timeout: None,
|
||||
..HealOptions::default()
|
||||
},
|
||||
HealPriority::Low,
|
||||
);
|
||||
if reuse_single_generation && let Some(task_id) = recoveries[0].0.take() {
|
||||
req.id = task_id;
|
||||
}
|
||||
let recovery_anchor = reuse_single_generation.then(|| recoveries[0].3.take()).flatten();
|
||||
req.source = HealRequestSource::AutoHeal;
|
||||
req.heal_endpoints = heal_endpoints;
|
||||
let request_id = req.id.clone();
|
||||
if let Some(anchor) = &recovery_anchor {
|
||||
self.replacement_recovery_anchors
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.insert(request_id.clone(), anchor.clone());
|
||||
}
|
||||
match self.submit_heal_request(req).await {
|
||||
Ok(HealAdmissionResult::Accepted) => {}
|
||||
Ok(_) => {
|
||||
self.replacement_recovery_anchors
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.remove(&request_id);
|
||||
}
|
||||
Err(err) => {
|
||||
self.replacement_recovery_anchors
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.remove(&request_id);
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
set_disk_id,
|
||||
error = %err,
|
||||
"Replacement recovery enqueue failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !unclean || set_disk_ids.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
set_count = set_disk_ids.len(),
|
||||
"Unclean shutdown detected; scheduling erasure-set heal for local sets"
|
||||
);
|
||||
|
||||
let buckets = match self.storage.list_buckets().await {
|
||||
Ok(buckets) => buckets.iter().map(|b| b.name.clone()).collect::<Vec<String>>(),
|
||||
Err(err) => {
|
||||
error!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
error = %err,
|
||||
"Unclean-shutdown heal skipped: bucket listing failed"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
for set_disk_id in set_disk_ids {
|
||||
let mut req = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: buckets.clone(),
|
||||
set_disk_id: set_disk_id.clone(),
|
||||
},
|
||||
HealOptions {
|
||||
timeout: None,
|
||||
..HealOptions::default()
|
||||
},
|
||||
HealPriority::Low,
|
||||
);
|
||||
req.source = HealRequestSource::AutoHeal;
|
||||
if let Err(err) = self.submit_heal_request(req).await {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
set_disk_id,
|
||||
error = %err,
|
||||
"Unclean-shutdown heal enqueue failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
pub mod channel;
|
||||
pub mod erasure_healer;
|
||||
pub mod event;
|
||||
pub mod manager;
|
||||
pub mod mrf_queue;
|
||||
pub mod progress;
|
||||
|
||||
@@ -25,9 +25,12 @@
|
||||
//! set, rewritten on a group-commit cadence (every flush interval or flush
|
||||
//! threshold new intents). A rewrite is atomic at the record level only — a
|
||||
//! torn tail simply truncates during replay because every record carries its
|
||||
//! own CRC32. Losing the last flush window (≤500 ms) is acceptable: replayed
|
||||
//! duplicates are merged by the manager's dedup key, and read-repair remains
|
||||
//! the safety net.
|
||||
//! own CRC32. Losing the last flush window (≤500 ms) is acceptable because
|
||||
//! every producer keeps its own safety net: read-repair re-detects on the
|
||||
//! next failing read, and the scanner's corrupt-metadata branch leaves a
|
||||
//! pending-ledger entry behind even when its MRF intent is accepted
|
||||
//! (backlog#1894 axis A), so a lost intent is retried by the ledger rather
|
||||
//! than waiting for the failed-object TTL to re-scan the path.
|
||||
|
||||
use super::{DiskStore, HealDiskExt as _, local_disk_map_read};
|
||||
use crate::heal::manager::HealManager;
|
||||
@@ -276,20 +279,26 @@ async fn read_journal() -> Option<Vec<u8>> {
|
||||
None
|
||||
}
|
||||
|
||||
async fn write_journal(data: &[u8]) {
|
||||
/// Write the snapshot to every local disk; returns true when at least one
|
||||
/// disk accepted it, so a total write failure keeps the runtime dirty and
|
||||
/// the next tick retries the persist.
|
||||
async fn write_journal(data: &[u8]) -> bool {
|
||||
let payload = bytes::Bytes::copy_from_slice(data);
|
||||
let mut any_persisted = false;
|
||||
for disk in journal_disks().await {
|
||||
if let Err(err) = disk
|
||||
match disk
|
||||
.write_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload.clone())
|
||||
.await
|
||||
{
|
||||
warn_mrf_journal_write(&err);
|
||||
Ok(()) => any_persisted = true,
|
||||
Err(err) => warn_mrf_journal_write(&err),
|
||||
}
|
||||
}
|
||||
if !data.is_empty() {
|
||||
counter!("rustfs_heal_mrf_journal_fsync_total").increment(1);
|
||||
}
|
||||
gauge!("rustfs_heal_mrf_journal_bytes").set(data.len() as f64);
|
||||
any_persisted
|
||||
}
|
||||
|
||||
async fn delete_journal() {
|
||||
@@ -347,10 +356,28 @@ pub(crate) fn build_heal_request(intent: &MrfIntent) -> HealRequest {
|
||||
request
|
||||
}
|
||||
|
||||
async fn submit_mrf_heal_request(manager: &HealManager, intent: &MrfIntent) -> crate::Result<HealAdmissionResult> {
|
||||
let receipt = manager
|
||||
.submit_mrf_heal_request_with_receipt(
|
||||
build_heal_request(intent),
|
||||
intent.bucket.clone(),
|
||||
intent.object.clone(),
|
||||
intent.version_id,
|
||||
)
|
||||
.await?;
|
||||
Ok(receipt.result)
|
||||
}
|
||||
|
||||
struct MrfRuntime {
|
||||
queue: MrfQueue,
|
||||
config: MrfConsumerConfig,
|
||||
new_since_flush: usize,
|
||||
/// True while the in-memory pending set has changed since the last
|
||||
/// journal flush (push, pop, or an attempts bump that alters the encoded
|
||||
/// bytes). Only a dirty state rewrites the snapshot: a steady backlog
|
||||
/// waiting out an admission backoff must not re-fsync every local disk
|
||||
/// twice a second.
|
||||
dirty: bool,
|
||||
/// True while a journal snapshot exists on disk that no longer reflects
|
||||
/// an all-consumed pending set; the next idle tick removes it (MinIO
|
||||
/// deletes its `list.bin` after replay for the same reason).
|
||||
@@ -360,11 +387,6 @@ struct MrfRuntime {
|
||||
}
|
||||
|
||||
impl MrfRuntime {
|
||||
fn record_accept(&mut self) {
|
||||
// Accepted intents leave the pending set; the next flush persists the
|
||||
// smaller snapshot, which is the journal's compaction.
|
||||
}
|
||||
|
||||
fn snapshot(&self) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
for intent in self.queue.intents() {
|
||||
@@ -374,8 +396,14 @@ impl MrfRuntime {
|
||||
}
|
||||
|
||||
async fn flush(&mut self) {
|
||||
write_journal(&self.snapshot()).await;
|
||||
let persisted = write_journal(&self.snapshot()).await;
|
||||
self.new_since_flush = 0;
|
||||
// Keep the dirty flag when every disk write failed: a clean backlog
|
||||
// would otherwise never rewrite, losing the periodic persist retry a
|
||||
// non-empty queue used to provide.
|
||||
if persisted {
|
||||
self.dirty = false;
|
||||
}
|
||||
self.journal_on_disk = true;
|
||||
}
|
||||
|
||||
@@ -389,9 +417,15 @@ impl MrfRuntime {
|
||||
self.backoff_until = None;
|
||||
}
|
||||
while let Some(mut intent) = self.queue.pop_front() {
|
||||
let request = build_heal_request(&intent);
|
||||
match manager.submit_heal_request(request).await {
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => self.record_accept(),
|
||||
// Leaving the pending set (consumed or re-queued with a bumped
|
||||
// attempts counter) changes the encoded snapshot; mark it dirty
|
||||
// either way.
|
||||
self.dirty = true;
|
||||
match submit_mrf_heal_request(manager, &intent).await {
|
||||
// Accepted intents leave the pending set; the next flush persists the
|
||||
// smaller snapshot. The scanner ledger is cleared later, when the
|
||||
// canonical heal task reaches a successful terminal completion.
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {}
|
||||
Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
if intent.attempts >= MRF_MAX_ATTEMPTS {
|
||||
@@ -493,8 +527,7 @@ async fn replay_into(
|
||||
// stays armed in `queue` for the consumer's retry loop.
|
||||
if backoff_until.is_none() {
|
||||
while let Some(mut intent) = queue.pop_front() {
|
||||
let request = build_heal_request(&intent);
|
||||
match manager.submit_heal_request(request).await {
|
||||
match submit_mrf_heal_request(manager, &intent).await {
|
||||
Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {}
|
||||
Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => {
|
||||
intent.attempts = intent.attempts.saturating_add(1);
|
||||
@@ -519,6 +552,7 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
queue: MrfQueue::new(config.queue_capacity, config.journal_max_bytes),
|
||||
config: config.clone(),
|
||||
new_since_flush: 0,
|
||||
dirty: false,
|
||||
journal_on_disk: false,
|
||||
backoff_until: None,
|
||||
};
|
||||
@@ -526,6 +560,10 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
// Replay: read the journal, re-arm intents (duplicates are merged by the
|
||||
// manager's dedup key), then drop the file so the next flush starts clean.
|
||||
replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await;
|
||||
// The replay deleted the journal file; anything still pending (e.g. the
|
||||
// manager was full and backoff armed) must be re-persisted by the next
|
||||
// flush or a crash before it would lose those intents.
|
||||
runtime.dirty = runtime.queue.depth() > 0;
|
||||
|
||||
let mut flush_tick = tokio::time::interval(runtime.config.flush_interval);
|
||||
flush_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
@@ -535,8 +573,13 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
tokio::select! {
|
||||
received = receiver.recv_many(&mut batch, runtime.config.replay_batch) => {
|
||||
if received == 0 {
|
||||
// Channel closed: flush once more and stop.
|
||||
runtime.flush().await;
|
||||
// Channel closed: flush once more unless the snapshot is
|
||||
// provably current AND idle (a dirty or pending state
|
||||
// gets one last persist attempt, matching the shutdown
|
||||
// retry the unconditional flush used to provide).
|
||||
if runtime.dirty || runtime.queue.depth() > 0 {
|
||||
runtime.flush().await;
|
||||
}
|
||||
tracing::info!(
|
||||
target: "rustfs::heal::mrf",
|
||||
"MRF channel closed; consumer stopped after final flush"
|
||||
@@ -544,8 +587,10 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
return;
|
||||
}
|
||||
for intent in batch.drain(..) {
|
||||
runtime.queue.try_push(intent);
|
||||
runtime.new_since_flush += 1;
|
||||
if runtime.queue.try_push(intent) {
|
||||
runtime.new_since_flush += 1;
|
||||
runtime.dirty = true;
|
||||
}
|
||||
}
|
||||
runtime.dispatch(manager.as_ref()).await;
|
||||
if runtime.new_since_flush >= runtime.config.flush_threshold {
|
||||
@@ -553,15 +598,26 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
}
|
||||
}
|
||||
_ = flush_tick.tick() => {
|
||||
if runtime.new_since_flush > 0 || runtime.queue.depth() > 0 {
|
||||
runtime.flush().await;
|
||||
runtime.dispatch(manager.as_ref()).await;
|
||||
} else if runtime.journal_on_disk {
|
||||
// All intents consumed: remove the journal so a restart
|
||||
// replays nothing (mirrors MinIO's post-replay unlink).
|
||||
delete_journal().await;
|
||||
runtime.journal_on_disk = false;
|
||||
gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
|
||||
match tick_action(runtime.dirty, runtime.queue.depth(), runtime.journal_on_disk) {
|
||||
TickAction::Flush => {
|
||||
runtime.flush().await;
|
||||
runtime.dispatch(manager.as_ref()).await;
|
||||
}
|
||||
TickAction::Retry => {
|
||||
// Pending set unchanged since the last flush (a
|
||||
// backlog waiting out an admission backoff): skip the
|
||||
// rewrite but keep dispatching so the retry fires on
|
||||
// time.
|
||||
runtime.dispatch(manager.as_ref()).await;
|
||||
}
|
||||
TickAction::DeleteJournal => {
|
||||
// All intents consumed: remove the journal so a restart
|
||||
// replays nothing (mirrors MinIO's post-replay unlink).
|
||||
delete_journal().await;
|
||||
runtime.journal_on_disk = false;
|
||||
gauge!("rustfs_heal_mrf_journal_bytes").set(0.0);
|
||||
}
|
||||
TickAction::Idle => {}
|
||||
}
|
||||
gauge!("rustfs_heal_mrf_queue_depth").set(runtime.queue.depth() as f64);
|
||||
}
|
||||
@@ -569,6 +625,33 @@ async fn run_mrf_consumer(manager: Arc<HealManager>, mut receiver: mpsc::Receive
|
||||
}
|
||||
}
|
||||
|
||||
/// What the periodic tick should do, as a pure function of the runtime state
|
||||
/// so the decision table is unit-testable.
|
||||
enum TickAction {
|
||||
/// The pending set changed since the last snapshot: rewrite it, then
|
||||
/// drain.
|
||||
Flush,
|
||||
/// Pending intents exist but the snapshot is current: only drain (an
|
||||
/// admission backoff may have expired).
|
||||
Retry,
|
||||
/// Nothing pending and a stale journal file remains: remove it.
|
||||
DeleteJournal,
|
||||
/// Quiescent: nothing to do.
|
||||
Idle,
|
||||
}
|
||||
|
||||
fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool) -> TickAction {
|
||||
if dirty {
|
||||
TickAction::Flush
|
||||
} else if depth > 0 {
|
||||
TickAction::Retry
|
||||
} else if journal_on_disk {
|
||||
TickAction::DeleteJournal
|
||||
} else {
|
||||
TickAction::Idle
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -586,6 +669,27 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tick_action_table() {
|
||||
use TickAction::*;
|
||||
|
||||
// Dirty dominates: a changed pending set flushes even when idle
|
||||
// otherwise.
|
||||
assert!(matches!(tick_action(true, 0, false), Flush));
|
||||
assert!(matches!(tick_action(true, 3, true), Flush));
|
||||
|
||||
// Clean backlog: no rewrite, but keep draining so an expired
|
||||
// admission backoff retries on time.
|
||||
assert!(matches!(tick_action(false, 1, false), Retry));
|
||||
assert!(matches!(tick_action(false, 2, true), Retry));
|
||||
|
||||
// Quiescent with a stale journal file on disk: remove it.
|
||||
assert!(matches!(tick_action(false, 0, true), DeleteJournal));
|
||||
|
||||
// Fully quiescent: nothing to do.
|
||||
assert!(matches!(tick_action(false, 0, false), Idle));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queue_enforces_count_and_byte_ceilings() {
|
||||
let mut queue = MrfQueue::new(2, usize::MAX);
|
||||
|
||||
@@ -218,15 +218,6 @@ impl HealStatistics {
|
||||
self.total_bytes_healed += bytes;
|
||||
self.last_update_time = SystemTime::now();
|
||||
}
|
||||
|
||||
pub fn get_success_rate(&self) -> f64 {
|
||||
let total = self.successful_tasks + self.failed_tasks;
|
||||
if total > 0 {
|
||||
(self.successful_tasks as f64 / total as f64) * 100.0
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -539,38 +530,4 @@ mod tests {
|
||||
assert_eq!(stats.total_objects_healed, 8);
|
||||
assert_eq!(stats.total_bytes_healed, 8192);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_statistics_get_success_rate() {
|
||||
let mut stats = HealStatistics::new();
|
||||
stats.successful_tasks = 8;
|
||||
stats.failed_tasks = 2;
|
||||
|
||||
// success_rate = 8 / (8 + 2) * 100 = 80%
|
||||
assert!((stats.get_success_rate() - 80.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_statistics_get_success_rate_zero_total() {
|
||||
let stats = HealStatistics::new();
|
||||
assert_eq!(stats.get_success_rate(), 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_statistics_get_success_rate_all_success() {
|
||||
let mut stats = HealStatistics::new();
|
||||
stats.successful_tasks = 10;
|
||||
stats.failed_tasks = 0;
|
||||
|
||||
assert!((stats.get_success_rate() - 100.0).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_statistics_get_success_rate_all_failure() {
|
||||
let mut stats = HealStatistics::new();
|
||||
stats.successful_tasks = 0;
|
||||
stats.failed_tasks = 5;
|
||||
|
||||
assert_eq!(stats.get_success_rate(), 0.0);
|
||||
}
|
||||
}
|
||||
|
||||
+14
-3231
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,351 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{Error, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use super::super::{BUCKET_META_PREFIX, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET};
|
||||
use super::{
|
||||
LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, PersistThrottle, RESUME_CHECKPOINT_FILE, delete_resume_file, path_to_str,
|
||||
validate_resume_task_id,
|
||||
};
|
||||
|
||||
const EVENT_HEAL_CHECKPOINT_STATE: &str = "heal_checkpoint_state";
|
||||
|
||||
/// Current on-disk schema version for `ResumeCheckpoint`. Same rationale as
|
||||
/// `CURRENT_RESUME_SCHEMA`: pre-per-version dedup identities are not comparable
|
||||
/// to the new `compose_key` identities, so a stale checkpoint is discarded.
|
||||
pub(super) const CURRENT_CHECKPOINT_SCHEMA: u32 = 5;
|
||||
|
||||
/// resume checkpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResumeCheckpoint {
|
||||
/// on-disk schema version; absent in legacy snapshots (defaults to 0)
|
||||
#[serde(default)]
|
||||
pub schema_version: u32,
|
||||
/// task id
|
||||
pub task_id: String,
|
||||
/// checkpoint time
|
||||
pub checkpoint_time: u64,
|
||||
/// current bucket index
|
||||
pub current_bucket_index: usize,
|
||||
/// current object index
|
||||
pub current_object_index: usize,
|
||||
/// Objects healed since the last completed page. HashSet: with the
|
||||
/// previous Vec the per-object `contains` was O(n) and made large-bucket
|
||||
/// heals O(N²). Only spans the in-flight page — completed pages are
|
||||
/// covered by `current_object_index`, so `complete_page` prunes the sets.
|
||||
pub processed_objects: HashSet<String>,
|
||||
/// failed objects
|
||||
pub failed_objects: HashSet<String>,
|
||||
/// skipped objects
|
||||
pub skipped_objects: HashSet<String>,
|
||||
}
|
||||
|
||||
impl ResumeCheckpoint {
|
||||
pub fn new(task_id: String) -> Self {
|
||||
Self {
|
||||
schema_version: CURRENT_CHECKPOINT_SCHEMA,
|
||||
task_id,
|
||||
checkpoint_time: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(),
|
||||
current_bucket_index: 0,
|
||||
current_object_index: 0,
|
||||
processed_objects: HashSet::new(),
|
||||
failed_objects: HashSet::new(),
|
||||
skipped_objects: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_position(&mut self, bucket_index: usize, object_index: usize) {
|
||||
self.current_bucket_index = bucket_index;
|
||||
self.current_object_index = object_index;
|
||||
self.checkpoint_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
}
|
||||
|
||||
pub fn add_processed_object(&mut self, object: String) {
|
||||
self.processed_objects.insert(object);
|
||||
}
|
||||
|
||||
pub fn add_failed_object(&mut self, object: String) {
|
||||
self.failed_objects.insert(object);
|
||||
}
|
||||
|
||||
pub fn add_skipped_object(&mut self, object: String) {
|
||||
self.skipped_objects.insert(object);
|
||||
}
|
||||
|
||||
/// Advance past a fully-processed page: objects below `object_index` are
|
||||
/// skipped by position on resume, so the per-object sets no longer need
|
||||
/// their entries and would otherwise grow with the whole bucket.
|
||||
pub fn complete_page(&mut self, bucket_index: usize, object_index: usize) {
|
||||
self.update_position(bucket_index, object_index);
|
||||
self.processed_objects.clear();
|
||||
self.skipped_objects.clear();
|
||||
self.failed_objects.clear();
|
||||
}
|
||||
|
||||
/// Reset the scan to the start and clear the per-object sets so a retry
|
||||
/// re-scans the whole set.
|
||||
pub fn reset_for_retry(&mut self) {
|
||||
self.update_position(0, 0);
|
||||
self.processed_objects.clear();
|
||||
self.skipped_objects.clear();
|
||||
self.failed_objects.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/// resume checkpoint manager
|
||||
pub struct CheckpointManager {
|
||||
disk: DiskStore,
|
||||
checkpoint: Arc<RwLock<ResumeCheckpoint>>,
|
||||
throttle: Mutex<PersistThrottle>,
|
||||
}
|
||||
|
||||
impl CheckpointManager {
|
||||
/// create new checkpoint manager
|
||||
pub async fn new(disk: DiskStore, task_id: String) -> Result<Self> {
|
||||
validate_resume_task_id(&task_id)?;
|
||||
let checkpoint = ResumeCheckpoint::new(task_id);
|
||||
let manager = Self {
|
||||
disk,
|
||||
checkpoint: Arc::new(RwLock::new(checkpoint)),
|
||||
throttle: Mutex::new(PersistThrottle::new()),
|
||||
};
|
||||
|
||||
// save initial checkpoint
|
||||
if let Err(e) = manager.save_checkpoint().await {
|
||||
warn!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_CHECKPOINT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
state = "initial_save_failed",
|
||||
error = %e,
|
||||
"Heal checkpoint persistence failed"
|
||||
);
|
||||
}
|
||||
Ok(manager)
|
||||
}
|
||||
|
||||
/// load checkpoint from disk
|
||||
pub async fn load_from_disk(disk: DiskStore, task_id: &str) -> Result<Self> {
|
||||
validate_resume_task_id(task_id)?;
|
||||
let checkpoint_data = Self::read_checkpoint_file(&disk, task_id).await?;
|
||||
let mut checkpoint: ResumeCheckpoint =
|
||||
serde_json::from_slice(&checkpoint_data).map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to deserialize checkpoint: {e}"),
|
||||
})?;
|
||||
if checkpoint.task_id != task_id {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: "Resume checkpoint task id does not match filename".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// A checkpoint from an older schema stored latest-only dedup identities
|
||||
// that are not comparable to the new per-version `compose_key`
|
||||
// identities. Discard the stale sets and position, then stamp the
|
||||
// current schema so the scan restarts cleanly.
|
||||
if checkpoint.schema_version > CURRENT_CHECKPOINT_SCHEMA {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!(
|
||||
"Checkpoint schema {} is newer than supported schema {CURRENT_CHECKPOINT_SCHEMA}",
|
||||
checkpoint.schema_version
|
||||
),
|
||||
});
|
||||
}
|
||||
if checkpoint.schema_version < CURRENT_CHECKPOINT_SCHEMA {
|
||||
warn!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_CHECKPOINT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id,
|
||||
found_schema = checkpoint.schema_version,
|
||||
current_schema = CURRENT_CHECKPOINT_SCHEMA,
|
||||
state = "schema_discarded",
|
||||
"Heal checkpoint schema is stale; discarding dedup sets and position"
|
||||
);
|
||||
checkpoint.processed_objects.clear();
|
||||
checkpoint.failed_objects.clear();
|
||||
checkpoint.skipped_objects.clear();
|
||||
checkpoint.current_bucket_index = 0;
|
||||
checkpoint.current_object_index = 0;
|
||||
checkpoint.schema_version = CURRENT_CHECKPOINT_SCHEMA;
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
disk,
|
||||
checkpoint: Arc::new(RwLock::new(checkpoint)),
|
||||
throttle: Mutex::new(PersistThrottle::new()),
|
||||
})
|
||||
}
|
||||
|
||||
/// check if checkpoint exists
|
||||
pub async fn has_checkpoint(disk: &DiskStore, task_id: &str) -> bool {
|
||||
if validate_resume_task_id(task_id).is_err() {
|
||||
return false;
|
||||
}
|
||||
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
|
||||
match path_to_str(&file_path) {
|
||||
Ok(path_str) => match disk.read_all(RUSTFS_META_BUCKET, path_str).await {
|
||||
Ok(data) => !data.is_empty(),
|
||||
Err(_) => false,
|
||||
},
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// get current checkpoint
|
||||
pub async fn get_checkpoint(&self) -> ResumeCheckpoint {
|
||||
self.checkpoint.read().await.clone()
|
||||
}
|
||||
|
||||
/// update position
|
||||
pub async fn update_position(&self, bucket_index: usize, object_index: usize) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
checkpoint.update_position(bucket_index, object_index);
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint_throttled().await
|
||||
}
|
||||
|
||||
/// Advance past a completed page and prune the per-object sets, then persist.
|
||||
pub async fn complete_page(&self, bucket_index: usize, object_index: usize) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
checkpoint.complete_page(bucket_index, object_index);
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint_throttled().await
|
||||
}
|
||||
|
||||
/// Reset the checkpoint to the start of the scan for a retry, then persist.
|
||||
pub async fn reset_for_retry(&self) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
checkpoint.reset_for_retry();
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint_throttled().await
|
||||
}
|
||||
|
||||
/// Add a processed object. Called once per healed object, so persistence
|
||||
/// is batched (`PERSIST_EVERY_MUTATIONS` / `PERSIST_INTERVAL`); positions
|
||||
/// and page boundaries still persist unconditionally.
|
||||
pub async fn add_processed_object(&self, object: String) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
checkpoint.add_processed_object(object);
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint_if_due().await
|
||||
}
|
||||
|
||||
/// add failed object (batched, see `add_processed_object`)
|
||||
pub async fn add_failed_object(&self, object: String) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
checkpoint.add_failed_object(object);
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint_if_due().await
|
||||
}
|
||||
|
||||
/// add skipped object (batched, see `add_processed_object`)
|
||||
pub async fn add_skipped_object(&self, object: String) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
checkpoint.add_skipped_object(object);
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint_if_due().await
|
||||
}
|
||||
|
||||
async fn save_checkpoint_if_due(&self) -> Result<()> {
|
||||
let should_save = self.throttle.lock().map(|mut throttle| throttle.record()).unwrap_or(true);
|
||||
if !should_save {
|
||||
return Ok(());
|
||||
}
|
||||
self.save_checkpoint_throttled().await
|
||||
}
|
||||
|
||||
async fn save_checkpoint_throttled(&self) -> Result<()> {
|
||||
let result = self.save_checkpoint().await;
|
||||
if result.is_ok()
|
||||
&& let Ok(mut throttle) = self.throttle.lock()
|
||||
{
|
||||
throttle.mark_saved();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// cleanup checkpoint
|
||||
pub async fn cleanup(&self) -> Result<()> {
|
||||
let task_id = self.checkpoint.read().await.task_id.clone();
|
||||
validate_resume_task_id(&task_id)?;
|
||||
|
||||
let checkpoint_file = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
|
||||
delete_resume_file(&self.disk, &checkpoint_file).await?;
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_CHECKPOINT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id,
|
||||
state = "cleaned",
|
||||
"Heal checkpoint cleaned"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// save checkpoint to disk
|
||||
async fn save_checkpoint(&self) -> Result<()> {
|
||||
let checkpoint = self.checkpoint.read().await;
|
||||
validate_resume_task_id(&checkpoint.task_id)?;
|
||||
let checkpoint_data = serde_json::to_vec(&*checkpoint).map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to serialize checkpoint: {e}"),
|
||||
})?;
|
||||
|
||||
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{}", checkpoint.task_id, RESUME_CHECKPOINT_FILE));
|
||||
|
||||
let path_str = path_to_str(&file_path)?;
|
||||
self.disk
|
||||
.write_all(RUSTFS_META_BUCKET, path_str, checkpoint_data.into())
|
||||
.await
|
||||
.map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to save checkpoint: {e}"),
|
||||
})?;
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_CHECKPOINT_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id = %checkpoint.task_id,
|
||||
state = "saved",
|
||||
"Heal checkpoint persisted"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// read checkpoint file from disk
|
||||
async fn read_checkpoint_file(disk: &DiskStore, task_id: &str) -> Result<Vec<u8>> {
|
||||
validate_resume_task_id(task_id)?;
|
||||
let file_path = Path::new(BUCKET_META_PREFIX).join(format!("{task_id}_{RESUME_CHECKPOINT_FILE}"));
|
||||
|
||||
let path_str = path_to_str(&file_path)?;
|
||||
disk.read_all(RUSTFS_META_BUCKET, path_str)
|
||||
.await
|
||||
.map(|bytes| bytes.to_vec())
|
||||
.map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read checkpoint file: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,688 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{Error, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashSet;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use super::super::HealDiskExt as _;
|
||||
|
||||
use super::super::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskBytes};
|
||||
use super::{
|
||||
DiskError, DiskStore, RUSTFS_META_BUCKET, ResumeManager, ResumeState, delete_resume_file, ensure_replacement_recovery_dir,
|
||||
injected_replacement_proof_write_error, is_replacement_intent, legacy_replacement_completion_proof_path, path_to_str,
|
||||
replacement_completion_proof_path, replacement_intent_seal_path, replacement_recovery_conflict,
|
||||
replacement_recovery_corruption, validate_resume_task_id,
|
||||
};
|
||||
|
||||
/// Durable-proof schema version.
|
||||
const CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA: u32 = 1;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ReplacementPhase {
|
||||
#[default]
|
||||
None,
|
||||
Intent,
|
||||
Rebuilding,
|
||||
Verified,
|
||||
CleanupPending,
|
||||
Abandoned,
|
||||
}
|
||||
|
||||
/// Target-specific state for a durable automatic replacement generation.
|
||||
///
|
||||
/// This is deliberately separate from the legacy background-heal status
|
||||
/// contract. Consumers must treat [`Self::Unknown`] as non-definitive.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ReplacementRecoveryState {
|
||||
WaitingForReplacement,
|
||||
Running,
|
||||
Incomplete,
|
||||
Unrecoverable,
|
||||
CleanupPending,
|
||||
Completed,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Read-only status derived from one durable replacement generation.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ReplacementRecoveryRecord {
|
||||
pub task_id: String,
|
||||
pub state: ReplacementRecoveryState,
|
||||
pub generation: Option<String>,
|
||||
pub set_disk_id: Option<String>,
|
||||
pub target_slots: Vec<String>,
|
||||
pub reason: Option<String>,
|
||||
pub verified_at: Option<u64>,
|
||||
}
|
||||
|
||||
impl ReplacementRecoveryRecord {
|
||||
pub(super) fn from_state(state: ResumeState) -> Option<Self> {
|
||||
if !is_replacement_intent(&state) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let invariant_holds = state.replacement_generation.as_deref() == Some(state.task_id.as_str())
|
||||
&& replacement_targets_match_identities(&state.replacement_targets, &state.replacement_target_identities);
|
||||
if !invariant_holds {
|
||||
return Some(Self::unknown(
|
||||
state.task_id,
|
||||
"durable replacement state violates its generation or target identity binding",
|
||||
));
|
||||
}
|
||||
|
||||
let (state_kind, reason) = if !state.completed && state.retry_count >= state.max_retries {
|
||||
(
|
||||
ReplacementRecoveryState::Unrecoverable,
|
||||
Some("replacement retry budget exhausted".to_string()),
|
||||
)
|
||||
} else if let Some(reason) = state.error_message.clone() {
|
||||
(ReplacementRecoveryState::Incomplete, Some(reason))
|
||||
} else {
|
||||
match state.replacement_phase {
|
||||
ReplacementPhase::Intent => (ReplacementRecoveryState::WaitingForReplacement, None),
|
||||
ReplacementPhase::Rebuilding => (ReplacementRecoveryState::Running, None),
|
||||
ReplacementPhase::Verified | ReplacementPhase::CleanupPending => (ReplacementRecoveryState::CleanupPending, None),
|
||||
ReplacementPhase::Abandoned => (
|
||||
ReplacementRecoveryState::Unrecoverable,
|
||||
Some("replacement generation was abandoned".to_string()),
|
||||
),
|
||||
ReplacementPhase::None => (ReplacementRecoveryState::Unknown, Some("replacement phase is missing".to_string())),
|
||||
}
|
||||
};
|
||||
|
||||
Some(Self {
|
||||
task_id: state.task_id,
|
||||
state: state_kind,
|
||||
generation: state.replacement_generation,
|
||||
set_disk_id: Some(state.set_disk_id),
|
||||
target_slots: state.replacement_targets,
|
||||
reason,
|
||||
verified_at: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn from_completion_proof(proof: &ReplacementCompletionProof) -> Self {
|
||||
Self {
|
||||
task_id: proof.task_id.clone(),
|
||||
state: ReplacementRecoveryState::Completed,
|
||||
generation: Some(proof.replacement_generation.clone()),
|
||||
set_disk_id: Some(proof.set_disk_id.clone()),
|
||||
target_slots: proof.replacement_targets.clone(),
|
||||
reason: None,
|
||||
verified_at: Some(proof.verified_at),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn unknown(task_id: String, reason: &str) -> Self {
|
||||
Self {
|
||||
task_id,
|
||||
state: ReplacementRecoveryState::Unknown,
|
||||
generation: None,
|
||||
set_disk_id: None,
|
||||
target_slots: Vec::new(),
|
||||
reason: Some(reason.to_string()),
|
||||
verified_at: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn replacement_targets_match_identities(targets: &[String], identities: &[ReplacementTargetIdentity]) -> bool {
|
||||
!targets.is_empty()
|
||||
&& targets.len() == identities.len()
|
||||
&& targets.iter().collect::<HashSet<_>>().len() == targets.len()
|
||||
&& identities.iter().map(|identity| &identity.endpoint).eq(targets.iter())
|
||||
}
|
||||
|
||||
/// Stable evidence for the mounted replacement instance that owns a repair
|
||||
/// generation. Endpoint text alone is not sufficient because a later disk can
|
||||
/// be mounted at the same configured path.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ReplacementTargetIdentity {
|
||||
pub endpoint: String,
|
||||
pub canonical_path: String,
|
||||
pub physical_device_ids: Vec<String>,
|
||||
pub filesystem_identity: String,
|
||||
}
|
||||
|
||||
/// Durable terminal evidence for one automatic replacement generation. This
|
||||
/// lives on the healthy non-target anchor rather than in the resumable state,
|
||||
/// because resume cleanup must not erase proof that the generation completed.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) struct ReplacementCompletionProof {
|
||||
pub schema_version: u32,
|
||||
pub task_id: String,
|
||||
pub replacement_generation: String,
|
||||
pub set_disk_id: String,
|
||||
pub replacement_targets: Vec<String>,
|
||||
pub replacement_target_identities: Vec<ReplacementTargetIdentity>,
|
||||
pub verified_at: u64,
|
||||
}
|
||||
|
||||
impl ReplacementCompletionProof {
|
||||
pub(super) fn from_state(state: &ResumeState, verified_at: u64) -> Result<Self> {
|
||||
let replacement_generation = state
|
||||
.replacement_generation
|
||||
.clone()
|
||||
.ok_or_else(|| Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion has no generation for task {}", state.task_id),
|
||||
})?;
|
||||
if replacement_generation != state.task_id
|
||||
|| state.replacement_targets.is_empty()
|
||||
|| state
|
||||
.replacement_target_identities
|
||||
.iter()
|
||||
.map(|identity| &identity.endpoint)
|
||||
.collect::<Vec<_>>()
|
||||
!= state.replacement_targets.iter().collect::<Vec<_>>()
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion identity does not match task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
schema_version: CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA,
|
||||
task_id: state.task_id.clone(),
|
||||
replacement_generation,
|
||||
set_disk_id: state.set_disk_id.clone(),
|
||||
replacement_targets: state.replacement_targets.clone(),
|
||||
replacement_target_identities: state.replacement_target_identities.clone(),
|
||||
verified_at,
|
||||
})
|
||||
}
|
||||
|
||||
fn matches_state(&self, state: &ResumeState) -> bool {
|
||||
self.schema_version == CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA
|
||||
&& self.task_id == state.task_id
|
||||
&& state.replacement_generation.as_deref() == Some(self.replacement_generation.as_str())
|
||||
&& self.set_disk_id == state.set_disk_id
|
||||
&& self.replacement_targets == state.replacement_targets
|
||||
&& self.replacement_target_identities == state.replacement_target_identities
|
||||
}
|
||||
|
||||
fn validate(&self, expected_task_id: &str) -> Result<()> {
|
||||
if self.schema_version != CURRENT_REPLACEMENT_COMPLETION_PROOF_SCHEMA {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion proof schema {} is unsupported", self.schema_version),
|
||||
});
|
||||
}
|
||||
validate_resume_task_id(expected_task_id)?;
|
||||
if self.task_id != expected_task_id
|
||||
|| self.replacement_generation != self.task_id
|
||||
|| self.set_disk_id.is_empty()
|
||||
|| self.verified_at == 0
|
||||
|| !replacement_targets_match_identities(&self.replacement_targets, &self.replacement_target_identities)
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion proof does not match task {expected_task_id}"),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn replacement_target_identities_match(
|
||||
expected: &[ReplacementTargetIdentity],
|
||||
actual: &[ReplacementTargetIdentity],
|
||||
) -> bool {
|
||||
let mut expected = expected.to_vec();
|
||||
let mut actual = actual.to_vec();
|
||||
expected.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
actual.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
expected == actual
|
||||
}
|
||||
|
||||
/// Build the canonical, provably-injective dedup identity for an object
|
||||
/// version. Length-prefixing the object key makes the encoding injective: no
|
||||
/// two distinct `(object, version_id)` pairs can collide, even for adversarial
|
||||
/// keys containing `:` or embedded null bytes. This is the single source of
|
||||
/// truth for per-version dedup across the heal loop and the checkpoint sets.
|
||||
pub fn compose_key(object: &str, version_id: Option<&str>) -> String {
|
||||
format!("{}:{}{}", object.len(), object, version_id.unwrap_or(""))
|
||||
}
|
||||
|
||||
impl ResumeManager {
|
||||
/// Seal a durably published intent before the caller may format a target.
|
||||
/// A torn intent without this seal is known to have failed before its
|
||||
/// creator returned and can be atomically recreated on retry.
|
||||
pub(super) async fn ensure_replacement_intent_seal(&self) -> Result<()> {
|
||||
let task_id = self.state.read().await.task_id.clone();
|
||||
validate_resume_task_id(&task_id)?;
|
||||
let path = replacement_intent_seal_path(&task_id);
|
||||
let path = path_to_str(&path)?;
|
||||
match self.disk.read_all(RUSTFS_META_BUCKET, path).await {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(DiskError::FileNotFound) => {}
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read replacement intent seal: {error}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
self.disk
|
||||
.write_all(RUSTFS_META_BUCKET, path, b"sealed".as_slice().into())
|
||||
.await
|
||||
.map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to save replacement intent seal: {error}"),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn mark_replacement_rebuilding(
|
||||
&self,
|
||||
mut replacement_target_identities: Vec<ReplacementTargetIdentity>,
|
||||
) -> Result<()> {
|
||||
replacement_target_identities.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
replacement_target_identities.dedup_by(|left, right| left.endpoint == right.endpoint);
|
||||
let mut state = self.state.write().await;
|
||||
if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement intent is not active for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
if replacement_target_identities
|
||||
.iter()
|
||||
.map(|identity| &identity.endpoint)
|
||||
.collect::<Vec<_>>()
|
||||
!= state.replacement_targets.iter().collect::<Vec<_>>()
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement identities do not match targets for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
if !replacement_target_identities_match(&state.replacement_target_identities, &replacement_target_identities) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement target changed after format for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
state.replacement_phase = ReplacementPhase::Rebuilding;
|
||||
state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
drop(state);
|
||||
self.save_state_strict().await
|
||||
}
|
||||
|
||||
/// Persist survivor-anchor completion proof before transitioning this
|
||||
/// resumable state to `Verified`. If proof persistence fails, this state
|
||||
/// stays rebuildable and the caller must retain the healing marker.
|
||||
pub async fn mark_replacement_completed_and_verified(&self) -> Result<()> {
|
||||
let state = self.state.read().await.clone();
|
||||
if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement verification is not active for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
let proof = self.write_replacement_completion_proof(&state, None).await?;
|
||||
|
||||
let mut state = self.state.write().await;
|
||||
if !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement verification changed for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
state.mark_completed();
|
||||
state.replacement_phase = ReplacementPhase::Verified;
|
||||
state.last_update = proof.verified_at;
|
||||
drop(state);
|
||||
self.save_state_strict().await
|
||||
}
|
||||
|
||||
/// Verify or backfill the terminal proof before marker removal or resume
|
||||
/// cleanup. This supports restart recovery from a `Verified` state written
|
||||
/// by a prior binary that did not yet have a separate proof record.
|
||||
pub(crate) async fn ensure_replacement_completion_proof(&self) -> Result<ReplacementCompletionProof> {
|
||||
let state = self.state.read().await.clone();
|
||||
if !state.completed || !matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion is not verified for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
self.write_replacement_completion_proof(&state, Some(state.last_update)).await
|
||||
}
|
||||
|
||||
/// Record that the healing markers have been removed, so a later retry can
|
||||
/// safely delete the remaining resume artifacts without touching markers.
|
||||
pub async fn mark_replacement_cleanup_pending(&self) -> Result<()> {
|
||||
let mut state = self.state.write().await;
|
||||
if !state.completed || !matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement cleanup is not ready for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
state.replacement_phase = ReplacementPhase::CleanupPending;
|
||||
state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
drop(state);
|
||||
self.save_state_strict().await
|
||||
}
|
||||
|
||||
/// Load the durable terminal proof from the healthy survivor anchor.
|
||||
pub(crate) async fn load_replacement_completion_proof(disk: DiskStore, task_id: &str) -> Result<ReplacementCompletionProof> {
|
||||
Self::replacement_completion_proof_if_present(disk, task_id)
|
||||
.await?
|
||||
.ok_or_else(|| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read replacement completion proof: proof is missing for task {task_id}"),
|
||||
})
|
||||
}
|
||||
|
||||
async fn replacement_completion_proof_if_present(
|
||||
disk: DiskStore,
|
||||
task_id: &str,
|
||||
) -> Result<Option<ReplacementCompletionProof>> {
|
||||
validate_resume_task_id(task_id)?;
|
||||
let mut proofs = Vec::new();
|
||||
for path in [
|
||||
replacement_completion_proof_path(task_id),
|
||||
legacy_replacement_completion_proof_path(task_id),
|
||||
] {
|
||||
let path_str = path_to_str(&path)?;
|
||||
let bytes = match disk.read_all(RUSTFS_META_BUCKET, path_str).await {
|
||||
Ok(bytes) => bytes,
|
||||
Err(DiskError::FileNotFound) => continue,
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read replacement completion proof: {error}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
let proof: ReplacementCompletionProof =
|
||||
serde_json::from_slice(&bytes).map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to deserialize replacement completion proof: {error}"),
|
||||
})?;
|
||||
proof.validate(task_id)?;
|
||||
proofs.push(proof);
|
||||
}
|
||||
|
||||
match proofs.as_slice() {
|
||||
[] => Ok(None),
|
||||
[proof] => Ok(Some(proof.clone())),
|
||||
[proof, legacy_proof] if proof == legacy_proof => Ok(Some(proof.clone())),
|
||||
_ => Err(replacement_recovery_conflict(format!(
|
||||
"Replacement completion proof conflicts with legacy proof for task {task_id}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reconcile the proof-first publication order after a crash. A matching
|
||||
/// proof is durable evidence that rebuilding finished, so it must win over
|
||||
/// an older active state before a retry may format the target again.
|
||||
pub(super) async fn reconcile_replacement_completion_proof(&self) -> Result<()> {
|
||||
let task_id = self.state.read().await.task_id.clone();
|
||||
let Some(proof) = Self::replacement_completion_proof_if_present(self.disk.clone(), &task_id).await? else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let mut state = self.state.write().await;
|
||||
if !proof.matches_state(&state) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion proof does not match active intent for task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
if state.completed && matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending) {
|
||||
return Ok(());
|
||||
}
|
||||
if state.completed || !matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding) {
|
||||
return Err(replacement_recovery_conflict(format!(
|
||||
"Replacement completion proof conflicts with state for task {}",
|
||||
state.task_id
|
||||
)));
|
||||
}
|
||||
|
||||
state.mark_completed();
|
||||
state.replacement_phase = ReplacementPhase::Verified;
|
||||
state.last_update = proof.verified_at;
|
||||
drop(state);
|
||||
self.save_state_strict().await
|
||||
}
|
||||
|
||||
pub(super) async fn migrate_legacy_replacement_completion_proof(disk: &DiskStore, task_id: &str) -> Result<bool> {
|
||||
validate_resume_task_id(task_id)?;
|
||||
let legacy_path = legacy_replacement_completion_proof_path(task_id);
|
||||
let legacy_path_str = path_to_str(&legacy_path)?;
|
||||
let legacy_bytes = match disk.read_all(RUSTFS_META_BUCKET, legacy_path_str).await {
|
||||
Ok(bytes) => bytes,
|
||||
Err(DiskError::FileNotFound) => return Ok(false),
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read legacy replacement completion proof: {error}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
let legacy_proof: ReplacementCompletionProof = serde_json::from_slice(&legacy_bytes).map_err(|error| {
|
||||
replacement_recovery_corruption(format!("Failed to deserialize legacy replacement completion proof: {error}"))
|
||||
})?;
|
||||
legacy_proof
|
||||
.validate(task_id)
|
||||
.map_err(|error| replacement_recovery_corruption(format!("Invalid legacy replacement completion proof: {error}")))?;
|
||||
|
||||
ensure_replacement_recovery_dir(disk)
|
||||
.await
|
||||
.map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to create replacement recovery directory: {error}"),
|
||||
})?;
|
||||
let path = replacement_completion_proof_path(task_id);
|
||||
let path_str = path_to_str(&path)?;
|
||||
for _ in 0..2 {
|
||||
match disk.read_all(RUSTFS_META_BUCKET, path_str).await {
|
||||
Ok(bytes) => {
|
||||
let proof: ReplacementCompletionProof =
|
||||
serde_json::from_slice(&bytes).map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to deserialize replacement completion proof: {error}"),
|
||||
})?;
|
||||
proof.validate(task_id).map_err(|error| {
|
||||
replacement_recovery_corruption(format!("Invalid replacement completion proof: {error}"))
|
||||
})?;
|
||||
if proof != legacy_proof {
|
||||
return Err(replacement_recovery_conflict(format!(
|
||||
"Replacement completion proof conflicts with legacy proof for task {task_id}"
|
||||
)));
|
||||
}
|
||||
delete_resume_file(disk, &legacy_path).await?;
|
||||
return Ok(true);
|
||||
}
|
||||
Err(DiskError::FileNotFound) => {}
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read replacement completion proof: {error}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
match super::super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path_str,
|
||||
None,
|
||||
Some(legacy_bytes.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(EcstoreConditionalFileUpdate::Updated) => {
|
||||
delete_resume_file(disk, &legacy_path).await?;
|
||||
return Ok(true);
|
||||
}
|
||||
Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue,
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to migrate replacement completion proof: {error}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion proof changed while migrating task {task_id}"),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn abandon_replacement_intent(&self) -> Result<()> {
|
||||
let mut state = self.state.write().await;
|
||||
if matches!(state.replacement_phase, ReplacementPhase::Abandoned) {
|
||||
return Ok(());
|
||||
}
|
||||
state.replacement_phase = ReplacementPhase::Abandoned;
|
||||
state.last_update = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
drop(state);
|
||||
self.save_state_strict().await
|
||||
}
|
||||
|
||||
pub async fn set_replacement_targets(&self, replacement_targets: Vec<String>) -> Result<()> {
|
||||
{
|
||||
let mut state = self.state.write().await;
|
||||
state.replacement_targets = replacement_targets;
|
||||
}
|
||||
self.save_state().await
|
||||
}
|
||||
|
||||
pub(super) async fn publish_new_replacement_intent(&self, expected: Option<EcstoreDiskBytes>) -> Result<()> {
|
||||
let state = self.state.read().await.clone();
|
||||
validate_resume_task_id(&state.task_id)?;
|
||||
let state_data = EcstoreDiskBytes::from(serde_json::to_vec(&state).map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to serialize resume state: {error}"),
|
||||
})?);
|
||||
let path = self.state_file.path(&state.task_id);
|
||||
let path = path_to_str(&path)?;
|
||||
|
||||
ensure_replacement_recovery_dir(&self.disk)
|
||||
.await
|
||||
.map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to create replacement recovery directory: {error}"),
|
||||
})?;
|
||||
match super::super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file(
|
||||
self.disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
expected,
|
||||
Some(state_data),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(EcstoreConditionalFileUpdate::Updated) => Ok(()),
|
||||
Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => {
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement intent changed before publication for task {}", state.task_id),
|
||||
})
|
||||
}
|
||||
Err(error) => Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to save resume state: {error}"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_replacement_completion_proof(
|
||||
&self,
|
||||
state: &ResumeState,
|
||||
verified_at: Option<u64>,
|
||||
) -> Result<ReplacementCompletionProof> {
|
||||
ensure_replacement_recovery_dir(&self.disk)
|
||||
.await
|
||||
.map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to create replacement recovery directory: {error}"),
|
||||
})?;
|
||||
let path = replacement_completion_proof_path(&state.task_id);
|
||||
let path_str = path_to_str(&path)?;
|
||||
let proof = ReplacementCompletionProof::from_state(
|
||||
state,
|
||||
verified_at.unwrap_or_else(|| SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs()),
|
||||
)?;
|
||||
let proof_data = EcstoreDiskBytes::from(serde_json::to_vec(&proof).map_err(|e| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to serialize replacement completion proof: {e}"),
|
||||
})?);
|
||||
if let Some(error) = injected_replacement_proof_write_error(path_str) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to save replacement completion proof: {error}"),
|
||||
});
|
||||
}
|
||||
|
||||
// Publish through the disk CAS primitive: `write_all` can expose a
|
||||
// partially written proof to a crash/restart reader. If a prior
|
||||
// version left torn bytes behind, replace exactly the observed bytes;
|
||||
// a concurrently published valid proof is never overwritten.
|
||||
for _ in 0..2 {
|
||||
let expected = match self.disk.read_all(RUSTFS_META_BUCKET, path_str).await {
|
||||
Ok(existing) => match serde_json::from_slice::<ReplacementCompletionProof>(&existing) {
|
||||
Ok(existing_proof) => {
|
||||
existing_proof.validate(&state.task_id)?;
|
||||
if existing_proof.matches_state(state) {
|
||||
return Ok(existing_proof);
|
||||
}
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion proof does not match task {}", state.task_id),
|
||||
});
|
||||
}
|
||||
Err(_) => Some(existing),
|
||||
},
|
||||
Err(DiskError::FileNotFound) => None,
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to read replacement completion proof: {error}"),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
match super::super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file(
|
||||
self.disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path_str,
|
||||
expected,
|
||||
Some(proof_data.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(EcstoreConditionalFileUpdate::Updated) => return Ok(proof),
|
||||
Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue,
|
||||
Err(error) => {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to save replacement completion proof: {error}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement completion proof changed while publishing task {}", state.task_id),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn write_replacement_intent_state(
|
||||
&self,
|
||||
path: &str,
|
||||
state_data: EcstoreDiskBytes,
|
||||
) -> std::result::Result<(), DiskError> {
|
||||
ensure_replacement_recovery_dir(&self.disk).await?;
|
||||
for _ in 0..2 {
|
||||
let expected = match self.disk.read_all(RUSTFS_META_BUCKET, path).await {
|
||||
Ok(existing) => Some(existing),
|
||||
Err(DiskError::FileNotFound) => None,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
match super::super::storage_api::owner::EcstoreDiskAPI::compare_and_update_file(
|
||||
self.disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
expected,
|
||||
Some(state_data.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(EcstoreConditionalFileUpdate::Updated) => return Ok(()),
|
||||
Ok(EcstoreConditionalFileUpdate::Missing | EcstoreConditionalFileUpdate::Mismatch) => continue,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
Err(DiskError::other("replacement intent changed while publishing"))
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,311 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{Error, Result};
|
||||
use std::collections::HashSet;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tracing::{debug, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::super::{BUCKET_META_PREFIX, DiskError, DiskStore, HealDiskExt as _, RUSTFS_META_BUCKET};
|
||||
use super::replacement::{ReplacementPhase, ReplacementRecoveryRecord};
|
||||
use super::{
|
||||
EVENT_HEAL_RESUME_STATE, LOG_COMPONENT_HEAL, LOG_SUBSYSTEM_RESUME, REPLACEMENT_COMPLETION_PROOF_FILE,
|
||||
REPLACEMENT_INTENT_FILE, RESUME_STATE_FILE, ResumeManager, ResumeStateFile, is_replacement_intent, path_to_str,
|
||||
replacement_recovery_corruption_for_state_load, replacement_recovery_dir, validate_resume_task_id,
|
||||
};
|
||||
|
||||
/// resume utils
|
||||
pub struct ResumeUtils;
|
||||
|
||||
impl ResumeUtils {
|
||||
/// generate unique task id
|
||||
pub fn generate_task_id() -> String {
|
||||
Uuid::new_v4().to_string()
|
||||
}
|
||||
|
||||
/// check if task can be resumed
|
||||
pub async fn can_resume_task(disk: &DiskStore, task_id: &str) -> bool {
|
||||
ResumeManager::has_resume_state(disk, task_id).await
|
||||
}
|
||||
|
||||
/// get all resumable task ids
|
||||
pub async fn get_resumable_tasks(disk: &DiskStore) -> Result<Vec<String>> {
|
||||
// List all files in the buckets metadata directory
|
||||
let entries = match disk.list_dir("", RUSTFS_META_BUCKET, BUCKET_META_PREFIX, -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(e) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
state = "list_failed",
|
||||
error = %e,
|
||||
"Heal resume state listing failed"
|
||||
);
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
};
|
||||
|
||||
let mut task_ids = Vec::new();
|
||||
|
||||
// Filter files that end with ahm_resume_state.json and extract task IDs
|
||||
for entry in entries {
|
||||
if entry.ends_with(&format!("_{RESUME_STATE_FILE}")) {
|
||||
// Extract task ID from filename: {task_id}_ahm_resume_state.json
|
||||
if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}"))
|
||||
&& validate_resume_task_id(task_id).is_ok()
|
||||
{
|
||||
task_ids.push(task_id.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_count = task_ids.len(),
|
||||
state = "listed",
|
||||
"Heal resume states listed"
|
||||
);
|
||||
Ok(task_ids)
|
||||
}
|
||||
|
||||
/// Return replacement intent task IDs from the dedicated recovery
|
||||
/// directory. Periodic recovery must never enumerate the ordinary resume
|
||||
/// directory, whose cardinality is unrelated to replacement work.
|
||||
pub async fn get_replacement_intent_tasks(disk: &DiskStore) -> Result<Vec<String>> {
|
||||
let entries = Self::replacement_recovery_entries(disk).await?;
|
||||
let suffix = format!("_{REPLACEMENT_INTENT_FILE}");
|
||||
let mut task_ids = HashSet::new();
|
||||
|
||||
for entry in entries {
|
||||
if let Some(task_id) = entry.strip_suffix(&suffix)
|
||||
&& validate_resume_task_id(task_id).is_ok()
|
||||
{
|
||||
task_ids.insert(task_id.to_string());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let mut task_ids = task_ids.into_iter().collect::<Vec<_>>();
|
||||
task_ids.sort_unstable();
|
||||
Ok(task_ids)
|
||||
}
|
||||
|
||||
async fn replacement_recovery_entries(disk: &DiskStore) -> Result<Vec<String>> {
|
||||
let recovery_dir = replacement_recovery_dir();
|
||||
let recovery_dir = path_to_str(&recovery_dir)?;
|
||||
match disk.list_dir("", RUSTFS_META_BUCKET, recovery_dir, -1).await {
|
||||
Ok(entries) => Ok(entries),
|
||||
Err(DiskError::FileNotFound) => Ok(Vec::new()),
|
||||
Err(error @ DiskError::UnformattedDisk) => Err(error.into()),
|
||||
Err(error) => Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to list replacement recovery records: {error}"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Migrate flat replacement artifacts from earlier builds exactly once at
|
||||
/// manager startup. The normal scanner only uses the dedicated directory;
|
||||
/// ordinary resume JSON is never read on its periodic path.
|
||||
pub async fn migrate_legacy_replacement_records(disk: &DiskStore) -> Result<()> {
|
||||
let entries = disk
|
||||
.list_dir("", RUSTFS_META_BUCKET, BUCKET_META_PREFIX, -1)
|
||||
.await
|
||||
.map_err(|error| Error::TaskExecutionFailed {
|
||||
message: format!("Failed to list legacy replacement records: {error}"),
|
||||
})?;
|
||||
let ordinary_suffix = format!("_{RESUME_STATE_FILE}");
|
||||
let intent_suffix = format!("_{REPLACEMENT_INTENT_FILE}");
|
||||
let proof_suffix = format!("_{REPLACEMENT_COMPLETION_PROOF_FILE}");
|
||||
let mut ordinary_task_ids = HashSet::new();
|
||||
let mut intent_task_ids = HashSet::new();
|
||||
let mut proof_task_ids = HashSet::new();
|
||||
|
||||
for entry in entries {
|
||||
if let Some(task_id) = entry.strip_suffix(&intent_suffix)
|
||||
&& validate_resume_task_id(task_id).is_ok()
|
||||
{
|
||||
intent_task_ids.insert(task_id.to_string());
|
||||
continue;
|
||||
}
|
||||
if let Some(task_id) = entry.strip_suffix(&ordinary_suffix)
|
||||
&& validate_resume_task_id(task_id).is_ok()
|
||||
{
|
||||
ordinary_task_ids.insert(task_id.to_string());
|
||||
continue;
|
||||
}
|
||||
if let Some(task_id) = entry.strip_suffix(&proof_suffix)
|
||||
&& validate_resume_task_id(task_id).is_ok()
|
||||
{
|
||||
proof_task_ids.insert(task_id.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let mut state_task_ids = intent_task_ids.into_iter().collect::<Vec<_>>();
|
||||
state_task_ids.extend(ordinary_task_ids);
|
||||
state_task_ids.sort_unstable();
|
||||
state_task_ids.dedup();
|
||||
for task_id in state_task_ids {
|
||||
let has_flat_intent = ResumeManager::has_state_file(disk, &task_id, ResumeStateFile::LegacyReplacementIntent).await;
|
||||
if !has_flat_intent {
|
||||
let manager = ResumeManager::load_from_disk(disk.clone(), &task_id).await.map_err(|error| {
|
||||
replacement_recovery_corruption_for_state_load(
|
||||
format!("Failed to load legacy replacement recovery candidate {task_id}"),
|
||||
error,
|
||||
)
|
||||
})?;
|
||||
if !is_replacement_intent(&manager.get_state().await) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
ResumeManager::load_replacement_intent(disk.clone(), &task_id).await?;
|
||||
}
|
||||
|
||||
for task_id in proof_task_ids {
|
||||
ResumeManager::migrate_legacy_replacement_completion_proof(disk, &task_id).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return all durable replacement states and completion proofs stored on
|
||||
/// one survivor disk. Unlike the legacy resumable-task helper, listing
|
||||
/// failures are returned to the caller so an observability surface cannot
|
||||
/// silently turn an unreadable durable record into a green result.
|
||||
pub async fn get_replacement_recovery_records(disk: &DiskStore) -> Result<Vec<ReplacementRecoveryRecord>> {
|
||||
let entries = Self::replacement_recovery_entries(disk).await?;
|
||||
let proof_suffix = format!("_{REPLACEMENT_COMPLETION_PROOF_FILE}");
|
||||
let mut records = Vec::new();
|
||||
let mut intent_task_ids = HashSet::new();
|
||||
|
||||
for task_id in Self::get_replacement_intent_tasks(disk).await? {
|
||||
let state = ResumeManager::load_replacement_intent(disk.clone(), &task_id)
|
||||
.await?
|
||||
.get_state()
|
||||
.await;
|
||||
intent_task_ids.insert(task_id.clone());
|
||||
records.push(ReplacementRecoveryRecord::from_state(state).unwrap_or_else(|| {
|
||||
ReplacementRecoveryRecord::unknown(
|
||||
task_id,
|
||||
"isolated replacement intent violates its generation or target identity binding",
|
||||
)
|
||||
}));
|
||||
}
|
||||
|
||||
for entry in entries {
|
||||
let Some(task_id) = entry.strip_suffix(&proof_suffix) else {
|
||||
continue;
|
||||
};
|
||||
if validate_resume_task_id(task_id).is_err() {
|
||||
continue;
|
||||
}
|
||||
if intent_task_ids.contains(task_id) {
|
||||
continue;
|
||||
}
|
||||
let proof = ResumeManager::load_replacement_completion_proof(disk.clone(), task_id).await?;
|
||||
records.push(ReplacementRecoveryRecord::from_completion_proof(&proof));
|
||||
}
|
||||
|
||||
records.sort_by(|left, right| left.task_id.cmp(&right.task_id).then(left.state.cmp(&right.state)));
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
/// cleanup expired resume states
|
||||
pub async fn cleanup_expired_states(disk: &DiskStore, max_age_hours: u64) -> Result<()> {
|
||||
let task_ids = Self::get_resumable_tasks(disk).await?;
|
||||
let current_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
|
||||
|
||||
for task_id in task_ids {
|
||||
if let Ok(resume_manager) = ResumeManager::load_from_disk(disk.clone(), &task_id).await {
|
||||
let state = resume_manager.get_state().await;
|
||||
let age_hours = current_time.saturating_sub(state.last_update) / 3600;
|
||||
|
||||
if !state.completed && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if state.completed
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if age_hours > max_age_hours {
|
||||
debug!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id,
|
||||
age_hours,
|
||||
state = "expired_cleanup_started",
|
||||
"Heal resume cleanup started"
|
||||
);
|
||||
if let Err(e) = resume_manager.cleanup().await {
|
||||
warn!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id,
|
||||
age_hours,
|
||||
state = "expired_cleanup_failed",
|
||||
error = %e,
|
||||
"Heal resume state cleanup failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for task_id in Self::get_replacement_intent_tasks(disk).await? {
|
||||
if let Ok(resume_manager) = ResumeManager::load_replacement_intent(disk.clone(), &task_id).await {
|
||||
let state = resume_manager.get_state().await;
|
||||
let age_hours = current_time.saturating_sub(state.last_update) / 3600;
|
||||
|
||||
if !state.completed && matches!(state.replacement_phase, ReplacementPhase::Intent | ReplacementPhase::Rebuilding)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if state.completed
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::Verified | ReplacementPhase::CleanupPending)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if age_hours > max_age_hours
|
||||
&& let Err(e) = resume_manager.cleanup().await
|
||||
{
|
||||
warn!(
|
||||
target: "rustfs::heal::resume",
|
||||
event = EVENT_HEAL_RESUME_STATE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_RESUME,
|
||||
task_id,
|
||||
age_hours,
|
||||
state = "expired_cleanup_failed",
|
||||
error = %e,
|
||||
"Replacement intent cleanup failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
+93
-550
@@ -27,7 +27,7 @@ use super::storage_api::storage::{
|
||||
BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _,
|
||||
ObjectOperations as _, StorageAdminApi,
|
||||
};
|
||||
use super::{DiskStore, ECStore, Endpoint, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity};
|
||||
use super::{DiskStore, ECStore, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity};
|
||||
pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
@@ -42,7 +42,10 @@ pub struct HealLifecycleExpiryContext {
|
||||
|
||||
enum HealLifecycleExpiryContextInner {
|
||||
Ecstore(EcstoreHealLifecycleExpiryContext),
|
||||
#[allow(dead_code)]
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "constructed by the #[cfg(test)] `test()` helper; the lib target cannot see test-only consumers (backlog#1823)"
|
||||
)]
|
||||
Test,
|
||||
}
|
||||
|
||||
@@ -65,7 +68,6 @@ const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_STORAGE: &str = "storage";
|
||||
const EVENT_HEAL_STORAGE_OBJECT_IO: &str = "heal_storage_object_io";
|
||||
const EVENT_HEAL_STORAGE_OBJECT_READ_LIMIT: &str = "heal_storage_object_read_limit";
|
||||
const EVENT_HEAL_STORAGE_OBJECT_VERIFY: &str = "heal_storage_object_verify";
|
||||
const EVENT_HEAL_STORAGE_ADMIN_OP: &str = "heal_storage_admin_op";
|
||||
const EVENT_HEAL_STORAGE_REPAIR_OP: &str = "heal_storage_repair_op";
|
||||
|
||||
@@ -312,56 +314,23 @@ pub struct HealListItem {
|
||||
pub is_delete_marker: bool,
|
||||
}
|
||||
|
||||
/// Disk status for heal operations
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum DiskStatus {
|
||||
/// Ok
|
||||
Ok,
|
||||
/// Offline
|
||||
Offline,
|
||||
/// Corrupt
|
||||
Corrupt,
|
||||
/// Missing
|
||||
Missing,
|
||||
/// Permission denied
|
||||
PermissionDenied,
|
||||
/// Faulty
|
||||
Faulty,
|
||||
/// Root mount
|
||||
RootMount,
|
||||
/// Unknown
|
||||
Unknown,
|
||||
/// Unformatted
|
||||
Unformatted,
|
||||
}
|
||||
|
||||
/// Heal storage layer interface
|
||||
#[async_trait]
|
||||
pub trait HealStorageAPI: Send + Sync {
|
||||
/// Get object meta
|
||||
///
|
||||
/// Reserved for HS-01 MRF wiring (rustfs/backlog#1865): MRF intents
|
||||
/// currently execute through `heal_object`; keep this entry point for the
|
||||
/// metadata-corruption variant that must inspect metadata first.
|
||||
async fn get_object_meta(&self, bucket: &str, object: &str) -> Result<Option<HealObjectInfo>>;
|
||||
|
||||
/// Get object data
|
||||
async fn get_object_data(&self, bucket: &str, object: &str) -> Result<Option<Vec<u8>>>;
|
||||
|
||||
/// Put object data
|
||||
async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()>;
|
||||
|
||||
/// Delete object
|
||||
async fn delete_object(&self, bucket: &str, object: &str) -> Result<()>;
|
||||
|
||||
/// Check object integrity
|
||||
async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result<bool>;
|
||||
|
||||
/// EC decode rebuild
|
||||
///
|
||||
/// Reserved for HS-01 MRF wiring (rustfs/backlog#1865): urgent ECDecode
|
||||
/// requests currently execute through `heal_object`; keep the explicit
|
||||
/// rebuild-and-read path for the decode-failure fast variant.
|
||||
async fn ec_decode_rebuild(&self, bucket: &str, object: &str) -> Result<Vec<u8>>;
|
||||
|
||||
/// Get disk status
|
||||
async fn get_disk_status(&self, endpoint: &Endpoint) -> Result<DiskStatus>;
|
||||
|
||||
/// Format disk
|
||||
async fn format_disk(&self, endpoint: &Endpoint) -> Result<()>;
|
||||
|
||||
/// Get bucket info
|
||||
async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>>;
|
||||
|
||||
@@ -387,21 +356,12 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Fix bucket metadata
|
||||
async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()>;
|
||||
|
||||
/// Get all buckets
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>>;
|
||||
|
||||
/// Check object exists
|
||||
async fn object_exists(&self, bucket: &str, object: &str) -> Result<bool>;
|
||||
|
||||
/// Get object size
|
||||
async fn get_object_size(&self, bucket: &str, object: &str) -> Result<Option<u64>>;
|
||||
|
||||
/// Get object checksum
|
||||
async fn get_object_checksum(&self, bucket: &str, object: &str) -> Result<Option<String>>;
|
||||
|
||||
/// Heal object using ecstore
|
||||
async fn heal_object(
|
||||
&self,
|
||||
@@ -453,12 +413,6 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// List object versions for healing (returns all versions, may use significant memory for large buckets)
|
||||
///
|
||||
/// WARNING: This method loads all object versions into memory at once. For buckets with many
|
||||
/// objects/versions, consider using `list_objects_for_heal_page` instead to process versions in pages.
|
||||
async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result<Vec<HealListItem>>;
|
||||
|
||||
/// List object versions for healing with pagination (returns one page and continuation token)
|
||||
/// Returns (versions, next_continuation_token, is_truncated). The continuation token is an
|
||||
/// opaque composite `(marker, version_marker)` value — see `encode_heal_token`/`decode_heal_token`.
|
||||
@@ -527,89 +481,11 @@ impl ECStoreHealStorage {
|
||||
pub fn new(ecstore: Arc<ECStore>) -> Self {
|
||||
Self { ecstore }
|
||||
}
|
||||
}
|
||||
|
||||
fn is_transient_object_exists_message(message: &str) -> bool {
|
||||
let message = message.to_ascii_lowercase();
|
||||
|
||||
[
|
||||
"failed to acquire read lock",
|
||||
"lock acquisition failed",
|
||||
"lock acquisition timeout",
|
||||
"quorum not reached",
|
||||
"deadline has elapsed",
|
||||
"timed out",
|
||||
"network error",
|
||||
"transport error",
|
||||
"connection refused",
|
||||
]
|
||||
.iter()
|
||||
.any(|pattern| message.contains(pattern))
|
||||
}
|
||||
|
||||
fn is_transient_object_exists_error(err: &StorageError) -> bool {
|
||||
if err.is_quorum_error() {
|
||||
return true;
|
||||
}
|
||||
|
||||
match err {
|
||||
StorageError::Lock(lock_err) => lock_err.is_retryable() || is_transient_object_exists_message(&lock_err.to_string()),
|
||||
StorageError::Io(io_err) => is_transient_object_exists_message(&io_err.to_string()),
|
||||
StorageError::SlowDown | StorageError::OperationCanceled => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl HealStorageAPI for ECStoreHealStorage {
|
||||
async fn get_object_meta(&self, bucket: &str, object: &str) -> Result<Option<HealObjectInfo>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_meta",
|
||||
bucket,
|
||||
object,
|
||||
"Heal storage request started"
|
||||
);
|
||||
|
||||
match self.ecstore.get_object_info(bucket, object, &Default::default()).await {
|
||||
Ok(info) => Ok(Some(info)),
|
||||
Err(e) => {
|
||||
// Map ObjectNotFound to None to align with Option return type
|
||||
if matches!(e, StorageError::ObjectNotFound(_, _)) {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_meta",
|
||||
bucket,
|
||||
object,
|
||||
result = "not_found",
|
||||
"Heal storage object metadata missing"
|
||||
);
|
||||
Ok(None)
|
||||
} else {
|
||||
error!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_meta",
|
||||
bucket,
|
||||
object,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal storage request failed"
|
||||
);
|
||||
Err(Error::other(e))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read back an object's bytes, capped to bound memory.
|
||||
///
|
||||
/// Private support for the reserved `ec_decode_rebuild` (HS-01); not part
|
||||
/// of the storage trait surface.
|
||||
async fn get_object_data(&self, bucket: &str, object: &str) -> Result<Option<Vec<u8>>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
@@ -695,196 +571,85 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
Ok(Some(buf))
|
||||
}
|
||||
}
|
||||
|
||||
async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()> {
|
||||
fn is_transient_object_exists_message(message: &str) -> bool {
|
||||
let message = message.to_ascii_lowercase();
|
||||
|
||||
[
|
||||
"failed to acquire read lock",
|
||||
"lock acquisition failed",
|
||||
"lock acquisition timeout",
|
||||
"quorum not reached",
|
||||
"deadline has elapsed",
|
||||
"timed out",
|
||||
"network error",
|
||||
"transport error",
|
||||
"connection refused",
|
||||
]
|
||||
.iter()
|
||||
.any(|pattern| message.contains(pattern))
|
||||
}
|
||||
|
||||
fn is_transient_object_exists_error(err: &StorageError) -> bool {
|
||||
if err.is_quorum_error() {
|
||||
return true;
|
||||
}
|
||||
|
||||
match err {
|
||||
StorageError::Lock(lock_err) => lock_err.is_retryable() || is_transient_object_exists_message(&lock_err.to_string()),
|
||||
StorageError::Io(io_err) => is_transient_object_exists_message(&io_err.to_string()),
|
||||
StorageError::SlowDown | StorageError::OperationCanceled => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl HealStorageAPI for ECStoreHealStorage {
|
||||
async fn get_object_meta(&self, bucket: &str, object: &str) -> Result<Option<HealObjectInfo>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "put_object_data",
|
||||
bucket,
|
||||
object,
|
||||
bytes = data.len(),
|
||||
"Heal storage request started"
|
||||
);
|
||||
|
||||
let mut reader = HealPutObjReader::from_vec(data.to_vec());
|
||||
match (*self.ecstore)
|
||||
.put_object(bucket, object, &mut reader, &Default::default())
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "put_object_data",
|
||||
bucket,
|
||||
object,
|
||||
result = "ok",
|
||||
"Heal storage object write completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "put_object_data",
|
||||
bucket,
|
||||
object,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal storage request failed"
|
||||
);
|
||||
Err(Error::other(e))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_object(&self, bucket: &str, object: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "delete_object",
|
||||
operation = "get_object_meta",
|
||||
bucket,
|
||||
object,
|
||||
"Heal storage request started"
|
||||
);
|
||||
|
||||
match self.ecstore.delete_object(bucket, object, Default::default()).await {
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "delete_object",
|
||||
bucket,
|
||||
object,
|
||||
result = "ok",
|
||||
"Heal storage object delete completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
match self.ecstore.get_object_info(bucket, object, &Default::default()).await {
|
||||
Ok(info) => Ok(Some(info)),
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "delete_object",
|
||||
bucket,
|
||||
object,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal storage request failed"
|
||||
);
|
||||
Err(Error::other(e))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result<bool> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
bucket,
|
||||
object,
|
||||
state = "started",
|
||||
"Heal storage object verification started"
|
||||
);
|
||||
|
||||
// Check object metadata first
|
||||
match self.get_object_meta(bucket, object).await? {
|
||||
Some(obj_info) => {
|
||||
if obj_info.size < 0 {
|
||||
warn!(
|
||||
// Map ObjectNotFound to None to align with Option return type
|
||||
if matches!(e, StorageError::ObjectNotFound(_, _)) {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_meta",
|
||||
bucket,
|
||||
object,
|
||||
state = "invalid_size",
|
||||
"Heal storage object verification failed"
|
||||
result = "not_found",
|
||||
"Heal storage object metadata missing"
|
||||
);
|
||||
return Ok(false);
|
||||
Ok(None)
|
||||
} else {
|
||||
error!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_meta",
|
||||
bucket,
|
||||
object,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal storage request failed"
|
||||
);
|
||||
Err(Error::other(e))
|
||||
}
|
||||
|
||||
// Stream-read the object to a sink to avoid loading into memory
|
||||
match (*self.ecstore)
|
||||
.get_object_reader(bucket, object, None, Default::default(), &Default::default())
|
||||
.await
|
||||
{
|
||||
Ok(reader) => {
|
||||
let mut stream = reader.stream;
|
||||
match tokio::io::copy(&mut stream, &mut tokio::io::sink()).await {
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
bucket,
|
||||
object,
|
||||
state = "ok",
|
||||
"Heal storage object verified"
|
||||
);
|
||||
Ok(true)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
bucket,
|
||||
object,
|
||||
state = "stream_read_failed",
|
||||
error = %e,
|
||||
"Heal storage object verification failed"
|
||||
);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
bucket,
|
||||
object,
|
||||
state = "reader_open_failed",
|
||||
error = %e,
|
||||
"Heal storage object verification failed"
|
||||
);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
None => {
|
||||
warn!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_VERIFY,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
bucket,
|
||||
object,
|
||||
state = "metadata_missing",
|
||||
"Heal storage object verification failed"
|
||||
);
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -976,81 +741,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_disk_status(&self, endpoint: &Endpoint) -> Result<DiskStatus> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_disk_status",
|
||||
endpoint = ?endpoint,
|
||||
state = "started",
|
||||
"Heal storage admin operation started"
|
||||
);
|
||||
|
||||
// TODO: implement disk status check using ecstore
|
||||
// For now, return Ok status
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_disk_status",
|
||||
endpoint = ?endpoint,
|
||||
result = "ok",
|
||||
disk_status = "ok",
|
||||
"Heal storage disk status resolved"
|
||||
);
|
||||
Ok(DiskStatus::Ok)
|
||||
}
|
||||
|
||||
async fn format_disk(&self, endpoint: &Endpoint) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "format_disk",
|
||||
endpoint = ?endpoint,
|
||||
state = "started",
|
||||
"Heal storage admin operation started"
|
||||
);
|
||||
|
||||
// Use ecstore's heal_format
|
||||
match self.heal_format(false).await {
|
||||
Ok((_, error)) => {
|
||||
if error.is_some() {
|
||||
return Err(Error::other(format!("Format failed: {error:?}")));
|
||||
}
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "format_disk",
|
||||
endpoint = ?endpoint,
|
||||
result = "ok",
|
||||
"Heal storage disk format completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "format_disk",
|
||||
endpoint = ?endpoint,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal storage admin operation failed"
|
||||
);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_bucket_info(&self, bucket: &str) -> Result<Option<BucketInfo>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
@@ -1161,61 +851,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_REPAIR_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "heal_bucket_metadata",
|
||||
bucket,
|
||||
state = "started",
|
||||
"Heal storage repair started"
|
||||
);
|
||||
|
||||
let heal_opts = HealOpts {
|
||||
recursive: true,
|
||||
dry_run: false,
|
||||
remove: false,
|
||||
recreate: false,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
update_parity: false,
|
||||
no_lock: false,
|
||||
pool: None,
|
||||
set: None,
|
||||
};
|
||||
|
||||
match self.heal_bucket(bucket, &heal_opts).await {
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_REPAIR_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "heal_bucket_metadata",
|
||||
bucket,
|
||||
result = "ok",
|
||||
"Heal storage bucket metadata repaired"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_REPAIR_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "heal_bucket_metadata",
|
||||
bucket,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal storage repair failed"
|
||||
);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
@@ -1315,48 +950,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_object_size(&self, bucket: &str, object: &str) -> Result<Option<u64>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_size",
|
||||
bucket,
|
||||
object,
|
||||
"Heal storage request started"
|
||||
);
|
||||
|
||||
match self.get_object_meta(bucket, object).await {
|
||||
Ok(Some(obj_info)) => Ok(Some(obj_info.size as u64)),
|
||||
Ok(None) => Ok(None),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_object_checksum(&self, bucket: &str, object: &str) -> Result<Option<String>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_OBJECT_IO,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "get_object_checksum",
|
||||
bucket,
|
||||
object,
|
||||
"Heal storage request started"
|
||||
);
|
||||
|
||||
match self.get_object_meta(bucket, object).await {
|
||||
Ok(Some(obj_info)) => {
|
||||
// Convert checksum bytes to hex string
|
||||
let checksum = obj_info.checksum.iter().map(|b| format!("{b:02x}")).collect::<String>();
|
||||
Ok(Some(checksum))
|
||||
}
|
||||
Ok(None) => Ok(None),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
async fn heal_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1547,65 +1140,6 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
|
||||
async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result<Vec<HealListItem>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "list_objects_for_heal",
|
||||
bucket,
|
||||
prefix,
|
||||
state = "started",
|
||||
"Heal storage admin operation started"
|
||||
);
|
||||
warn!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "list_objects_for_heal",
|
||||
bucket,
|
||||
prefix,
|
||||
state = "memory_heavy",
|
||||
"Heal storage version listing loads all versions into memory (footprint is per-version, not per-object)"
|
||||
);
|
||||
|
||||
let mut all_objects: Vec<HealListItem> = Vec::new();
|
||||
let mut continuation_token: Option<String> = None;
|
||||
|
||||
loop {
|
||||
let (page_objects, next_token, is_truncated) = self
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false)
|
||||
.await?;
|
||||
|
||||
all_objects.extend(page_objects);
|
||||
|
||||
if !is_truncated {
|
||||
break;
|
||||
}
|
||||
|
||||
continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
|
||||
if continuation_token.is_none() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
event = EVENT_HEAL_STORAGE_ADMIN_OP,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_STORAGE,
|
||||
operation = "list_objects_for_heal",
|
||||
bucket,
|
||||
prefix,
|
||||
object_count = all_objects.len(),
|
||||
result = "ok",
|
||||
"Heal storage object listing completed"
|
||||
);
|
||||
Ok(all_objects)
|
||||
}
|
||||
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1668,13 +1202,22 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
let version_id = obj.version_id.map(|u| u.to_string());
|
||||
let mod_time_unix_nanos = obj.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos());
|
||||
let is_delete_marker = obj.delete_marker;
|
||||
let lifecycle_object_info = include_lifecycle_object_info.then(|| obj.clone());
|
||||
HealListItem {
|
||||
name: obj.name,
|
||||
version_id,
|
||||
mod_time_unix_nanos,
|
||||
lifecycle_object_info,
|
||||
is_delete_marker,
|
||||
if include_lifecycle_object_info {
|
||||
HealListItem {
|
||||
name: obj.name.clone(),
|
||||
version_id,
|
||||
mod_time_unix_nanos,
|
||||
lifecycle_object_info: Some(obj),
|
||||
is_delete_marker,
|
||||
}
|
||||
} else {
|
||||
HealListItem {
|
||||
name: obj.name,
|
||||
version_id,
|
||||
mod_time_unix_nanos,
|
||||
lifecycle_object_info: None,
|
||||
is_delete_marker,
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
+48
-4099
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,450 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// bucket/cluster/prefix heal: the recursive bucket-objects sweep and the erasure-set usage baseline
|
||||
use super::*;
|
||||
|
||||
impl HealTask {
|
||||
pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
stage = "start",
|
||||
recursive = self.options.recursive,
|
||||
"Heal bucket started"
|
||||
);
|
||||
|
||||
// update progress
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("bucket: {bucket}")));
|
||||
progress.update_progress(0, 3, 0, 0);
|
||||
}
|
||||
|
||||
// Step 1: Check if bucket exists
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
stage = "check_existence",
|
||||
"Heal bucket stage entered"
|
||||
);
|
||||
self.check_control_flags().await?;
|
||||
let bucket_exists = self.await_with_control(self.storage.get_bucket_info(bucket)).await?.is_some();
|
||||
if !bucket_exists {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
result = "missing",
|
||||
"Heal bucket failed because the bucket does not exist"
|
||||
);
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Bucket not found: {bucket}"),
|
||||
});
|
||||
}
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(1, 3, 0, 0);
|
||||
}
|
||||
|
||||
// Step 2: Perform bucket heal using ecstore
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
stage = "heal_with_ecstore",
|
||||
dry_run = self.options.dry_run,
|
||||
"Heal bucket stage entered"
|
||||
);
|
||||
let heal_opts = HealOpts {
|
||||
recursive: self.options.recursive,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: if self.options.recursive {
|
||||
false
|
||||
} else {
|
||||
self.options.remove_corrupted
|
||||
},
|
||||
recreate: self.options.recreate_missing,
|
||||
scan_mode: self.options.scan_mode,
|
||||
update_parity: self.options.update_parity,
|
||||
no_lock: self.options.no_lock,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
|
||||
let heal_result = self.await_with_control(self.storage.heal_bucket(bucket, &heal_opts)).await;
|
||||
|
||||
match heal_result {
|
||||
Ok(result) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
drives_healed = result.drives_healed(),
|
||||
drives_total = result.drives_reported(),
|
||||
recursive = self.options.recursive,
|
||||
result = "ok",
|
||||
"Heal bucket completed"
|
||||
);
|
||||
self.record_result_item(result).await;
|
||||
|
||||
if self.options.recursive {
|
||||
self.heal_bucket_objects(bucket, "").await?;
|
||||
}
|
||||
|
||||
if !self.options.recursive {
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal bucket failed"
|
||||
);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal bucket {bucket}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn heal_cluster(&self) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
stage = "cluster_recursive",
|
||||
"Heal cluster started"
|
||||
);
|
||||
|
||||
let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?;
|
||||
let mut failed = 0_u64;
|
||||
let mut retryable = 0_u64;
|
||||
let mut permanent = 0_u64;
|
||||
let mut first_object = None;
|
||||
let mut first_error = None;
|
||||
for bucket_info in bucket_infos {
|
||||
self.check_control_flags().await?;
|
||||
let mut retry_attempt = 0_u32;
|
||||
loop {
|
||||
match self.heal_bucket(&bucket_info.name).await {
|
||||
Ok(()) => break,
|
||||
Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => return Err(Error::TaskTimeout),
|
||||
Err(err) => {
|
||||
if let Some(failure) = self.take_batch_failure().await {
|
||||
failed = failed.saturating_add(failure.failed);
|
||||
retryable = retryable.saturating_add(failure.retryable);
|
||||
permanent = permanent.saturating_add(failure.permanent);
|
||||
first_object.get_or_insert(failure.first_object);
|
||||
first_error.get_or_insert(failure.first_error);
|
||||
break;
|
||||
}
|
||||
if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
|
||||
retry_attempt = retry_attempt.saturating_add(1);
|
||||
self.await_with_control(async {
|
||||
tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
continue;
|
||||
}
|
||||
failed = failed.saturating_add(1);
|
||||
if err.is_recoverable_heal() {
|
||||
retryable = retryable.saturating_add(1);
|
||||
} else {
|
||||
permanent = permanent.saturating_add(1);
|
||||
}
|
||||
first_object.get_or_insert(bucket_info.name.clone());
|
||||
first_error.get_or_insert_with(|| err.to_string());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if failed > 0 {
|
||||
let failure = BatchHealFailure {
|
||||
scope: "cluster".to_string(),
|
||||
failed,
|
||||
retryable,
|
||||
permanent,
|
||||
first_object: first_object.unwrap_or_default(),
|
||||
first_error: first_error.unwrap_or_default(),
|
||||
};
|
||||
return Err(self.record_batch_failure(failure).await);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn heal_prefix(&self, bucket: &str, prefix: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
prefix,
|
||||
stage = "prefix_recursive",
|
||||
"Heal prefix started"
|
||||
);
|
||||
|
||||
self.heal_bucket_objects(bucket, prefix).await
|
||||
}
|
||||
|
||||
#[hotpath::measure]
|
||||
async fn heal_bucket_objects(&self, bucket: &str, prefix: &str) -> Result<()> {
|
||||
let mut continuation_token: Option<String> = None;
|
||||
let mut scanned = 0u64;
|
||||
let mut healed = 0u64;
|
||||
let mut failed = 0u64;
|
||||
let mut retryable_failed = 0u64;
|
||||
let mut permanent_failed = 0u64;
|
||||
let mut bytes = 0u64;
|
||||
let mut first_failed_object = None;
|
||||
let mut first_error = None;
|
||||
let mut failure_samples_logged = 0_u64;
|
||||
|
||||
let heal_opts = HealOpts {
|
||||
recursive: false,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: self.options.remove_corrupted,
|
||||
recreate: self.options.recreate_missing,
|
||||
scan_mode: self.options.scan_mode,
|
||||
update_parity: self.options.update_parity,
|
||||
no_lock: self.options.no_lock,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
|
||||
loop {
|
||||
self.check_control_flags().await?;
|
||||
let (objects, next_token, is_truncated) = self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut pending = objects;
|
||||
let mut retry_attempt = 0_u32;
|
||||
while !pending.is_empty() {
|
||||
if retry_attempt > 0 {
|
||||
self.await_with_control(async {
|
||||
tokio::time::sleep(self.bucket_object_retry_delay(retry_attempt)).await;
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
}
|
||||
let mut retry = Vec::with_capacity(pending.len());
|
||||
for item in pending {
|
||||
self.check_control_flags().await?;
|
||||
let object = item.name.as_str();
|
||||
if retry_attempt == 0 {
|
||||
scanned = scanned.saturating_add(1);
|
||||
}
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{bucket}/{object}")));
|
||||
progress.update_progress(scanned, healed, failed, bytes);
|
||||
}
|
||||
|
||||
let error = match self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.heal_object(bucket, object, item.version_id.as_deref(), &heal_opts),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok((result, None)) => {
|
||||
healed = healed.saturating_add(1);
|
||||
bytes = bytes.saturating_add(u64::try_from(result.object_size).unwrap_or_default());
|
||||
self.record_result_item(result).await;
|
||||
None
|
||||
}
|
||||
Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => {
|
||||
healed = healed.saturating_add(1);
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "object_dir_not_found_skipped",
|
||||
"Heal bucket object-dir candidate skipped after not-found result"
|
||||
);
|
||||
None
|
||||
}
|
||||
Ok((_, Some(err))) | Err(err) => Some(err),
|
||||
};
|
||||
|
||||
if let Some(err) = error {
|
||||
if Self::should_skip_data_usage_cache_heal_error(bucket, object, &err) {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "transient_skip",
|
||||
error = %err,
|
||||
"Heal bucket object repair skipped due to transient metadata error"
|
||||
);
|
||||
} else if err.is_recoverable_heal() && retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
retry_attempt = retry_attempt.saturating_add(1),
|
||||
error = %err,
|
||||
result = "object_retry_scheduled",
|
||||
"Heal bucket object retry scheduled"
|
||||
);
|
||||
retry.push(item);
|
||||
} else {
|
||||
failed = failed.saturating_add(1);
|
||||
if err.is_recoverable_heal() {
|
||||
retryable_failed = retryable_failed.saturating_add(1);
|
||||
} else {
|
||||
permanent_failed = permanent_failed.saturating_add(1);
|
||||
}
|
||||
first_failed_object.get_or_insert_with(|| object.to_string());
|
||||
first_error.get_or_insert_with(|| err.to_string());
|
||||
if take_failure_log_sample(&mut failure_samples_logged) {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
retry_attempt,
|
||||
error = %err,
|
||||
result = "object_failed",
|
||||
"Heal bucket object repair failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(scanned, healed, failed, bytes);
|
||||
}
|
||||
pending = retry;
|
||||
retry_attempt = retry_attempt.saturating_add(1);
|
||||
}
|
||||
|
||||
if !is_truncated {
|
||||
break;
|
||||
}
|
||||
|
||||
continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?;
|
||||
if continuation_token.is_none() {
|
||||
// Truncated but no continuation token: end of listing.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if failed > 0 {
|
||||
let failure = BatchHealFailure {
|
||||
scope: format!("bucket:{bucket}"),
|
||||
failed,
|
||||
retryable: retryable_failed,
|
||||
permanent: permanent_failed,
|
||||
first_object: first_failed_object.unwrap_or_default(),
|
||||
first_error: first_error.unwrap_or_default(),
|
||||
};
|
||||
return Err(self.record_batch_failure(failure).await);
|
||||
}
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
prefix,
|
||||
scanned,
|
||||
healed,
|
||||
failed,
|
||||
bytes_processed = bytes,
|
||||
result = "recursive_ok",
|
||||
"Heal bucket recursive pass completed"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn apply_erasure_set_usage_baseline(&self, buckets: &[String]) -> Result<()> {
|
||||
let baseline = match self
|
||||
.await_with_control(self.storage.erasure_set_usage_baseline(buckets))
|
||||
.await
|
||||
{
|
||||
Ok(Some(baseline)) => baseline,
|
||||
Ok(None) => return Ok(()),
|
||||
Err(err @ Error::TaskCancelled) | Err(err @ Error::TaskTimeout) => return Err(err),
|
||||
Err(_) => return Ok(()),
|
||||
};
|
||||
|
||||
let HealBucketUsageBaseline { objects_count, bytes } = baseline;
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_total_baseline(objects_count, bytes);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,506 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// erasure-set heal: drives the ErasureSetHealer across the set's buckets
|
||||
use super::*;
|
||||
|
||||
impl HealTask {
|
||||
pub(super) async fn heal_erasure_set(&self, buckets: Vec<String>, set_disk_id: String) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
bucket_count = buckets.len(),
|
||||
stage = "start",
|
||||
"Heal erasure set started"
|
||||
);
|
||||
|
||||
// update progress
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("erasure_set: {} ({} buckets)", set_disk_id, buckets.len())));
|
||||
progress.update_progress(0, 4, 0, 0);
|
||||
}
|
||||
|
||||
let is_auto_replacement = matches!(self.source, HealRequestSource::AutoHeal) && !self.heal_endpoints.is_empty();
|
||||
let replacement_resume_disk = if is_auto_replacement {
|
||||
let mut requested_targets = self.heal_endpoints.clone();
|
||||
requested_targets.sort_unstable();
|
||||
requested_targets.dedup();
|
||||
let selection = self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.get_replacement_resume_disk(&set_disk_id, &self.id, &self.heal_endpoints),
|
||||
)
|
||||
.await?;
|
||||
let disk = match selection {
|
||||
crate::heal::storage::ReplacementResumeDisk::Existing(disk) => {
|
||||
if let Some(anchor) = &self.replacement_resume_endpoint
|
||||
&& disk.endpoint().to_string() != *anchor
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement resume anchor changed for automatic heal {set_disk_id}"),
|
||||
});
|
||||
}
|
||||
Some(disk)
|
||||
}
|
||||
crate::heal::storage::ReplacementResumeDisk::Fresh => {
|
||||
if self.replacement_resume_endpoint.is_some() {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement resume anchor is unavailable for automatic heal {set_disk_id}"),
|
||||
});
|
||||
}
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some(disk) = disk.as_ref()
|
||||
&& ResumeManager::has_replacement_intent(disk, &self.id).await
|
||||
{
|
||||
let resume_manager = ResumeManager::load_replacement_intent(disk.clone(), &self.id).await?;
|
||||
let state = resume_manager.get_state().await;
|
||||
if state.completed
|
||||
&& matches!(state.replacement_phase, ReplacementPhase::CleanupPending)
|
||||
&& state.set_disk_id == set_disk_id
|
||||
&& state.replacement_targets == requested_targets
|
||||
&& state.replacement_generation.as_deref() == Some(self.id.as_str())
|
||||
{
|
||||
resume_manager.ensure_replacement_completion_proof().await?;
|
||||
if CheckpointManager::has_checkpoint(disk, &self.id).await {
|
||||
CheckpointManager::load_from_disk(disk.clone(), &self.id)
|
||||
.await?
|
||||
.cleanup()
|
||||
.await?;
|
||||
}
|
||||
resume_manager.cleanup().await?;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
disk
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if is_auto_replacement
|
||||
&& !self
|
||||
.await_with_control(self.storage.replacement_targets_ready(&self.heal_endpoints))
|
||||
.await?
|
||||
{
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement target is no longer ready for automatic heal {set_disk_id}"),
|
||||
});
|
||||
}
|
||||
|
||||
let replacement_resume_disk = if is_auto_replacement {
|
||||
Some(match replacement_resume_disk {
|
||||
Some(disk) => disk,
|
||||
None => {
|
||||
self.await_with_control(self.storage.get_disk_for_resume_excluding(&set_disk_id, &self.heal_endpoints))
|
||||
.await?
|
||||
}
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut buckets = if buckets.is_empty() {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
stage = "list_buckets",
|
||||
"Heal erasure set bucket list resolved"
|
||||
);
|
||||
let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?;
|
||||
bucket_infos.into_iter().map(|info| info.name).collect()
|
||||
} else {
|
||||
buckets
|
||||
};
|
||||
|
||||
// Persist automatic replacement intent on a surviving disk before the
|
||||
// first target format write. A task retry keeps this id; a newly
|
||||
// admitted blank replacement gets a fresh id and cannot reuse cursor
|
||||
// progress from an older disk at the same endpoint.
|
||||
let replacement_resume = if is_auto_replacement {
|
||||
let identities = self
|
||||
.await_with_control(self.storage.replacement_target_identities(&self.heal_endpoints))
|
||||
.await?;
|
||||
let disk = replacement_resume_disk.clone().ok_or_else(|| Error::TaskExecutionFailed {
|
||||
message: format!("Replacement resume disk is missing for automatic heal {set_disk_id}"),
|
||||
})?;
|
||||
let manager = ResumeManager::new_replacement_intent(
|
||||
disk.clone(),
|
||||
self.id.clone(),
|
||||
set_disk_id.clone(),
|
||||
buckets.clone(),
|
||||
self.heal_endpoints.clone(),
|
||||
identities.clone(),
|
||||
)
|
||||
.await?;
|
||||
buckets = manager.get_state().await.replacement_buckets;
|
||||
Some((disk, manager, identities))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
self.apply_erasure_set_usage_baseline(&buckets).await?;
|
||||
|
||||
let healing_marker = format!("{set_disk_id}:{}", self.id);
|
||||
if let Some((disk, resume_manager, _)) = replacement_resume.as_ref() {
|
||||
let state = resume_manager.get_state().await;
|
||||
if state.completed && matches!(state.replacement_phase, ReplacementPhase::Verified) {
|
||||
resume_manager.ensure_replacement_completion_proof().await?;
|
||||
super::super::clear_healing_markers_after_verified(&self.heal_endpoints, &healing_marker).await?;
|
||||
resume_manager.mark_replacement_cleanup_pending().await?;
|
||||
if CheckpointManager::has_checkpoint(disk, &self.id).await {
|
||||
CheckpointManager::load_from_disk(disk.clone(), &self.id)
|
||||
.await?
|
||||
.cleanup()
|
||||
.await?;
|
||||
}
|
||||
resume_manager.cleanup().await?;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
// Step 1: Perform disk format heal using ecstore
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
stage = "heal_format",
|
||||
"Heal erasure set stage entered"
|
||||
);
|
||||
if is_auto_replacement {
|
||||
let Some((_, _, expected_identities)) = replacement_resume.as_ref() else {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement intent is missing for automatic heal {set_disk_id}"),
|
||||
});
|
||||
};
|
||||
self.verify_replacement_identity_fence(expected_identities, &set_disk_id, "format")
|
||||
.await?;
|
||||
}
|
||||
let format_result = if is_auto_replacement {
|
||||
let pool_index = self.options.pool_index.ok_or_else(|| Error::TaskExecutionFailed {
|
||||
message: format!("Missing pool scope for automatic replacement heal {set_disk_id}"),
|
||||
})?;
|
||||
let set_index = self.options.set_index.ok_or_else(|| Error::TaskExecutionFailed {
|
||||
message: format!("Missing set scope for automatic replacement heal {set_disk_id}"),
|
||||
})?;
|
||||
self.await_with_control(self.storage.heal_replacement_format(
|
||||
self.options.dry_run,
|
||||
pool_index,
|
||||
set_index,
|
||||
&self.heal_endpoints,
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
self.await_with_control(self.storage.heal_format(self.options.dry_run)).await
|
||||
};
|
||||
|
||||
match format_result {
|
||||
Ok((result, error)) => {
|
||||
if let Some(e) = error {
|
||||
if Self::is_no_heal_required_error(&e) {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
result = "format_noop",
|
||||
"Heal erasure set format repair skipped because no format heal was required"
|
||||
);
|
||||
} else {
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
result = "format_failed",
|
||||
error = %e,
|
||||
"Heal erasure set failed"
|
||||
);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(4, 4, 0, 0);
|
||||
}
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal disk format for {set_disk_id}: {e}"),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
drives_healed = result.drives_healed(),
|
||||
drives_total = result.drives_reported(),
|
||||
result = "format_ok",
|
||||
"Heal erasure set format repaired"
|
||||
);
|
||||
}
|
||||
if !self.options.dry_run && !target_outcomes_complete(&result, &self.heal_endpoints) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to verify formatted replacement targets for {set_disk_id}"),
|
||||
});
|
||||
}
|
||||
if let Some((_, replacement_resume, expected_identities)) = &replacement_resume {
|
||||
let identities = self
|
||||
.await_with_control(self.storage.replacement_target_identities(&self.heal_endpoints))
|
||||
.await?;
|
||||
if !replacement_target_identities_match(expected_identities, &identities) {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Replacement target changed after format for automatic heal {set_disk_id}"),
|
||||
});
|
||||
}
|
||||
replacement_resume.mark_replacement_rebuilding(identities).await?;
|
||||
}
|
||||
}
|
||||
Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => return Err(Error::TaskTimeout),
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
result = "format_failed",
|
||||
error = %e,
|
||||
"Heal erasure set failed"
|
||||
);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(4, 4, 0, 0);
|
||||
}
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal disk format for {set_disk_id}: {e}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(1, 4, 0, 0);
|
||||
}
|
||||
|
||||
// The rebuilt disks are formatted now: mark them as healing so
|
||||
// DiskInfo.healing reflects the rebuild until it completes.
|
||||
super::super::set_healing_markers(&self.heal_endpoints, &healing_marker).await?;
|
||||
|
||||
// Step 2: Get disk for resume functionality
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
stage = "resolve_resume_disk",
|
||||
"Heal erasure set stage entered"
|
||||
);
|
||||
let replacement_target_identities = replacement_resume.as_ref().map(|(_, _, identities)| identities.clone());
|
||||
let disk = match replacement_resume.as_ref() {
|
||||
Some((disk, _, _)) => disk.clone(),
|
||||
None => {
|
||||
self.await_with_control(self.storage.get_disk_for_resume(&set_disk_id))
|
||||
.await?
|
||||
}
|
||||
};
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(2, 4, 0, 0);
|
||||
}
|
||||
|
||||
// Step 3: Heal bucket structure
|
||||
// Check control flags before each iteration to ensure timely cancellation.
|
||||
let bucket_heal_opts = HealOpts {
|
||||
recursive: false,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: false,
|
||||
recreate: self.options.recreate_missing,
|
||||
scan_mode: self.options.scan_mode,
|
||||
update_parity: self.options.update_parity,
|
||||
no_lock: self.options.no_lock,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
|
||||
for bucket in buckets.iter() {
|
||||
// Check control flags before starting each bucket heal
|
||||
self.check_control_flags().await?;
|
||||
if let Some(expected_identities) = replacement_target_identities.as_ref() {
|
||||
self.verify_replacement_identity_fence(expected_identities, &set_disk_id, "bucket prepass")
|
||||
.await?;
|
||||
}
|
||||
let heal_result = self
|
||||
.await_with_control(self.storage.heal_bucket(bucket, &bucket_heal_opts))
|
||||
.await;
|
||||
match heal_result {
|
||||
Ok(result) => {
|
||||
self.record_result_item(result).await;
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
bucket,
|
||||
result = "bucket_failed",
|
||||
error = %err,
|
||||
"Heal erasure set bucket prepass failed"
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create erasure set healer with resume support
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
stage = "build_resumable_healer",
|
||||
"Heal erasure set stage entered"
|
||||
);
|
||||
let heal_opts = HealOpts {
|
||||
recursive: self.options.recursive,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: self.options.remove_corrupted,
|
||||
recreate: self.options.recreate_missing,
|
||||
scan_mode: self.options.scan_mode,
|
||||
update_parity: self.options.update_parity,
|
||||
no_lock: self.options.no_lock,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
let erasure_healer = ErasureSetHealer::new(
|
||||
self.storage.clone(),
|
||||
self.progress.clone(),
|
||||
self.cancel_token.clone(),
|
||||
disk,
|
||||
heal_opts,
|
||||
self.source,
|
||||
)
|
||||
.with_replacement_targets(self.heal_endpoints.clone(), is_auto_replacement.then(|| self.id.clone()))
|
||||
.with_replacement_identity_fence(replacement_target_identities.clone());
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 4, 0, 0);
|
||||
}
|
||||
|
||||
// Step 4: Execute erasure set heal with resume
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
stage = "execute_resumable_heal",
|
||||
"Heal erasure set stage entered"
|
||||
);
|
||||
let result = self
|
||||
.await_with_control(erasure_healer.heal_erasure_set(&buckets, &set_disk_id))
|
||||
.await;
|
||||
|
||||
// Keep the markers on failure: the resume state also persists, and the
|
||||
// next run of this set heal re-marks and eventually clears them.
|
||||
let result = match result {
|
||||
Ok(()) => {
|
||||
if let Some(expected_identities) = replacement_target_identities.as_ref() {
|
||||
self.verify_replacement_identity_fence(expected_identities, &set_disk_id, "marker completion")
|
||||
.await?;
|
||||
}
|
||||
super::super::clear_healing_markers_after_verified(&self.heal_endpoints, &healing_marker).await?;
|
||||
if let Some((disk, resume_manager, _)) = replacement_resume.as_ref() {
|
||||
resume_manager.mark_replacement_cleanup_pending().await?;
|
||||
if CheckpointManager::has_checkpoint(disk, &self.id).await {
|
||||
CheckpointManager::load_from_disk(disk.clone(), &self.id)
|
||||
.await?
|
||||
.cleanup()
|
||||
.await?;
|
||||
}
|
||||
resume_manager.cleanup().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
let bytes_processed = progress.bytes_processed;
|
||||
progress.update_progress(4, 4, 0, bytes_processed);
|
||||
}
|
||||
|
||||
match result {
|
||||
Ok(_) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
bucket_count = buckets.len(),
|
||||
result = "ok",
|
||||
"Heal erasure set repaired"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_ERASURE_SET_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
set_disk_id,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal erasure set failed"
|
||||
);
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal erasure set {set_disk_id}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// metadata and erasure-decode heal for a single object version
|
||||
use super::*;
|
||||
|
||||
impl HealTask {
|
||||
pub(super) async fn heal_metadata(&self, bucket: &str, object: &str) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_METADATA_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
stage = "start",
|
||||
"Heal metadata started"
|
||||
);
|
||||
|
||||
// update progress
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("metadata: {bucket}/{object}")));
|
||||
progress.update_progress(0, 3, 0, 0);
|
||||
}
|
||||
|
||||
// Step 1: Check if object exists
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_METADATA_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
stage = "check_existence",
|
||||
"Heal metadata stage entered"
|
||||
);
|
||||
self.check_control_flags().await?;
|
||||
let object_exists = match self.await_with_control(self.storage.object_exists(bucket, object)).await {
|
||||
Ok(exists) => exists,
|
||||
Err(err @ Error::TransientSkip { .. }) => {
|
||||
return self.skip_due_to_transient_object_exists(bucket, object, &err).await;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if !object_exists {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_METADATA_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "missing",
|
||||
"Heal metadata failed because object is missing"
|
||||
);
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Object not found: {bucket}/{object}"),
|
||||
});
|
||||
}
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(1, 3, 0, 0);
|
||||
}
|
||||
|
||||
// Step 2: Perform metadata heal using ecstore
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_METADATA_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
stage = "heal_with_ecstore",
|
||||
"Heal metadata stage entered"
|
||||
);
|
||||
let heal_opts = HealOpts {
|
||||
recursive: false,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: false,
|
||||
recreate: false,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
update_parity: false,
|
||||
no_lock: self.options.no_lock,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
|
||||
let heal_result = self
|
||||
.await_with_control(self.storage.heal_object(bucket, object, None, &heal_opts))
|
||||
.await;
|
||||
|
||||
match heal_result {
|
||||
Ok((result, error)) => {
|
||||
if let Some(e) = error {
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_METADATA_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal metadata failed"
|
||||
);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal metadata {bucket}/{object}: {e}"),
|
||||
});
|
||||
}
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_METADATA_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
drives_healed = result.drives_healed(),
|
||||
drives_total = result.drives_reported(),
|
||||
result = "ok",
|
||||
"Heal metadata repaired"
|
||||
);
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
self.record_result_item(result).await;
|
||||
Ok(())
|
||||
}
|
||||
Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_METADATA_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal metadata failed"
|
||||
);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal metadata {bucket}/{object}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn heal_ec_decode(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_EC_DECODE_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
version_id = ?version_id,
|
||||
stage = "start",
|
||||
"Heal EC decode started"
|
||||
);
|
||||
|
||||
// update progress
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("ec_decode: {bucket}/{object}")));
|
||||
progress.update_progress(0, 3, 0, 0);
|
||||
}
|
||||
|
||||
// Step 1: Check if object exists
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_EC_DECODE_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
stage = "check_existence",
|
||||
"Heal EC decode stage entered"
|
||||
);
|
||||
self.check_control_flags().await?;
|
||||
let object_exists = match self.await_with_control(self.storage.object_exists(bucket, object)).await {
|
||||
Ok(exists) => exists,
|
||||
Err(err @ Error::TransientSkip { .. }) => {
|
||||
return self.skip_due_to_transient_object_exists(bucket, object, &err).await;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if !object_exists {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_EC_DECODE_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "missing",
|
||||
"Heal EC decode failed because object is missing"
|
||||
);
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Object not found: {bucket}/{object}"),
|
||||
});
|
||||
}
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(1, 3, 0, 0);
|
||||
}
|
||||
|
||||
// Step 2: Perform EC decode heal using ecstore
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_EC_DECODE_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
stage = "heal_with_ecstore",
|
||||
"Heal EC decode stage entered"
|
||||
);
|
||||
let heal_opts = HealOpts {
|
||||
recursive: false,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: false,
|
||||
recreate: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
update_parity: true,
|
||||
no_lock: self.options.no_lock,
|
||||
pool: None,
|
||||
set: None,
|
||||
};
|
||||
|
||||
let heal_result = self
|
||||
.await_with_control(self.storage.heal_object(bucket, object, version_id, &heal_opts))
|
||||
.await;
|
||||
|
||||
match heal_result {
|
||||
Ok((result, error)) => {
|
||||
if let Some(e) = error {
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_EC_DECODE_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal EC decode failed"
|
||||
);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal EC decode {bucket}/{object}: {e}"),
|
||||
});
|
||||
}
|
||||
|
||||
let object_size = result.object_size as u64;
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_EC_DECODE_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
object_size,
|
||||
drives_healed = result.drives_healed(),
|
||||
drives_total = result.drives_reported(),
|
||||
result = "ok",
|
||||
"Heal EC decode repaired"
|
||||
);
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, object_size);
|
||||
}
|
||||
self.record_result_item(result).await;
|
||||
Ok(())
|
||||
}
|
||||
Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
|
||||
Err(e) => {
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_EC_DECODE_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_TASK,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal EC decode failed"
|
||||
);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal EC decode {bucket}/{object}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,447 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
/// object-level heal: metadata dir canonicalization and missing-object recreation
|
||||
use super::*;
|
||||
|
||||
impl HealTask {
|
||||
// specific heal implementation method
|
||||
#[tracing::instrument(skip(self), fields(bucket = %bucket, object = %object, version_id = ?version_id))]
|
||||
#[hotpath::measure]
|
||||
pub(super) async fn heal_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
version_id = ?version_id,
|
||||
stage = "start",
|
||||
"Heal object started"
|
||||
);
|
||||
|
||||
// update progress
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{bucket}/{object}")));
|
||||
progress.update_progress(0, 4, 0, 0);
|
||||
}
|
||||
|
||||
// Step 1: Check if object exists and get metadata
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
stage = "check_existence",
|
||||
"Heal object stage entered"
|
||||
);
|
||||
self.check_control_flags().await?;
|
||||
let mut object_exists = match self.await_with_control(self.storage.object_exists(bucket, object)).await {
|
||||
Ok(exists) => exists,
|
||||
Err(err @ Error::TransientSkip { .. }) => {
|
||||
return self.skip_due_to_transient_object_exists(bucket, object, &err).await;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
let canonicalized_object = if !object_exists {
|
||||
match self.canonicalize_scanner_missing_object_dir(bucket, object).await {
|
||||
Ok(canonicalized_object) => canonicalized_object,
|
||||
Err(err @ Error::TransientSkip { .. }) => {
|
||||
return self.skip_due_to_transient_object_exists(bucket, object, &err).await;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let object = if let Some(canonicalized_object) = canonicalized_object.as_deref() {
|
||||
object_exists = true;
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_current_object(Some(format!("{bucket}/{canonicalized_object}")));
|
||||
}
|
||||
canonicalized_object
|
||||
} else {
|
||||
object
|
||||
};
|
||||
|
||||
if !object_exists {
|
||||
// Background loops (scanner/MRF/autoheal/read-repair) routinely
|
||||
// race object deletion, so a missing target is per-object noise
|
||||
// for them; only foreground admin/internal requests keep the warn.
|
||||
let background_source = !matches!(self.source, HealRequestSource::Admin | HealRequestSource::Internal);
|
||||
demote_to_debug_when!(background_source, warn, target: "rustfs::heal::task", {
|
||||
event = EVENT_HEAL_OBJECT_MISSING,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
source = self.source.as_str(),
|
||||
recreate_missing = self.options.recreate_missing,
|
||||
"Heal target object is missing"
|
||||
});
|
||||
if self.options.recreate_missing {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
stage = "recreate_missing",
|
||||
"Heal object recreate requested"
|
||||
);
|
||||
return self.recreate_missing_object(bucket, object, version_id).await;
|
||||
} else if self.source == HealRequestSource::Scanner {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
stage = "scanner_missing_probe",
|
||||
"Heal scanner missing object will be checked by storage layer"
|
||||
);
|
||||
} else {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Object not found: {bucket}/{object}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(1, 3, 0, 0);
|
||||
}
|
||||
|
||||
// Step 2: directly call ecstore to perform heal
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
stage = "heal_with_ecstore",
|
||||
dry_run = self.options.dry_run,
|
||||
remove_corrupted = self.options.remove_corrupted,
|
||||
update_parity = self.options.update_parity,
|
||||
"Heal object stage entered"
|
||||
);
|
||||
let heal_opts = HealOpts {
|
||||
recursive: self.options.recursive,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: self.options.remove_corrupted,
|
||||
recreate: self.options.recreate_missing,
|
||||
scan_mode: self.options.scan_mode,
|
||||
update_parity: self.options.update_parity,
|
||||
no_lock: self.options.no_lock,
|
||||
pool: self.options.pool_index,
|
||||
set: self.options.set_index,
|
||||
};
|
||||
|
||||
let heal_result = self
|
||||
.await_with_control(self.storage.heal_object(bucket, object, version_id, &heal_opts))
|
||||
.await;
|
||||
|
||||
match heal_result {
|
||||
Ok((result, error)) => {
|
||||
if let Some(e) = error {
|
||||
if self.skip_data_usage_cache_heal_error(bucket, object, &e).await {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if Self::is_object_not_found_heal_error(&e) {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "treated_as_deleted",
|
||||
"Heal missing object treated as deleted"
|
||||
);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal object operation failed"
|
||||
);
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
|
||||
if Self::should_return_typed_heal_error(&e) {
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal object {bucket}/{object}: {e}"),
|
||||
});
|
||||
}
|
||||
|
||||
// Step 3: Verify heal result
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
stage = "verify_result",
|
||||
"Heal object stage entered"
|
||||
);
|
||||
let object_size = result.object_size as u64;
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
object_size = object_size,
|
||||
drives_healed = result.drives_healed(),
|
||||
drives_total = result.drives_reported(),
|
||||
result = "ok",
|
||||
"Heal object repaired"
|
||||
);
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, object_size);
|
||||
}
|
||||
self.record_result_item(result).await;
|
||||
Ok(())
|
||||
}
|
||||
Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
|
||||
Err(e) => {
|
||||
if self.skip_data_usage_cache_heal_error(bucket, object, &e).await {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if Self::is_object_not_found_heal_error(&e) {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "treated_as_deleted",
|
||||
"Heal missing object treated as deleted"
|
||||
);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "failed",
|
||||
error = %e,
|
||||
"Heal object operation failed"
|
||||
);
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(3, 3, 0, 0);
|
||||
}
|
||||
|
||||
if Self::should_return_typed_heal_error(&e) {
|
||||
Err(e)
|
||||
} else {
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal object {bucket}/{object}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn canonicalize_scanner_missing_object_dir(&self, bucket: &str, object: &str) -> Result<Option<String>> {
|
||||
if self.source != HealRequestSource::Scanner {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let Some(candidate) = object.strip_suffix(SLASH_SEPARATOR) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if candidate.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
match self.await_with_control(self.storage.object_exists(bucket, candidate)).await {
|
||||
Ok(true) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object = %candidate,
|
||||
canonicalized_from = %object,
|
||||
stage = "canonicalize_scanner_object_dir",
|
||||
result = "canonicalized",
|
||||
"Heal scanner object-dir candidate canonicalized"
|
||||
);
|
||||
Ok(Some(candidate.to_string()))
|
||||
}
|
||||
Ok(false) => Ok(None),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
/// Recreate missing object (for EC decode scenarios)
|
||||
async fn recreate_missing_object(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> {
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_STAGE,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
version_id = ?version_id,
|
||||
stage = "recreate_missing",
|
||||
"Heal object recreate started"
|
||||
);
|
||||
|
||||
// Use ecstore's heal_object with recreate option
|
||||
let heal_opts = HealOpts {
|
||||
recursive: false,
|
||||
dry_run: self.options.dry_run,
|
||||
remove: false,
|
||||
recreate: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
update_parity: true,
|
||||
no_lock: self.options.no_lock,
|
||||
pool: None,
|
||||
set: None,
|
||||
};
|
||||
|
||||
match self
|
||||
.await_with_control(self.storage.heal_object(bucket, object, version_id, &heal_opts))
|
||||
.await
|
||||
{
|
||||
Ok((result, error)) => {
|
||||
if let Some(e) = error {
|
||||
if self.skip_scanner_synthetic_object_dir_missing(bucket, object, &e).await {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "recreate_failed",
|
||||
error = %e,
|
||||
"Heal object recovery failed"
|
||||
);
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to recreate missing object {bucket}/{object}: {e}"),
|
||||
});
|
||||
}
|
||||
|
||||
let object_size = result.object_size as u64;
|
||||
debug!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
object_size,
|
||||
result = "recreated",
|
||||
"Heal object recreated"
|
||||
);
|
||||
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_progress(4, 4, 0, object_size);
|
||||
}
|
||||
self.record_result_item(result).await;
|
||||
Ok(())
|
||||
}
|
||||
Err(Error::TaskCancelled) => Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => Err(Error::TaskTimeout),
|
||||
Err(e) => {
|
||||
if self.skip_scanner_synthetic_object_dir_missing(bucket, object, &e).await {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
error!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_OBJECT_RESULT,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT,
|
||||
task_id = %self.id,
|
||||
bucket,
|
||||
object,
|
||||
result = "recreate_failed",
|
||||
error = %e,
|
||||
"Heal object recovery failed"
|
||||
);
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to recreate missing object {bucket}/{object}: {e}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+2
-42
@@ -352,8 +352,8 @@ pub(crate) fn set_heal_queue_length(count: usize) {
|
||||
mod tests {
|
||||
use super::{
|
||||
Error, HEAL_RUNTIME_INIT_TEST_HOOK, HealRuntimeInitTestHook, get_heal_channel_processor, get_heal_manager,
|
||||
heal::DiskStore, heal::Endpoint, heal::manager::HealConfig, heal::storage::DiskStatus, heal::storage::HealListItem,
|
||||
heal::storage::HealObjectInfo, heal::storage::HealStorageAPI, init_heal_manager, run_owned_initialization,
|
||||
heal::DiskStore, heal::manager::HealConfig, heal::storage::HealListItem, heal::storage::HealObjectInfo,
|
||||
heal::storage::HealStorageAPI, init_heal_manager, run_owned_initialization,
|
||||
};
|
||||
use crate::heal::storage_api::status::BucketInfo;
|
||||
use rustfs_common::heal_channel::HealOpts;
|
||||
@@ -370,42 +370,14 @@ mod tests {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn get_object_data(&self, _bucket: &str, _object: &str) -> Result<Option<Vec<u8>>, Error> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_object(&self, _bucket: &str, _object: &str) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> Result<bool, Error> {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> Result<Vec<u8>, Error> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
async fn get_disk_status(&self, _endpoint: &Endpoint) -> Result<DiskStatus, Error> {
|
||||
Ok(DiskStatus::Ok)
|
||||
}
|
||||
|
||||
async fn format_disk(&self, _endpoint: &Endpoint) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_bucket_info(&self, _bucket: &str) -> Result<Option<BucketInfo>, Error> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>, Error> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
@@ -414,14 +386,6 @@ mod tests {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
async fn get_object_size(&self, _bucket: &str, _object: &str) -> Result<Option<u64>, Error> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> Result<Option<String>, Error> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn heal_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -440,10 +404,6 @@ mod tests {
|
||||
Ok((HealResultItem::default(), None))
|
||||
}
|
||||
|
||||
async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> Result<Vec<HealListItem>, Error> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
|
||||
@@ -13,93 +13,13 @@
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_heal::heal::{
|
||||
event::{HealEvent, Severity},
|
||||
task::{HealPriority, HealType},
|
||||
utils,
|
||||
};
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use storage_api::bug_fixes::{BucketInfo, DiskStore, Endpoint};
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_to_heal_request_no_panic() {
|
||||
// Test that invalid pool/set indices don't cause panic
|
||||
// Create endpoint using try_from or similar method
|
||||
let endpoint_result = Endpoint::try_from("http://localhost:9000");
|
||||
if let Ok(mut endpoint) = endpoint_result {
|
||||
endpoint.pool_idx = -1;
|
||||
endpoint.set_idx = -1;
|
||||
endpoint.disk_idx = 0;
|
||||
|
||||
let event = HealEvent::DiskStatusChange {
|
||||
endpoint,
|
||||
old_status: "ok".to_string(),
|
||||
new_status: "offline".to_string(),
|
||||
};
|
||||
|
||||
// Should return error instead of panicking
|
||||
let result = event.to_heal_request();
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().to_string().contains("Invalid heal type"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_to_heal_request_valid_indices() {
|
||||
// Test that valid indices work correctly
|
||||
let endpoint_result = Endpoint::try_from("http://localhost:9000");
|
||||
if let Ok(mut endpoint) = endpoint_result {
|
||||
endpoint.pool_idx = 0;
|
||||
endpoint.set_idx = 1;
|
||||
endpoint.disk_idx = 0;
|
||||
|
||||
let event = HealEvent::DiskStatusChange {
|
||||
endpoint,
|
||||
old_status: "ok".to_string(),
|
||||
new_status: "offline".to_string(),
|
||||
};
|
||||
|
||||
let result = event.to_heal_request();
|
||||
assert!(result.is_ok());
|
||||
let request = result.unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::ErasureSet { .. }));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_object_corruption() {
|
||||
let event = HealEvent::ObjectCorruption {
|
||||
bucket: "test-bucket".to_string(),
|
||||
object: "test-object".to_string(),
|
||||
version_id: None,
|
||||
corruption_type: rustfs_heal::heal::event::CorruptionType::DataCorruption,
|
||||
severity: Severity::High,
|
||||
};
|
||||
|
||||
let result = event.to_heal_request();
|
||||
assert!(result.is_ok());
|
||||
let request = result.unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::Object { .. }));
|
||||
assert_eq!(request.priority, HealPriority::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_ec_decode_failure() {
|
||||
let event = HealEvent::ECDecodeFailure {
|
||||
bucket: "test-bucket".to_string(),
|
||||
object: "test-object".to_string(),
|
||||
version_id: None,
|
||||
missing_shards: vec![0, 1],
|
||||
available_shards: vec![2, 3],
|
||||
};
|
||||
|
||||
let result = event.to_heal_request();
|
||||
assert!(result.is_ok());
|
||||
let request = result.unwrap();
|
||||
assert!(matches!(request.heal_type, HealType::ECDecode { .. }));
|
||||
assert_eq!(request.priority, HealPriority::Urgent);
|
||||
}
|
||||
use storage_api::bug_fixes::{BucketInfo, DiskStore};
|
||||
|
||||
#[test]
|
||||
fn test_format_set_disk_id_from_i32_negative() {
|
||||
@@ -117,13 +37,16 @@ fn test_format_set_disk_id_from_i32_valid() {
|
||||
assert_eq!(result.unwrap(), "pool_0_set_1");
|
||||
}
|
||||
|
||||
/// A wall-clock lower bound for "the timestamp was actually read from the
|
||||
/// clock": 2020-01-01. `unwrap_or_default()` on a pre-epoch clock yields 0, and
|
||||
/// the old versions of these tests bound the fields to `_` and so could not tell
|
||||
/// that apart from a real reading (rustfs/backlog#1836).
|
||||
const SANE_EPOCH_SECS: u64 = 1_577_836_800;
|
||||
|
||||
#[test]
|
||||
fn test_resume_state_timestamp_handling() {
|
||||
use rustfs_heal::heal::resume::ResumeState;
|
||||
|
||||
// Test that ResumeState creation doesn't panic even if system time is before epoch
|
||||
// This is a theoretical test - in practice, system time should never be before epoch
|
||||
// But we want to ensure unwrap_or_default handles edge cases
|
||||
let state = ResumeState::new(
|
||||
"test-task".to_string(),
|
||||
"test-type".to_string(),
|
||||
@@ -131,22 +54,30 @@ fn test_resume_state_timestamp_handling() {
|
||||
vec!["bucket1".to_string()],
|
||||
);
|
||||
|
||||
// Verify fields are initialized (u64 is always >= 0)
|
||||
// The important thing is that unwrap_or_default prevents panic
|
||||
let _ = state.start_time;
|
||||
let _ = state.last_update;
|
||||
assert!(
|
||||
state.start_time > SANE_EPOCH_SECS,
|
||||
"start_time fell back to the default instead of reading the clock: {}",
|
||||
state.start_time
|
||||
);
|
||||
assert!(
|
||||
state.last_update >= state.start_time,
|
||||
"last_update {} must not predate start_time {}",
|
||||
state.last_update,
|
||||
state.start_time
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resume_checkpoint_timestamp_handling() {
|
||||
use rustfs_heal::heal::resume::ResumeCheckpoint;
|
||||
|
||||
// Test that ResumeCheckpoint creation doesn't panic
|
||||
let checkpoint = ResumeCheckpoint::new("test-task".to_string());
|
||||
|
||||
// Verify field is initialized (u64 is always >= 0)
|
||||
// The important thing is that unwrap_or_default prevents panic
|
||||
let _ = checkpoint.checkpoint_time;
|
||||
assert!(
|
||||
checkpoint.checkpoint_time > SANE_EPOCH_SECS,
|
||||
"checkpoint_time fell back to the default instead of reading the clock: {}",
|
||||
checkpoint.checkpoint_time
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -173,45 +104,18 @@ fn test_heal_task_status_atomic_update() {
|
||||
async fn get_object_meta(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Option<HealObjectInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn get_object_data(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Option<Vec<u8>>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> rustfs_heal::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn delete_object(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Vec<u8>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
async fn get_disk_status(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<rustfs_heal::heal::storage::DiskStatus> {
|
||||
Ok(rustfs_heal::heal::storage::DiskStatus::Ok)
|
||||
}
|
||||
async fn format_disk(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<BucketInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_heal::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn list_buckets(&self) -> rustfs_heal::Result<Vec<BucketInfo>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
async fn object_exists(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
async fn get_object_size(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Option<u64>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Option<String>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn heal_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -234,9 +138,6 @@ fn test_heal_task_status_atomic_update() {
|
||||
) -> rustfs_heal::Result<(rustfs_madmin::heal_commands::HealResultItem, Option<rustfs_heal::Error>)> {
|
||||
Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None))
|
||||
}
|
||||
async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> rustfs_heal::Result<Vec<HealListItem>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -278,7 +179,7 @@ fn test_heal_task_status_atomic_update() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
|
||||
use rustfs_heal::heal::storage::{DiskStatus, HealListItem, HealObjectInfo, HealStorageAPI};
|
||||
use rustfs_heal::heal::storage::{HealListItem, HealObjectInfo, HealStorageAPI};
|
||||
use rustfs_heal::heal::task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
@@ -296,42 +197,14 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn get_object_data(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Option<Vec<u8>>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> rustfs_heal::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_object(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Vec<u8>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
async fn get_disk_status(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<DiskStatus> {
|
||||
Ok(DiskStatus::Ok)
|
||||
}
|
||||
|
||||
async fn format_disk(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<BucketInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_heal::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> rustfs_heal::Result<Vec<BucketInfo>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
@@ -343,14 +216,6 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
|
||||
))
|
||||
}
|
||||
|
||||
async fn get_object_size(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Option<u64>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<Option<String>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn heal_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
@@ -377,10 +242,6 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
|
||||
Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None))
|
||||
}
|
||||
|
||||
async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> rustfs_heal::Result<Vec<HealListItem>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
|
||||
@@ -127,6 +127,11 @@ async fn decode_failure_intent_maps_to_urgent_mrf_heal_request() {
|
||||
"MRF intent must reach the manager queue as an Urgent request (snapshot: {:?})",
|
||||
manager.operations_snapshot().await
|
||||
);
|
||||
|
||||
assert!(
|
||||
mrf_channel::take_mrf_repaired_events_for("mrf-bucket").is_empty(),
|
||||
"accepted intent must wait for successful heal completion before repaired notice fan-out"
|
||||
);
|
||||
}
|
||||
|
||||
/// A journal left behind by a previous process must be replayed into the
|
||||
|
||||
@@ -58,7 +58,7 @@ sysinfo = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
metrics-util = { version = "0.20", features = ["debugging"] }
|
||||
metrics-util = { workspace = true, features = ["debugging"] }
|
||||
tokio = { workspace = true, features = ["test-util", "macros", "fs", "rt-multi-thread"] }
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -47,6 +47,13 @@ pub const INTERNODE_MSGPACK_DIRECTION_REQUEST: &str = "request";
|
||||
pub const INTERNODE_MSGPACK_DIRECTION_RESPONSE: &str = "response";
|
||||
pub const INTERNODE_MSGPACK_CODEC_MSGPACK: &str = "msgpack";
|
||||
pub const INTERNODE_MSGPACK_CODEC_JSON: &str = "json";
|
||||
pub const INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE: &str = "read_version_request_encode";
|
||||
pub const INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE: &str = "read_version_request_decode";
|
||||
pub const INTERNODE_STAGE_READ_VERSION_DISK_READ: &str = "read_version_disk_read";
|
||||
pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "read_version_response_json_encode";
|
||||
pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "read_version_response_msgpack_encode";
|
||||
pub const INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP: &str = "read_version_rpc_roundtrip";
|
||||
pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE: &str = "read_version_response_decode";
|
||||
|
||||
const OPERATION_LABEL: &str = "operation";
|
||||
const BACKEND_LABEL: &str = "backend";
|
||||
@@ -67,6 +74,7 @@ const INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network
|
||||
const INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_incoming_total";
|
||||
const INTERNODE_OPERATION_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_errors_total";
|
||||
const INTERNODE_OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_operation_duration_ms";
|
||||
const INTERNODE_OPERATION_STAGE_DURATION_MS: &str = "rustfs_system_network_internode_operation_stage_duration_ms";
|
||||
const INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_classified_errors_total";
|
||||
const INTERNODE_OPERATION_RETRIES_TOTAL: &str = "rustfs_system_network_internode_operation_retries_total";
|
||||
const INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL: &str = "rustfs_system_network_internode_operation_retry_successes_total";
|
||||
@@ -105,6 +113,7 @@ const SERVER_OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[SERVER_LABEL, OP
|
||||
const SERVER_OPERATION_BACKEND_FAILURE_REASON_LABELS: &[&str] =
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, FAILURE_REASON_LABEL];
|
||||
const SERVER_OPERATION_BACKEND_RPC_PATH_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL];
|
||||
const SERVER_OPERATION_BACKEND_STAGE_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, STAGE_LABEL];
|
||||
const SERVER_LABELS: &[&str] = &[SERVER_LABEL];
|
||||
const SERVER_REASON_LABELS: &[&str] = &[SERVER_LABEL, REASON_LABEL];
|
||||
const SERVER_QUORUM_FAILURE_LABELS: &[&str] = &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL];
|
||||
@@ -134,6 +143,10 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
|
||||
name: INTERNODE_OPERATION_DURATION_MS,
|
||||
labels: SERVER_OPERATION_BACKEND_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_OPERATION_STAGE_DURATION_MS,
|
||||
labels: SERVER_OPERATION_BACKEND_STAGE_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL,
|
||||
labels: SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS,
|
||||
@@ -198,6 +211,146 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
|
||||
|
||||
static STABLE_SERVER_LABEL: OnceLock<String> = OnceLock::new();
|
||||
|
||||
#[cfg(not(test))]
|
||||
struct InternodeServerMetricHandles {
|
||||
sent_bytes: metrics::Counter,
|
||||
recv_bytes: metrics::Counter,
|
||||
outgoing_requests: metrics::Counter,
|
||||
incoming_requests: metrics::Counter,
|
||||
errors: metrics::Counter,
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
impl InternodeServerMetricHandles {
|
||||
fn new(server: &'static str) -> Self {
|
||||
Self {
|
||||
sent_bytes: counter!("rustfs_system_network_internode_sent_bytes_total", SERVER_LABEL => server),
|
||||
recv_bytes: counter!("rustfs_system_network_internode_recv_bytes_total", SERVER_LABEL => server),
|
||||
outgoing_requests: counter!("rustfs_system_network_internode_requests_outgoing_total", SERVER_LABEL => server),
|
||||
incoming_requests: counter!("rustfs_system_network_internode_requests_incoming_total", SERVER_LABEL => server),
|
||||
errors: counter!("rustfs_system_network_internode_errors_total", SERVER_LABEL => server),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
static INTERNODE_SERVER_METRIC_HANDLES: LazyLock<InternodeServerMetricHandles> =
|
||||
LazyLock::new(|| InternodeServerMetricHandles::new(current_server_label()));
|
||||
|
||||
#[cfg(not(test))]
|
||||
struct GrpcReadVersionMetricHandles {
|
||||
sent_bytes: metrics::Counter,
|
||||
recv_bytes: metrics::Counter,
|
||||
outgoing_requests: metrics::Counter,
|
||||
incoming_requests: metrics::Counter,
|
||||
errors: metrics::Counter,
|
||||
duration: metrics::Histogram,
|
||||
request_encode: metrics::Histogram,
|
||||
request_decode: metrics::Histogram,
|
||||
disk_read: metrics::Histogram,
|
||||
response_json_encode: metrics::Histogram,
|
||||
response_msgpack_encode: metrics::Histogram,
|
||||
rpc_roundtrip: metrics::Histogram,
|
||||
response_decode: metrics::Histogram,
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
impl GrpcReadVersionMetricHandles {
|
||||
fn new(server: &'static str) -> Self {
|
||||
Self {
|
||||
sent_bytes: counter!(
|
||||
INTERNODE_OPERATION_SENT_BYTES_TOTAL,
|
||||
SERVER_LABEL => server,
|
||||
OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
|
||||
),
|
||||
recv_bytes: counter!(
|
||||
INTERNODE_OPERATION_RECV_BYTES_TOTAL,
|
||||
SERVER_LABEL => server,
|
||||
OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
|
||||
),
|
||||
outgoing_requests: counter!(
|
||||
INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL,
|
||||
SERVER_LABEL => server,
|
||||
OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
|
||||
),
|
||||
incoming_requests: counter!(
|
||||
INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL,
|
||||
SERVER_LABEL => server,
|
||||
OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
|
||||
),
|
||||
errors: counter!(
|
||||
INTERNODE_OPERATION_ERRORS_TOTAL,
|
||||
SERVER_LABEL => server,
|
||||
OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
|
||||
),
|
||||
duration: metrics::histogram!(
|
||||
INTERNODE_OPERATION_DURATION_MS,
|
||||
SERVER_LABEL => server,
|
||||
OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC
|
||||
),
|
||||
request_encode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE),
|
||||
request_decode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE),
|
||||
disk_read: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_DISK_READ),
|
||||
response_json_encode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE),
|
||||
response_msgpack_encode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE),
|
||||
rpc_roundtrip: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP),
|
||||
response_decode: Self::stage_duration(server, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE),
|
||||
}
|
||||
}
|
||||
|
||||
fn stage_duration(server: &'static str, stage: &'static str) -> metrics::Histogram {
|
||||
metrics::histogram!(
|
||||
INTERNODE_OPERATION_STAGE_DURATION_MS,
|
||||
SERVER_LABEL => server,
|
||||
OPERATION_LABEL => INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
BACKEND_LABEL => INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
STAGE_LABEL => stage
|
||||
)
|
||||
}
|
||||
|
||||
fn stage_duration_for(&self, stage: &'static str) -> Option<&metrics::Histogram> {
|
||||
match stage {
|
||||
INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE => Some(&self.request_encode),
|
||||
INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE => Some(&self.request_decode),
|
||||
INTERNODE_STAGE_READ_VERSION_DISK_READ => Some(&self.disk_read),
|
||||
INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE => Some(&self.response_json_encode),
|
||||
INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE => Some(&self.response_msgpack_encode),
|
||||
INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP => Some(&self.rpc_roundtrip),
|
||||
INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE => Some(&self.response_decode),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
static GRPC_READ_VERSION_METRIC_HANDLES: LazyLock<GrpcReadVersionMetricHandles> =
|
||||
LazyLock::new(|| GrpcReadVersionMetricHandles::new(current_server_label()));
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn server_metric_handles_if_ready() -> Option<&'static InternodeServerMetricHandles> {
|
||||
STABLE_SERVER_LABEL.get()?;
|
||||
Some(&INTERNODE_SERVER_METRIC_HANDLES)
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn grpc_read_version_metric_handles_if_ready(
|
||||
operation: &'static str,
|
||||
backend: &'static str,
|
||||
) -> Option<&'static GrpcReadVersionMetricHandles> {
|
||||
STABLE_SERVER_LABEL.get()?;
|
||||
if operation == INTERNODE_OPERATION_GRPC_READ_VERSION && backend == INTERNODE_TRANSPORT_BACKEND_GRPC {
|
||||
Some(&GRPC_READ_VERSION_METRIC_HANDLES)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Injects the stable server label (node name or address) stamped on
|
||||
/// internode metrics. The runtime calls this when the local node name is
|
||||
/// published (see ecstore's `set_local_node_name`); the first write wins.
|
||||
@@ -271,6 +424,11 @@ impl InternodeMetrics {
|
||||
return;
|
||||
}
|
||||
self.sent_bytes_total.fetch_add(bytes, Ordering::Relaxed);
|
||||
#[cfg(not(test))]
|
||||
if let Some(handles) = server_metric_handles_if_ready() {
|
||||
handles.sent_bytes.increment(bytes);
|
||||
return;
|
||||
}
|
||||
counter!("rustfs_system_network_internode_sent_bytes_total", SERVER_LABEL => current_server_label()).increment(bytes);
|
||||
}
|
||||
|
||||
@@ -285,6 +443,11 @@ impl InternodeMetrics {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
|
||||
handles.sent_bytes.increment(bytes);
|
||||
return;
|
||||
}
|
||||
counter!(
|
||||
INTERNODE_OPERATION_SENT_BYTES_TOTAL,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
@@ -300,6 +463,11 @@ impl InternodeMetrics {
|
||||
return;
|
||||
}
|
||||
self.recv_bytes_total.fetch_add(bytes, Ordering::Relaxed);
|
||||
#[cfg(not(test))]
|
||||
if let Some(handles) = server_metric_handles_if_ready() {
|
||||
handles.recv_bytes.increment(bytes);
|
||||
return;
|
||||
}
|
||||
counter!("rustfs_system_network_internode_recv_bytes_total", SERVER_LABEL => current_server_label()).increment(bytes);
|
||||
}
|
||||
|
||||
@@ -314,6 +482,11 @@ impl InternodeMetrics {
|
||||
if bytes == 0 {
|
||||
return;
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
|
||||
handles.recv_bytes.increment(bytes);
|
||||
return;
|
||||
}
|
||||
counter!(
|
||||
INTERNODE_OPERATION_RECV_BYTES_TOTAL,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
@@ -325,6 +498,11 @@ impl InternodeMetrics {
|
||||
|
||||
pub fn record_outgoing_request(&self) {
|
||||
self.outgoing_requests_total.fetch_add(1, Ordering::Relaxed);
|
||||
#[cfg(not(test))]
|
||||
if let Some(handles) = server_metric_handles_if_ready() {
|
||||
handles.outgoing_requests.increment(1);
|
||||
return;
|
||||
}
|
||||
counter!("rustfs_system_network_internode_requests_outgoing_total", SERVER_LABEL => current_server_label()).increment(1);
|
||||
}
|
||||
|
||||
@@ -334,6 +512,11 @@ impl InternodeMetrics {
|
||||
|
||||
pub fn record_outgoing_request_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
|
||||
self.record_outgoing_request();
|
||||
#[cfg(not(test))]
|
||||
if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
|
||||
handles.outgoing_requests.increment(1);
|
||||
return;
|
||||
}
|
||||
counter!(
|
||||
INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
@@ -345,6 +528,11 @@ impl InternodeMetrics {
|
||||
|
||||
pub fn record_incoming_request(&self) {
|
||||
self.incoming_requests_total.fetch_add(1, Ordering::Relaxed);
|
||||
#[cfg(not(test))]
|
||||
if let Some(handles) = server_metric_handles_if_ready() {
|
||||
handles.incoming_requests.increment(1);
|
||||
return;
|
||||
}
|
||||
counter!("rustfs_system_network_internode_requests_incoming_total", SERVER_LABEL => current_server_label()).increment(1);
|
||||
}
|
||||
|
||||
@@ -354,6 +542,11 @@ impl InternodeMetrics {
|
||||
|
||||
pub fn record_incoming_request_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
|
||||
self.record_incoming_request();
|
||||
#[cfg(not(test))]
|
||||
if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
|
||||
handles.incoming_requests.increment(1);
|
||||
return;
|
||||
}
|
||||
counter!(
|
||||
INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
@@ -365,6 +558,11 @@ impl InternodeMetrics {
|
||||
|
||||
pub fn record_error(&self) {
|
||||
self.errors_total.fetch_add(1, Ordering::Relaxed);
|
||||
#[cfg(not(test))]
|
||||
if let Some(handles) = server_metric_handles_if_ready() {
|
||||
handles.errors.increment(1);
|
||||
return;
|
||||
}
|
||||
counter!("rustfs_system_network_internode_errors_total", SERVER_LABEL => current_server_label()).increment(1);
|
||||
}
|
||||
|
||||
@@ -374,6 +572,11 @@ impl InternodeMetrics {
|
||||
|
||||
pub fn record_error_for_operation_and_backend(&self, operation: &'static str, backend: &'static str) {
|
||||
self.record_error();
|
||||
#[cfg(not(test))]
|
||||
if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
|
||||
handles.errors.increment(1);
|
||||
return;
|
||||
}
|
||||
counter!(
|
||||
INTERNODE_OPERATION_ERRORS_TOTAL,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
@@ -385,6 +588,11 @@ impl InternodeMetrics {
|
||||
|
||||
pub fn record_duration_for_operation_and_backend(&self, operation: &'static str, backend: &'static str, duration: Duration) {
|
||||
let duration_ms = duration.as_secs_f64() * 1000.0;
|
||||
#[cfg(not(test))]
|
||||
if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend) {
|
||||
handles.duration.record(duration_ms);
|
||||
return;
|
||||
}
|
||||
metrics::histogram!(
|
||||
INTERNODE_OPERATION_DURATION_MS,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
@@ -394,6 +602,31 @@ impl InternodeMetrics {
|
||||
.record(duration_ms);
|
||||
}
|
||||
|
||||
pub fn record_stage_duration_for_operation_and_backend(
|
||||
&self,
|
||||
operation: &'static str,
|
||||
backend: &'static str,
|
||||
stage: &'static str,
|
||||
duration: Duration,
|
||||
) {
|
||||
let duration_ms = duration.as_secs_f64() * 1000.0;
|
||||
#[cfg(not(test))]
|
||||
if let Some(handles) = grpc_read_version_metric_handles_if_ready(operation, backend)
|
||||
&& let Some(histogram) = handles.stage_duration_for(stage)
|
||||
{
|
||||
histogram.record(duration_ms);
|
||||
return;
|
||||
}
|
||||
metrics::histogram!(
|
||||
INTERNODE_OPERATION_STAGE_DURATION_MS,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
OPERATION_LABEL => operation,
|
||||
BACKEND_LABEL => backend,
|
||||
STAGE_LABEL => stage
|
||||
)
|
||||
.record(duration_ms);
|
||||
}
|
||||
|
||||
pub fn record_classified_error_for_operation_and_backend(
|
||||
&self,
|
||||
operation: &'static str,
|
||||
@@ -988,42 +1221,90 @@ mod tests {
|
||||
assert_eq!(snapshot.replay_cache_evictions_total, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operation_stage_duration_records_low_cardinality_stage_labels() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let metrics = InternodeMetrics::default();
|
||||
|
||||
with_local_recorder(&recorder, || {
|
||||
metrics.record_stage_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
|
||||
Duration::from_micros(125),
|
||||
);
|
||||
});
|
||||
|
||||
let entries: Vec<_> = snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.filter(|(composite, _, _, _)| composite.key().name() == INTERNODE_OPERATION_STAGE_DURATION_MS)
|
||||
.collect();
|
||||
assert_eq!(entries.len(), 1);
|
||||
let labels: HashMap<_, _> = entries[0]
|
||||
.0
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key().to_string(), label.value().to_string()))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
labels.get(OPERATION_LABEL).map(String::as_str),
|
||||
Some(INTERNODE_OPERATION_GRPC_READ_VERSION)
|
||||
);
|
||||
assert_eq!(labels.get(BACKEND_LABEL).map(String::as_str), Some(INTERNODE_TRANSPORT_BACKEND_GRPC));
|
||||
assert_eq!(
|
||||
labels.get(STAGE_LABEL).map(String::as_str),
|
||||
Some(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP)
|
||||
);
|
||||
assert!(labels.get(SERVER_LABEL).is_some_and(|value| !value.is_empty()));
|
||||
match &entries[0].3 {
|
||||
DebugValue::Histogram(samples) => assert_eq!(samples.iter().map(|sample| sample.0).collect::<Vec<_>>(), vec![0.125]),
|
||||
other => panic!("{INTERNODE_OPERATION_STAGE_DURATION_MS} must be a histogram, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operation_metric_descriptors_include_backend_and_operation_labels() {
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 21);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 22);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[..6] {
|
||||
assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
}
|
||||
for metric in &INTERNODE_OPERATION_METRICS[6..9] {
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[6].labels,
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, STAGE_LABEL]
|
||||
);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[7..10] {
|
||||
assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]);
|
||||
}
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[9].labels,
|
||||
INTERNODE_OPERATION_METRICS[10].labels,
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL]
|
||||
);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[10..12] {
|
||||
for metric in &INTERNODE_OPERATION_METRICS[11..13] {
|
||||
assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
}
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[12].labels,
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, FAILURE_REASON_LABEL]
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[13].labels,
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL]
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, FAILURE_REASON_LABEL]
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[14].labels,
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL]
|
||||
);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[15..17] {
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[15].labels,
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL]
|
||||
);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[16..18] {
|
||||
assert_eq!(metric.labels, &[SERVER_LABEL]);
|
||||
}
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[17].labels, &[SERVER_LABEL, REASON_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[18].labels, &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[18].labels, &[SERVER_LABEL, REASON_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[19].labels, &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]);
|
||||
// Payload histogram + large-payload counter carry operation+backend labels.
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[19].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[20].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[21].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1054,62 +1335,66 @@ mod tests {
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[6].name,
|
||||
"rustfs_system_network_internode_operation_classified_errors_total"
|
||||
"rustfs_system_network_internode_operation_stage_duration_ms"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[7].name,
|
||||
"rustfs_system_network_internode_operation_retries_total"
|
||||
"rustfs_system_network_internode_operation_classified_errors_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[8].name,
|
||||
"rustfs_system_network_internode_operation_retry_successes_total"
|
||||
"rustfs_system_network_internode_operation_retries_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[9].name,
|
||||
"rustfs_system_network_internode_operation_http_versions_total"
|
||||
"rustfs_system_network_internode_operation_retry_successes_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[10].name,
|
||||
"rustfs_system_network_internode_operation_stall_timeouts_total"
|
||||
"rustfs_system_network_internode_operation_http_versions_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[11].name,
|
||||
"rustfs_system_network_internode_operation_write_shutdown_errors_total"
|
||||
"rustfs_system_network_internode_operation_stall_timeouts_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[12].name,
|
||||
"rustfs_system_network_internode_rpc_auth_failures_total"
|
||||
"rustfs_system_network_internode_operation_write_shutdown_errors_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[13].name,
|
||||
"rustfs_system_network_internode_replay_cache_overflow_by_operation_total"
|
||||
"rustfs_system_network_internode_rpc_auth_failures_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[14].name,
|
||||
"rustfs_system_network_internode_replay_cache_records_total"
|
||||
"rustfs_system_network_internode_replay_cache_overflow_by_operation_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[15].name,
|
||||
"rustfs_system_network_internode_replay_cache_entries"
|
||||
"rustfs_system_network_internode_replay_cache_records_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[16].name,
|
||||
"rustfs_system_network_internode_replay_cache_capacity"
|
||||
"rustfs_system_network_internode_replay_cache_entries"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[17].name,
|
||||
"rustfs_system_network_internode_replay_cache_evictions_total"
|
||||
"rustfs_system_network_internode_replay_cache_capacity"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[18].name,
|
||||
"rustfs_system_storage_erasure_write_quorum_failures_total"
|
||||
"rustfs_system_network_internode_replay_cache_evictions_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[19].name,
|
||||
"rustfs_system_network_internode_operation_payload_bytes"
|
||||
"rustfs_system_storage_erasure_write_quorum_failures_total"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[20].name,
|
||||
"rustfs_system_network_internode_operation_payload_bytes"
|
||||
);
|
||||
assert_eq!(
|
||||
INTERNODE_OPERATION_METRICS[21].name,
|
||||
"rustfs_system_network_internode_operation_large_payloads_total"
|
||||
);
|
||||
assert_eq!(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, "grpc_read_multiple");
|
||||
@@ -1129,6 +1414,16 @@ mod tests {
|
||||
assert_eq!(INTERNODE_MSGPACK_DIRECTION_RESPONSE, "response");
|
||||
assert_eq!(INTERNODE_MSGPACK_CODEC_MSGPACK, "msgpack");
|
||||
assert_eq!(INTERNODE_MSGPACK_CODEC_JSON, "json");
|
||||
assert_eq!(INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, "read_version_request_encode");
|
||||
assert_eq!(INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE, "read_version_request_decode");
|
||||
assert_eq!(INTERNODE_STAGE_READ_VERSION_DISK_READ, "read_version_disk_read");
|
||||
assert_eq!(INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE, "read_version_response_json_encode");
|
||||
assert_eq!(
|
||||
INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE,
|
||||
"read_version_response_msgpack_encode"
|
||||
);
|
||||
assert_eq!(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, "read_version_rpc_roundtrip");
|
||||
assert_eq!(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, "read_version_response_decode");
|
||||
assert_eq!(
|
||||
INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL,
|
||||
"rustfs_system_network_internode_signature_v1_fallback_total"
|
||||
|
||||
@@ -109,6 +109,7 @@ pub fn put_stage_timer() -> Option<std::time::Instant> {
|
||||
put_stage_metrics_enabled().then(std::time::Instant::now)
|
||||
}
|
||||
|
||||
pub const PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT: &str = "put_object_commit_namespace_lock_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT: &str = "set_disk_rename_quorum_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT: &str = "set_disk_rename_disk_wait";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_file_sync_permit_wait";
|
||||
@@ -120,6 +121,33 @@ pub const PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC: &str = "set_disk_rename_ba
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC: &str = "set_disk_rename_ancestor_dir_fsync";
|
||||
pub const PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL: &str = "set_disk_rename_rename_syscall";
|
||||
|
||||
pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED: &str = "disabled";
|
||||
pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS: &str = "le_250ms";
|
||||
pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS: &str = "le_500ms";
|
||||
pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS: &str = "le_1000ms";
|
||||
pub const PUT_COMMIT_LOCK_ADMISSION_BUDGET_GT_1000MS: &str = "gt_1000ms";
|
||||
|
||||
pub const PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED: &str = "acquired";
|
||||
pub const PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN: &str = "timeout_slowdown";
|
||||
pub const PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR: &str = "lock_error";
|
||||
|
||||
pub const PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL: &str = "serial";
|
||||
pub const PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL: &str = "parallel";
|
||||
pub const PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER: &str = "leader";
|
||||
pub const PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER: &str = "follower";
|
||||
pub const PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_WAITERS: &str = "enqueue_waiters";
|
||||
pub const PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_FILES: &str = "enqueue_files";
|
||||
pub const PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_BATCH_WAITERS: &str = "batch_waiters";
|
||||
pub const PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_BATCH_FILES: &str = "batch_files";
|
||||
pub const PUT_RENAME_QUORUM_FANOUT_STATE_SCHEDULED: &str = "scheduled";
|
||||
pub const PUT_RENAME_QUORUM_FANOUT_STATE_WRITE_QUORUM: &str = "write_quorum";
|
||||
pub const PUT_RENAME_QUORUM_FANOUT_STATE_SUCCESS: &str = "success";
|
||||
pub const PUT_RENAME_QUORUM_FANOUT_STATE_ERROR: &str = "error";
|
||||
pub const PUT_RENAME_QUORUM_FANOUT_STATE_PANIC: &str = "panic";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST: &str = "quorum_first";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL: &str = "quorum_tail";
|
||||
pub const PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR: &str = "error";
|
||||
|
||||
#[inline(always)]
|
||||
pub fn get_stage_metrics_enabled() -> bool {
|
||||
GET_STAGE_METRICS_ENABLED.load(Ordering::Relaxed)
|
||||
@@ -2042,6 +2070,76 @@ pub fn record_put_object_stage_duration_from(stage: &'static str, started_at: Op
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_put_object_commit_lock_admission(budget: &'static str, outcome: &'static str) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
counter!("rustfs_s3_put_object_commit_namespace_lock_admission_total", "budget" => budget, "outcome" => outcome).increment(1);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn put_stage_count_value(value: usize) -> f64 {
|
||||
match u32::try_from(value) {
|
||||
Ok(value) => f64::from(value),
|
||||
Err(_) => f64::from(u32::MAX),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_put_rename_fdatasync_batch(mode: &'static str, files: usize) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
histogram!("rustfs_s3_put_object_rename_fdatasync_batch_files", "mode" => mode).record(put_stage_count_value(files));
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_put_rename_fdatasync_group_wait(role: &'static str, duration_ms: f64) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
histogram!("rustfs_s3_put_object_rename_fdatasync_group_wait_ms", "role" => role).record(duration_ms);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_put_rename_fdatasync_group_outstanding(state: &'static str, count: usize) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
histogram!("rustfs_s3_put_object_rename_fdatasync_group_outstanding", "state" => state).record(put_stage_count_value(count));
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_put_rename_disk_wait_completion(position: &'static str, duration_ms: f64) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
histogram!("rustfs_s3_put_object_rename_disk_wait_completion_ms", "position" => position).record(duration_ms);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn record_put_rename_quorum_wait_fanout(
|
||||
scheduled: usize,
|
||||
write_quorum: usize,
|
||||
success: usize,
|
||||
error: usize,
|
||||
panicked: usize,
|
||||
) {
|
||||
if !put_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
for (state, count) in [
|
||||
(PUT_RENAME_QUORUM_FANOUT_STATE_SCHEDULED, scheduled),
|
||||
(PUT_RENAME_QUORUM_FANOUT_STATE_WRITE_QUORUM, write_quorum),
|
||||
(PUT_RENAME_QUORUM_FANOUT_STATE_SUCCESS, success),
|
||||
(PUT_RENAME_QUORUM_FANOUT_STATE_ERROR, error),
|
||||
(PUT_RENAME_QUORUM_FANOUT_STATE_PANIC, panicked),
|
||||
] {
|
||||
histogram!("rustfs_s3_put_object_rename_quorum_wait_fanout_disks", "state" => state).record(put_stage_count_value(count));
|
||||
}
|
||||
}
|
||||
|
||||
/// Record generic internal operation stage duration (non-PUT paths).
|
||||
/// Use this for metacache walks, listing, lifecycle, and other background
|
||||
/// operations that are NOT part of the PUT object hot path.
|
||||
@@ -3068,7 +3166,9 @@ mod tests {
|
||||
#[test]
|
||||
fn put_stage_sync_tail_labels_are_static_and_gated() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
assert_eq!(PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT, "put_object_commit_namespace_lock_wait");
|
||||
let stages = [
|
||||
PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_DISK_WAIT,
|
||||
PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT,
|
||||
@@ -3082,11 +3182,11 @@ mod tests {
|
||||
];
|
||||
let unique = stages.iter().copied().collect::<HashSet<_>>();
|
||||
assert_eq!(unique.len(), stages.len());
|
||||
assert!(
|
||||
stages
|
||||
.iter()
|
||||
.all(|stage| stage.starts_with("set_disk_rename_") && !stage.contains('/') && !stage.contains('{'))
|
||||
);
|
||||
assert!(stages.iter().all(|stage| {
|
||||
(stage.starts_with("set_disk_rename_") || *stage == PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_WAIT)
|
||||
&& !stage.contains('/')
|
||||
&& !stage.contains('{')
|
||||
}));
|
||||
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
@@ -3122,6 +3222,240 @@ mod tests {
|
||||
assert!(stages.iter().all(|stage| recorded.contains(*stage)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_commit_lock_admission_labels_are_static_and_gated() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let budgets = [
|
||||
PUT_COMMIT_LOCK_ADMISSION_BUDGET_DISABLED,
|
||||
PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
|
||||
PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS,
|
||||
PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_1000MS,
|
||||
PUT_COMMIT_LOCK_ADMISSION_BUDGET_GT_1000MS,
|
||||
];
|
||||
let outcomes = [
|
||||
PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
|
||||
PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
|
||||
PUT_COMMIT_LOCK_ADMISSION_OUTCOME_LOCK_ERROR,
|
||||
];
|
||||
assert_eq!(budgets.iter().copied().collect::<HashSet<_>>().len(), budgets.len());
|
||||
assert_eq!(outcomes.iter().copied().collect::<HashSet<_>>().len(), outcomes.len());
|
||||
assert!(budgets.iter().chain(outcomes.iter()).all(|label| {
|
||||
!label.contains('/')
|
||||
&& !label.contains('{')
|
||||
&& !label.contains('}')
|
||||
&& !label.contains(' ')
|
||||
&& label
|
||||
.chars()
|
||||
.all(|ch| ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '_')
|
||||
}));
|
||||
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
set_put_stage_metrics_enabled(false);
|
||||
record_put_object_commit_lock_admission(
|
||||
PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
|
||||
PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
|
||||
);
|
||||
|
||||
set_put_stage_metrics_enabled(true);
|
||||
record_put_object_commit_lock_admission(
|
||||
PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS,
|
||||
PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN,
|
||||
);
|
||||
record_put_object_commit_lock_admission(
|
||||
PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS,
|
||||
PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED,
|
||||
);
|
||||
set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(
|
||||
counter_total(&rows, "rustfs_s3_put_object_commit_namespace_lock_admission_total"),
|
||||
Some(2)
|
||||
);
|
||||
let label_sets = rows
|
||||
.iter()
|
||||
.filter(|(composite, _, _, _)| {
|
||||
composite.kind() == MetricKind::Counter
|
||||
&& composite.key().name() == "rustfs_s3_put_object_commit_namespace_lock_admission_total"
|
||||
})
|
||||
.map(|(composite, _, _, _)| {
|
||||
composite
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key().to_string(), label.value().to_string()))
|
||||
.collect::<HashSet<_>>()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert!(label_sets.contains(&HashSet::from([
|
||||
("budget".to_string(), PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_250MS.to_string()),
|
||||
("outcome".to_string(), PUT_COMMIT_LOCK_ADMISSION_OUTCOME_TIMEOUT_SLOWDOWN.to_string(),),
|
||||
])));
|
||||
assert!(label_sets.contains(&HashSet::from([
|
||||
("budget".to_string(), PUT_COMMIT_LOCK_ADMISSION_BUDGET_LE_500MS.to_string()),
|
||||
("outcome".to_string(), PUT_COMMIT_LOCK_ADMISSION_OUTCOME_ACQUIRED.to_string()),
|
||||
])));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_rename_code_level_metrics_are_static_and_gated() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
set_put_stage_metrics_enabled(false);
|
||||
record_put_rename_fdatasync_batch(PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL, 2);
|
||||
record_put_rename_fdatasync_group_wait(PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER, 1.0);
|
||||
record_put_rename_fdatasync_group_outstanding(PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_WAITERS, 2);
|
||||
record_put_rename_disk_wait_completion(PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST, 3.0);
|
||||
record_put_rename_quorum_wait_fanout(4, 3, 3, 1, 0);
|
||||
|
||||
set_put_stage_metrics_enabled(true);
|
||||
record_put_rename_fdatasync_batch(PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL, 9);
|
||||
record_put_rename_fdatasync_group_wait(PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER, 1.0);
|
||||
record_put_rename_fdatasync_group_wait(PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER, 2.0);
|
||||
record_put_rename_fdatasync_group_outstanding(PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_WAITERS, 2);
|
||||
record_put_rename_fdatasync_group_outstanding(PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_FILES, 4);
|
||||
record_put_rename_fdatasync_group_outstanding(PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_BATCH_WAITERS, 3);
|
||||
record_put_rename_fdatasync_group_outstanding(PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_BATCH_FILES, 6);
|
||||
record_put_rename_disk_wait_completion(PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST, 3.0);
|
||||
record_put_rename_disk_wait_completion(PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL, 4.0);
|
||||
record_put_rename_disk_wait_completion(PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR, 5.0);
|
||||
record_put_rename_quorum_wait_fanout(4, 3, 3, 1, 0);
|
||||
set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(histogram_samples(&rows, "rustfs_s3_put_object_rename_fdatasync_batch_files"), vec![9.0]);
|
||||
let batch_modes = rows
|
||||
.iter()
|
||||
.filter(|(composite, _, _, _)| {
|
||||
composite.kind() == MetricKind::Histogram
|
||||
&& composite.key().name() == "rustfs_s3_put_object_rename_fdatasync_batch_files"
|
||||
})
|
||||
.flat_map(|(composite, _, _, _)| {
|
||||
composite
|
||||
.key()
|
||||
.labels()
|
||||
.filter(|label| label.key() == "mode")
|
||||
.map(|label| label.value().to_string())
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(batch_modes, HashSet::from([PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL.to_string()]));
|
||||
|
||||
let quorum_samples = histogram_samples(&rows, "rustfs_s3_put_object_rename_quorum_wait_fanout_disks");
|
||||
assert_eq!(quorum_samples, vec![0.0, 1.0, 3.0, 3.0, 4.0]);
|
||||
assert_eq!(
|
||||
histogram_samples(&rows, "rustfs_s3_put_object_rename_fdatasync_group_wait_ms"),
|
||||
vec![1.0, 2.0]
|
||||
);
|
||||
assert_eq!(
|
||||
histogram_samples(&rows, "rustfs_s3_put_object_rename_fdatasync_group_outstanding"),
|
||||
vec![2.0, 3.0, 4.0, 6.0]
|
||||
);
|
||||
assert_eq!(
|
||||
histogram_samples(&rows, "rustfs_s3_put_object_rename_disk_wait_completion_ms"),
|
||||
vec![3.0, 4.0, 5.0]
|
||||
);
|
||||
let group_wait_roles = rows
|
||||
.iter()
|
||||
.filter(|(composite, _, _, _)| {
|
||||
composite.kind() == MetricKind::Histogram
|
||||
&& composite.key().name() == "rustfs_s3_put_object_rename_fdatasync_group_wait_ms"
|
||||
})
|
||||
.flat_map(|(composite, _, _, _)| {
|
||||
composite
|
||||
.key()
|
||||
.labels()
|
||||
.filter(|label| label.key() == "role")
|
||||
.map(|label| label.value().to_string())
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(
|
||||
group_wait_roles,
|
||||
HashSet::from([
|
||||
PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_LEADER.to_string(),
|
||||
PUT_RENAME_FDATASYNC_GROUP_WAIT_ROLE_FOLLOWER.to_string(),
|
||||
])
|
||||
);
|
||||
let group_outstanding_states = rows
|
||||
.iter()
|
||||
.filter(|(composite, _, _, _)| {
|
||||
composite.kind() == MetricKind::Histogram
|
||||
&& composite.key().name() == "rustfs_s3_put_object_rename_fdatasync_group_outstanding"
|
||||
})
|
||||
.flat_map(|(composite, _, _, _)| {
|
||||
composite
|
||||
.key()
|
||||
.labels()
|
||||
.filter(|label| label.key() == "state")
|
||||
.map(|label| label.value().to_string())
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(
|
||||
group_outstanding_states,
|
||||
HashSet::from([
|
||||
PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_WAITERS.to_string(),
|
||||
PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_ENQUEUE_FILES.to_string(),
|
||||
PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_BATCH_WAITERS.to_string(),
|
||||
PUT_RENAME_FDATASYNC_GROUP_OUTSTANDING_STATE_BATCH_FILES.to_string(),
|
||||
])
|
||||
);
|
||||
let disk_wait_positions = rows
|
||||
.iter()
|
||||
.filter(|(composite, _, _, _)| {
|
||||
composite.kind() == MetricKind::Histogram
|
||||
&& composite.key().name() == "rustfs_s3_put_object_rename_disk_wait_completion_ms"
|
||||
})
|
||||
.flat_map(|(composite, _, _, _)| {
|
||||
composite
|
||||
.key()
|
||||
.labels()
|
||||
.filter(|label| label.key() == "position")
|
||||
.map(|label| label.value().to_string())
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(
|
||||
disk_wait_positions,
|
||||
HashSet::from([
|
||||
PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_FIRST.to_string(),
|
||||
PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_QUORUM_TAIL.to_string(),
|
||||
PUT_RENAME_DISK_WAIT_COMPLETION_POSITION_ERROR.to_string(),
|
||||
])
|
||||
);
|
||||
let quorum_states = rows
|
||||
.iter()
|
||||
.filter(|(composite, _, _, _)| {
|
||||
composite.kind() == MetricKind::Histogram
|
||||
&& composite.key().name() == "rustfs_s3_put_object_rename_quorum_wait_fanout_disks"
|
||||
})
|
||||
.flat_map(|(composite, _, _, _)| {
|
||||
composite
|
||||
.key()
|
||||
.labels()
|
||||
.filter(|label| label.key() == "state")
|
||||
.map(|label| label.value().to_string())
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(
|
||||
quorum_states,
|
||||
HashSet::from([
|
||||
PUT_RENAME_QUORUM_FANOUT_STATE_SCHEDULED.to_string(),
|
||||
PUT_RENAME_QUORUM_FANOUT_STATE_WRITE_QUORUM.to_string(),
|
||||
PUT_RENAME_QUORUM_FANOUT_STATE_SUCCESS.to_string(),
|
||||
PUT_RENAME_QUORUM_FANOUT_STATE_ERROR.to_string(),
|
||||
PUT_RENAME_QUORUM_FANOUT_STATE_PANIC.to_string(),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_put_object_diagnostic_buckets() {
|
||||
assert_eq!(put_object_size_bucket(0), "unknown");
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use metrics::with_local_recorder;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_STAGE_READ_VERSION_DISK_READ, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC, InternodeMetrics, set_internode_server_label,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
type MetricRow = (
|
||||
metrics_util::CompositeKey,
|
||||
Option<metrics::Unit>,
|
||||
Option<metrics::SharedString>,
|
||||
DebugValue,
|
||||
);
|
||||
|
||||
const SERVER_LABEL: &str = "server";
|
||||
const OPERATION_LABEL: &str = "operation";
|
||||
const BACKEND_LABEL: &str = "backend";
|
||||
const STAGE_LABEL: &str = "stage";
|
||||
const SENT_BYTES_TOTAL: &str = "rustfs_system_network_internode_sent_bytes_total";
|
||||
const RECV_BYTES_TOTAL: &str = "rustfs_system_network_internode_recv_bytes_total";
|
||||
const REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network_internode_requests_outgoing_total";
|
||||
const REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network_internode_requests_incoming_total";
|
||||
const ERRORS_TOTAL: &str = "rustfs_system_network_internode_errors_total";
|
||||
const OPERATION_SENT_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_sent_bytes_total";
|
||||
const OPERATION_RECV_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_recv_bytes_total";
|
||||
const OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_outgoing_total";
|
||||
const OPERATION_REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_incoming_total";
|
||||
const OPERATION_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_errors_total";
|
||||
const OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_operation_duration_ms";
|
||||
const OPERATION_STAGE_DURATION_MS: &str = "rustfs_system_network_internode_operation_stage_duration_ms";
|
||||
|
||||
#[test]
|
||||
fn cached_grpc_read_version_metric_handles_preserve_labels_and_values() {
|
||||
set_internode_server_label("cached-grpc-read-version-test");
|
||||
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let metrics = InternodeMetrics::default();
|
||||
|
||||
with_local_recorder(&recorder, || {
|
||||
metrics.record_sent_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
17,
|
||||
);
|
||||
metrics.record_recv_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
23,
|
||||
);
|
||||
metrics.record_outgoing_request_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
);
|
||||
metrics.record_incoming_request_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
);
|
||||
metrics.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
metrics.record_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
Duration::from_micros(250),
|
||||
);
|
||||
metrics.record_stage_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
|
||||
Duration::from_micros(125),
|
||||
);
|
||||
metrics.record_stage_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
INTERNODE_STAGE_READ_VERSION_DISK_READ,
|
||||
Duration::from_micros(75),
|
||||
);
|
||||
});
|
||||
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_counter(&rows, SENT_BYTES_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 17);
|
||||
assert_counter(&rows, RECV_BYTES_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 23);
|
||||
assert_counter(&rows, REQUESTS_OUTGOING_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 1);
|
||||
assert_counter(&rows, REQUESTS_INCOMING_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 1);
|
||||
assert_counter(&rows, ERRORS_TOTAL, &[(SERVER_LABEL, "cached-grpc-read-version-test")], 1);
|
||||
assert_counter(
|
||||
&rows,
|
||||
OPERATION_SENT_BYTES_TOTAL,
|
||||
&[
|
||||
(SERVER_LABEL, "cached-grpc-read-version-test"),
|
||||
(OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
|
||||
(BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
|
||||
],
|
||||
17,
|
||||
);
|
||||
assert_counter(
|
||||
&rows,
|
||||
OPERATION_RECV_BYTES_TOTAL,
|
||||
&[
|
||||
(SERVER_LABEL, "cached-grpc-read-version-test"),
|
||||
(OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
|
||||
(BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
|
||||
],
|
||||
23,
|
||||
);
|
||||
assert_counter(
|
||||
&rows,
|
||||
OPERATION_REQUESTS_OUTGOING_TOTAL,
|
||||
&[
|
||||
(SERVER_LABEL, "cached-grpc-read-version-test"),
|
||||
(OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
|
||||
(BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
|
||||
],
|
||||
1,
|
||||
);
|
||||
assert_counter(
|
||||
&rows,
|
||||
OPERATION_REQUESTS_INCOMING_TOTAL,
|
||||
&[
|
||||
(SERVER_LABEL, "cached-grpc-read-version-test"),
|
||||
(OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
|
||||
(BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
|
||||
],
|
||||
1,
|
||||
);
|
||||
assert_counter(
|
||||
&rows,
|
||||
OPERATION_ERRORS_TOTAL,
|
||||
&[
|
||||
(SERVER_LABEL, "cached-grpc-read-version-test"),
|
||||
(OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
|
||||
(BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
|
||||
],
|
||||
1,
|
||||
);
|
||||
assert_histogram(
|
||||
&rows,
|
||||
OPERATION_DURATION_MS,
|
||||
&[
|
||||
(SERVER_LABEL, "cached-grpc-read-version-test"),
|
||||
(OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
|
||||
(BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
|
||||
],
|
||||
&[0.25],
|
||||
);
|
||||
assert_histogram(
|
||||
&rows,
|
||||
OPERATION_STAGE_DURATION_MS,
|
||||
&[
|
||||
(SERVER_LABEL, "cached-grpc-read-version-test"),
|
||||
(OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
|
||||
(BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
|
||||
(STAGE_LABEL, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP),
|
||||
],
|
||||
&[0.125],
|
||||
);
|
||||
assert_histogram(
|
||||
&rows,
|
||||
OPERATION_STAGE_DURATION_MS,
|
||||
&[
|
||||
(SERVER_LABEL, "cached-grpc-read-version-test"),
|
||||
(OPERATION_LABEL, INTERNODE_OPERATION_GRPC_READ_VERSION),
|
||||
(BACKEND_LABEL, INTERNODE_TRANSPORT_BACKEND_GRPC),
|
||||
(STAGE_LABEL, INTERNODE_STAGE_READ_VERSION_DISK_READ),
|
||||
],
|
||||
&[0.075],
|
||||
);
|
||||
}
|
||||
|
||||
fn assert_counter(rows: &[MetricRow], name: &str, labels: &[(&str, &str)], expected: u64) {
|
||||
match metric_value(rows, name, labels) {
|
||||
DebugValue::Counter(value) => assert_eq!(*value, expected),
|
||||
other => panic!("{name} should be a counter, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_histogram(rows: &[MetricRow], name: &str, labels: &[(&str, &str)], expected: &[f64]) {
|
||||
match metric_value(rows, name, labels) {
|
||||
DebugValue::Histogram(samples) => {
|
||||
let actual: Vec<_> = samples.iter().map(|sample| sample.0).collect();
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
other => panic!("{name} should be a histogram, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn metric_value<'a>(rows: &'a [MetricRow], name: &str, labels: &[(&str, &str)]) -> &'a DebugValue {
|
||||
let mut matches = rows.iter().filter(|(composite, _, _, _)| {
|
||||
composite.key().name() == name
|
||||
&& labels.iter().all(|(key, value)| {
|
||||
composite
|
||||
.key()
|
||||
.labels()
|
||||
.any(|label| label.key() == *key && label.value() == *value)
|
||||
})
|
||||
});
|
||||
let Some((_, _, _, value)) = matches.next() else {
|
||||
panic!("{name} with labels {labels:?} was not recorded; rows={rows:?}");
|
||||
};
|
||||
assert!(matches.next().is_none(), "{name} with labels {labels:?} must be unique; rows={rows:?}");
|
||||
value
|
||||
}
|
||||
@@ -94,7 +94,7 @@ aws-smithy-types = { workspace = true }
|
||||
[dev-dependencies]
|
||||
anyhow = { workspace = true }
|
||||
# Debugging recorder for asserting emitted metrics in tests.
|
||||
metrics-util = { version = "0.20", features = ["debugging"] }
|
||||
metrics-util = { workspace = true, features = ["debugging"] }
|
||||
insta = { workspace = true, features = ["yaml", "json"] }
|
||||
tempfile = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
|
||||
@@ -67,7 +67,7 @@ url.workspace = true
|
||||
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
[dev-dependencies]
|
||||
metrics-util = { version = "0.20", features = ["debugging"] }
|
||||
metrics-util = { workspace = true, features = ["debugging"] }
|
||||
proptest = "1"
|
||||
serial_test.workspace = true
|
||||
temp-env.workspace = true
|
||||
|
||||
+204
-39
@@ -50,6 +50,9 @@ const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str =
|
||||
const ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT: &str = "Expiration.Date must be at midnight UTC";
|
||||
const ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_DELETE_MARKER: &str =
|
||||
"ExpiredObjectDeleteMarker cannot be specified with Days or Date";
|
||||
const ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS: &str =
|
||||
"Days must be a positive integer and Date must not be specified inside Expiration with ExpiredObjectAllVersions";
|
||||
const ERR_LIFECYCLE_INVALID_DEL_MARKER_EXPIRATION_DAYS: &str = "Days must be a positive integer with DelMarkerExpiration";
|
||||
const ERR_LIFECYCLE_INVALID_RULE_ID_TOO_LONG: &str = "Rule ID must be at most 255 characters";
|
||||
const ERR_LIFECYCLE_INVALID_RULE_STATUS: &str = "Rule status must be either Enabled or Disabled";
|
||||
const ERR_LIFECYCLE_DEL_MARKER_WITH_TAGS: &str = "Rule with DelMarkerExpiration cannot have tags based filtering";
|
||||
@@ -155,6 +158,13 @@ impl RuleValidate for LifecycleRule {
|
||||
{
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_DELETE_MARKER));
|
||||
}
|
||||
if self
|
||||
.del_marker_expiration
|
||||
.as_ref()
|
||||
.is_some_and(|expiration| expiration.days.is_none_or(|days| days < 1))
|
||||
{
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_DEL_MARKER_EXPIRATION_DAYS));
|
||||
}
|
||||
// Rule must have at least one action
|
||||
let has_expiration = self.expiration.is_some();
|
||||
let has_transition = self.transitions.as_ref().is_some_and(|t| !t.is_empty());
|
||||
@@ -291,6 +301,14 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if rule
|
||||
.del_marker_expiration
|
||||
.as_ref()
|
||||
.and_then(|expiration| expiration.days)
|
||||
.is_some_and(|days| days > 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if let Some(rule_expiration) = &rule.expiration {
|
||||
if let Some(date1) = rule_expiration.date.clone()
|
||||
&& OffsetDateTime::from(date1).unix_timestamp() < OffsetDateTime::now_utc().unix_timestamp()
|
||||
@@ -337,6 +355,11 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
}
|
||||
|
||||
if let Some(expiration) = &r.expiration {
|
||||
if expiration.expired_object_all_versions.is_some()
|
||||
&& (expiration.days.is_none_or(|days| days < 1) || expiration.date.is_some())
|
||||
{
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS));
|
||||
}
|
||||
if let Some(expiration_date) = &expiration.date {
|
||||
let date = OffsetDateTime::from(expiration_date.clone());
|
||||
if date.hour() != 0 || date.minute() != 0 || date.second() != 0 || date.nanosecond() != 0 {
|
||||
@@ -563,25 +586,25 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
});
|
||||
}
|
||||
}
|
||||
// DelMarkerExpiration: expire delete marker after N days from mod_time
|
||||
if obj.delete_marker
|
||||
&& let Some(ref dme) = rule.del_marker_expiration
|
||||
&& let Some(days) = dme.days
|
||||
&& days > 0
|
||||
{
|
||||
let due = expected_expiry_time(mod_time, days);
|
||||
if now.unix_timestamp() >= due.unix_timestamp() {
|
||||
events.push(Event {
|
||||
action: IlmAction::DelMarkerDeleteAllVersionsAction,
|
||||
rule_id: rule.id.clone().unwrap_or_default(),
|
||||
due: Some(due),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
storage_class: "".into(),
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if obj.is_latest
|
||||
&& obj.delete_marker
|
||||
&& let Some(days) = rule.del_marker_expiration.as_ref().and_then(|expiration| expiration.days)
|
||||
&& days > 0
|
||||
{
|
||||
let due = expected_expiry_time(mod_time, days);
|
||||
if now.unix_timestamp() >= due.unix_timestamp() {
|
||||
events.push(Event {
|
||||
action: IlmAction::DelMarkerDeleteAllVersionsAction,
|
||||
rule_id: rule.id.clone().unwrap_or_default(),
|
||||
due: Some(due),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
storage_class: "".into(),
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if !obj.is_latest
|
||||
@@ -1105,6 +1128,25 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
fn enabled_rule(
|
||||
expiration: Option<LifecycleExpiration>,
|
||||
del_marker_expiration: Option<s3s::dto::DelMarkerExpiration>,
|
||||
id: Option<&str>,
|
||||
) -> LifecycleRule {
|
||||
LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration,
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration,
|
||||
filter: None,
|
||||
id: id.map(str::to_string),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eval_inner_reports_invalid_mod_time_without_expiring_object() {
|
||||
let lifecycle = BucketLifecycleConfiguration {
|
||||
@@ -1344,6 +1386,18 @@ mod tests {
|
||||
assert!(lc.has_active_rules("test/"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_active_rules_requires_valid_del_marker_expiration_days() {
|
||||
let lifecycle = |days| BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![enabled_rule(None, Some(s3s::dto::DelMarkerExpiration { days }), None)],
|
||||
};
|
||||
|
||||
assert!(lifecycle(Some(1)).has_active_rules(""));
|
||||
assert!(!lifecycle(Some(0)).has_active_rules(""));
|
||||
assert!(!lifecycle(None).has_active_rules(""));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_rejects_zero_noncurrent_expiration_days() {
|
||||
// S3 compatibility: NoncurrentVersionExpiration.NoncurrentDays must be a positive
|
||||
@@ -3182,33 +3236,68 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_rejects_zero_day_del_marker_expiration_on_locked_bucket() {
|
||||
async fn validate_rejects_invalid_del_marker_expiration_days_even_with_another_action() {
|
||||
for days in [None, Some(0), Some(-1)] {
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(30),
|
||||
..Default::default()
|
||||
}),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: Some(s3s::dto::DelMarkerExpiration { days }),
|
||||
filter: None,
|
||||
id: Some("test-rule".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
};
|
||||
|
||||
let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
|
||||
assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_DEL_MARKER_EXPIRATION_DAYS);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn del_marker_expiration_deletes_marker_and_older_versions_when_due() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid");
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(30),
|
||||
..Default::default()
|
||||
}),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: Some(s3s::dto::DelMarkerExpiration { days: Some(0) }),
|
||||
filter: None,
|
||||
id: Some("test-rule".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
rules: vec![enabled_rule(
|
||||
None,
|
||||
Some(s3s::dto::DelMarkerExpiration { days: Some(3) }),
|
||||
Some("delete-marker-history"),
|
||||
)],
|
||||
};
|
||||
|
||||
let locked_config = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
let marker_with_history = ObjectOpts {
|
||||
name: "obj".to_string(),
|
||||
mod_time: Some(base_time),
|
||||
is_latest: true,
|
||||
delete_marker: true,
|
||||
num_versions: 3,
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
let due = expected_expiry_time(base_time, 3);
|
||||
|
||||
let err = lc.validate(&locked_config).await.unwrap_err();
|
||||
assert_eq!(err.to_string(), ERR_LIFECYCLE_BUCKET_LOCKED);
|
||||
let before_due = lc.eval_inner(&marker_with_history, due - Duration::seconds(1), 0).await;
|
||||
assert_eq!(before_due.action, IlmAction::NoneAction);
|
||||
|
||||
let at_due = lc.eval_inner(&marker_with_history, due, 0).await;
|
||||
assert_eq!(at_due.action, IlmAction::DelMarkerDeleteAllVersionsAction);
|
||||
assert_eq!(at_due.rule_id, "delete-marker-history");
|
||||
assert_eq!(at_due.due, Some(due));
|
||||
|
||||
let current_data = ObjectOpts {
|
||||
delete_marker: false,
|
||||
..marker_with_history
|
||||
};
|
||||
assert_eq!(lc.eval_inner(¤t_data, due, 0).await.action, IlmAction::NoneAction);
|
||||
}
|
||||
|
||||
// --- TASK-003 tests: Round up to next UTC processing boundary ---
|
||||
@@ -3737,6 +3826,52 @@ mod tests {
|
||||
.expect("ExpiredObjectAllVersions should be allowed on unlocked bucket");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_rejects_expired_object_all_versions_without_days_or_with_date() {
|
||||
let expiration_date = datetime!(2025-01-01 00:00:00 UTC);
|
||||
let invalid_expirations = [
|
||||
LifecycleExpiration {
|
||||
expired_object_all_versions: Some(true),
|
||||
..Default::default()
|
||||
},
|
||||
LifecycleExpiration {
|
||||
date: Some(expiration_date.into()),
|
||||
days: Some(1),
|
||||
expired_object_all_versions: Some(true),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
|
||||
for expiration in invalid_expirations {
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![enabled_rule(Some(expiration), None, None)],
|
||||
};
|
||||
|
||||
let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
|
||||
assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn validate_rejects_false_expired_object_all_versions_with_date() {
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![enabled_rule(
|
||||
Some(LifecycleExpiration {
|
||||
date: Some(datetime!(2025-01-01 00:00:00 UTC).into()),
|
||||
expired_object_all_versions: Some(false),
|
||||
..Default::default()
|
||||
}),
|
||||
None,
|
||||
None,
|
||||
)],
|
||||
};
|
||||
|
||||
let err = lc.validate(&ObjectLockConfiguration::default()).await.unwrap_err();
|
||||
assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_ALL_VERSIONS);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_triggers_delete_all_versions_when_expired_object_all_versions_set() {
|
||||
@@ -3776,6 +3911,36 @@ mod tests {
|
||||
assert_eq!(event.rule_id, "all-versions-rule");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn expired_object_all_versions_does_not_apply_to_current_delete_marker() {
|
||||
let base_time = OffsetDateTime::from_unix_timestamp(1_000_000).expect("fixed timestamp should be valid");
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![enabled_rule(
|
||||
Some(LifecycleExpiration {
|
||||
days: Some(1),
|
||||
expired_object_all_versions: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
None,
|
||||
Some("all-versions-rule"),
|
||||
)],
|
||||
};
|
||||
let marker_with_history = ObjectOpts {
|
||||
name: "obj".to_string(),
|
||||
mod_time: Some(base_time),
|
||||
is_latest: true,
|
||||
delete_marker: true,
|
||||
num_versions: 2,
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let event = lc.eval_inner(&marker_with_history, base_time + Duration::days(2), 0).await;
|
||||
assert_eq!(event.action, IlmAction::NoneAction);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn eval_inner_uses_delete_action_when_all_versions_not_set() {
|
||||
|
||||
@@ -48,6 +48,7 @@ pub struct LocalClient {
|
||||
struct LocalGuardEntry {
|
||||
guard: FastLockGuard,
|
||||
acquired_at: SystemTime,
|
||||
last_refreshed: SystemTime,
|
||||
expires_at: SystemTime,
|
||||
deadline: Instant,
|
||||
ttl: Duration,
|
||||
@@ -60,6 +61,7 @@ impl LocalGuardEntry {
|
||||
Self {
|
||||
guard,
|
||||
acquired_at,
|
||||
last_refreshed: acquired_at,
|
||||
expires_at: acquired_at.checked_add(ttl).unwrap_or(acquired_at),
|
||||
deadline: monotonic_now.checked_add(ttl).unwrap_or(monotonic_now),
|
||||
ttl,
|
||||
@@ -74,6 +76,7 @@ impl LocalGuardEntry {
|
||||
let now = SystemTime::now();
|
||||
let monotonic_now = Instant::now();
|
||||
self.expires_at = now.checked_add(self.ttl).unwrap_or(now);
|
||||
self.last_refreshed = now;
|
||||
self.deadline = monotonic_now.checked_add(self.ttl).unwrap_or(monotonic_now);
|
||||
}
|
||||
}
|
||||
@@ -347,7 +350,7 @@ impl LockClient for LocalClient {
|
||||
owner: entry.guard.owner().to_string(),
|
||||
acquired_at: entry.acquired_at,
|
||||
expires_at: entry.expires_at,
|
||||
last_refreshed: SystemTime::now(),
|
||||
last_refreshed: entry.last_refreshed,
|
||||
metadata: LockMetadata::default(),
|
||||
priority: LockPriority::Normal,
|
||||
wait_start_time: None,
|
||||
@@ -371,6 +374,7 @@ impl LockClient for LocalClient {
|
||||
owner: entry.guard.owner().to_string(),
|
||||
acquired_at: entry.acquired_at,
|
||||
remaining_ttl: entry.deadline.saturating_duration_since(Instant::now()),
|
||||
guard_id: (!entry.guard.is_disabled()).then(|| entry.guard.guard_id()),
|
||||
}));
|
||||
}
|
||||
leases
|
||||
@@ -484,6 +488,12 @@ mod tests {
|
||||
.success
|
||||
);
|
||||
let initial = client.list_lock_leases().await.pop().expect("acquired lock should be listed");
|
||||
let initial_status = client
|
||||
.check_status(&lock_id)
|
||||
.await
|
||||
.expect("initial lock status should be readable")
|
||||
.expect("newly acquired lock should remain held");
|
||||
assert_eq!(initial_status.last_refreshed, initial_status.acquired_at);
|
||||
|
||||
tokio::time::advance(Duration::from_secs(20)).await;
|
||||
let aging = client
|
||||
@@ -492,6 +502,13 @@ mod tests {
|
||||
.pop()
|
||||
.expect("held lock should remain listed before refresh");
|
||||
assert_eq!(aging.remaining_ttl, Duration::from_secs(10));
|
||||
let aging_status = client
|
||||
.check_status(&lock_id)
|
||||
.await
|
||||
.expect("aging lock status should be readable")
|
||||
.expect("aging lock should remain held");
|
||||
assert_eq!(aging_status.last_refreshed, initial_status.last_refreshed);
|
||||
|
||||
assert!(client.refresh(&lock_id).await.expect("refresh should return a result"));
|
||||
|
||||
let refreshed = client
|
||||
@@ -506,7 +523,9 @@ mod tests {
|
||||
.expect("refreshed lock should remain held");
|
||||
|
||||
assert_eq!(refreshed.acquired_at, initial.acquired_at);
|
||||
assert_eq!(refreshed.guard_id, initial.guard_id);
|
||||
assert_eq!(status.acquired_at, initial.acquired_at);
|
||||
assert!(status.last_refreshed > initial_status.last_refreshed);
|
||||
assert_eq!(refreshed.remaining_ttl, Duration::from_secs(30));
|
||||
|
||||
tokio::time::advance(Duration::from_secs(30)).await;
|
||||
|
||||
@@ -100,7 +100,7 @@ impl FastObjectLockManager {
|
||||
Ok(()) => {
|
||||
let guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
|
||||
// Register guard to prevent premature cleanup
|
||||
shard.register_guard(guard.guard_id());
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
Ok(guard)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
@@ -223,7 +223,7 @@ impl FastObjectLockManager {
|
||||
|
||||
if acquired {
|
||||
let guard = FastLockGuard::new(key.clone(), mode, owner.clone(), shard.clone());
|
||||
shard.register_guard(guard.guard_id());
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
all_successful.push(key);
|
||||
guards.push(guard);
|
||||
}
|
||||
@@ -252,7 +252,7 @@ impl FastObjectLockManager {
|
||||
match shard.acquire_lock(request).await {
|
||||
Ok(()) => {
|
||||
let guard = FastLockGuard::new(request.key.clone(), request.mode, request.owner.clone(), shard.clone());
|
||||
shard.register_guard(guard.guard_id());
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
acquired_guards.push(guard);
|
||||
}
|
||||
Err(err) => {
|
||||
@@ -310,6 +310,15 @@ impl FastObjectLockManager {
|
||||
infos
|
||||
}
|
||||
|
||||
/// Enumerate held locks with holder counts and stable holder identities.
|
||||
pub fn list_locks_with_holder_generations(&self) -> Vec<(crate::fast_lock::types::ObjectLockInfo, u32, Option<Vec<u64>>)> {
|
||||
let mut infos = Vec::new();
|
||||
for shard in &self.shards {
|
||||
infos.extend(shard.list_locks_with_holder_generations());
|
||||
}
|
||||
infos
|
||||
}
|
||||
|
||||
/// Force-release every holder of the lock on `key`.
|
||||
///
|
||||
/// Returns the number of owners released (0 if the resource was not locked).
|
||||
@@ -556,15 +565,15 @@ mod tests {
|
||||
let write_key = ObjectKey::new("bucket", "write-object");
|
||||
let read_key = ObjectKey::new("bucket", "read-object");
|
||||
|
||||
let _write_guard = manager
|
||||
let write_guard = manager
|
||||
.acquire_write_lock(write_key.clone(), "writer")
|
||||
.await
|
||||
.expect("write lock should acquire");
|
||||
let _read_guard = manager
|
||||
let read_guard = manager
|
||||
.acquire_read_lock(read_key.clone(), "reader")
|
||||
.await
|
||||
.expect("read lock should acquire");
|
||||
let _second_read_guard = manager
|
||||
let second_read_guard = manager
|
||||
.acquire_read_lock(read_key.clone(), "reader")
|
||||
.await
|
||||
.expect("second read lock should acquire");
|
||||
@@ -593,6 +602,30 @@ mod tests {
|
||||
.expect("write holder count listed");
|
||||
assert_eq!(*write_holder_count, 1);
|
||||
|
||||
let generations = manager.list_locks_with_holder_generations();
|
||||
let (_, _, read_generations) = generations
|
||||
.iter()
|
||||
.find(|(info, _, _)| info.key == read_key)
|
||||
.expect("read holder generations listed");
|
||||
let mut expected_read_generations = vec![read_guard.guard_id(), second_read_guard.guard_id()];
|
||||
expected_read_generations.sort_unstable();
|
||||
assert_eq!(read_generations.as_ref(), Some(&expected_read_generations));
|
||||
|
||||
let (_, _, write_generations) = generations
|
||||
.iter()
|
||||
.find(|(info, _, _)| info.key == write_key)
|
||||
.expect("write holder generation listed");
|
||||
assert_eq!(write_generations.as_ref(), Some(&vec![write_guard.guard_id()]));
|
||||
|
||||
drop(read_guard);
|
||||
let remaining = manager.list_locks_with_holder_generations();
|
||||
let (_, remaining_count, remaining_generations) = remaining
|
||||
.iter()
|
||||
.find(|(info, _, _)| info.key == read_key)
|
||||
.expect("remaining read holder generation listed");
|
||||
assert_eq!(*remaining_count, 1);
|
||||
assert_eq!(remaining_generations.as_ref(), Some(&vec![second_read_guard.guard_id()]));
|
||||
|
||||
manager.shutdown().await;
|
||||
}
|
||||
|
||||
|
||||
@@ -24,7 +24,20 @@ use crate::fast_lock::{
|
||||
state::ObjectLockState,
|
||||
types::{LockMode, LockResult, ObjectKey, ObjectLockRequest},
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ActiveGuardInfo {
|
||||
key: ObjectKey,
|
||||
mode: LockMode,
|
||||
owner: Arc<str>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Hash)]
|
||||
struct GuardHolderKey {
|
||||
key: ObjectKey,
|
||||
mode: LockMode,
|
||||
owner: Arc<str>,
|
||||
}
|
||||
|
||||
/// Lock shard to reduce global contention
|
||||
#[derive(Debug)]
|
||||
@@ -38,7 +51,7 @@ pub struct LockShard {
|
||||
/// Shard ID for debugging
|
||||
_shard_id: usize,
|
||||
/// Active guard IDs to prevent cleanup of locks with live guards
|
||||
active_guards: parking_lot::Mutex<HashSet<u64>>,
|
||||
active_guards: parking_lot::Mutex<HashMap<u64, Option<ActiveGuardInfo>>>,
|
||||
}
|
||||
|
||||
/// Cancellation-safe waiter counter ticket.
|
||||
@@ -84,7 +97,7 @@ impl LockShard {
|
||||
object_pool: ObjectStatePool::new(),
|
||||
metrics: ShardMetrics::new(),
|
||||
_shard_id: shard_id,
|
||||
active_guards: parking_lot::Mutex::new(HashSet::new()),
|
||||
active_guards: parking_lot::Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -327,7 +340,7 @@ impl LockShard {
|
||||
// First, try to remove the guard from active set
|
||||
let guard_was_active = {
|
||||
let mut guards = self.active_guards.lock();
|
||||
guards.remove(&guard_id)
|
||||
guards.remove(&guard_id).is_some()
|
||||
};
|
||||
|
||||
// If guard was not active, this is a double-release attempt
|
||||
@@ -375,8 +388,19 @@ impl LockShard {
|
||||
|
||||
/// Register a guard to prevent premature cleanup
|
||||
pub fn register_guard(&self, guard_id: u64) {
|
||||
self.active_guards.lock().insert(guard_id, None);
|
||||
}
|
||||
|
||||
pub(crate) fn register_guard_with_info(&self, guard_id: u64, key: &ObjectKey, mode: LockMode, owner: &Arc<str>) {
|
||||
let mut guards = self.active_guards.lock();
|
||||
guards.insert(guard_id);
|
||||
guards.insert(
|
||||
guard_id,
|
||||
Some(ActiveGuardInfo {
|
||||
key: key.clone(),
|
||||
mode,
|
||||
owner: owner.clone(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/// Unregister a guard (called when guard is dropped)
|
||||
@@ -396,7 +420,7 @@ impl LockShard {
|
||||
#[cfg(test)]
|
||||
pub fn is_guard_active(&self, guard_id: u64) -> bool {
|
||||
let guards = self.active_guards.lock();
|
||||
guards.contains(&guard_id)
|
||||
guards.contains_key(&guard_id)
|
||||
}
|
||||
|
||||
/// Calculate adaptive timeout based on current system load and request priority
|
||||
@@ -602,6 +626,50 @@ impl LockShard {
|
||||
infos
|
||||
}
|
||||
|
||||
pub(crate) fn list_locks_with_holder_generations(
|
||||
&self,
|
||||
) -> Vec<(crate::fast_lock::types::ObjectLockInfo, u32, Option<Vec<u64>>)> {
|
||||
// Snapshot lock state before guard registrations. Acquires register after
|
||||
// mutating state, while releases unregister before mutating state, so a
|
||||
// concurrent transition can only make the cohort mismatch and fall back.
|
||||
let infos = self.list_locks_with_holder_counts();
|
||||
let guards = self.active_guards.lock();
|
||||
let mut guard_ids_by_holder: HashMap<GuardHolderKey, Vec<u64>> = HashMap::with_capacity(guards.len());
|
||||
for (&guard_id, guard) in guards
|
||||
.iter()
|
||||
.filter_map(|(guard_id, guard)| guard.as_ref().map(|guard| (guard_id, guard)))
|
||||
{
|
||||
let key = GuardHolderKey {
|
||||
key: guard.key.clone(),
|
||||
mode: guard.mode,
|
||||
owner: guard.owner.clone(),
|
||||
};
|
||||
guard_ids_by_holder
|
||||
.entry(key)
|
||||
.and_modify(|guard_ids| guard_ids.push(guard_id))
|
||||
.or_insert_with(|| vec![guard_id]);
|
||||
}
|
||||
drop(guards);
|
||||
for guard_ids in guard_ids_by_holder.values_mut() {
|
||||
guard_ids.sort_unstable();
|
||||
}
|
||||
|
||||
infos
|
||||
.into_iter()
|
||||
.map(|(info, holder_count)| {
|
||||
let key = GuardHolderKey {
|
||||
key: info.key.clone(),
|
||||
mode: info.mode,
|
||||
owner: info.owner.clone(),
|
||||
};
|
||||
let generation = guard_ids_by_holder
|
||||
.remove(&key)
|
||||
.filter(|guard_ids| u32::try_from(guard_ids.len()).ok() == Some(holder_count));
|
||||
(info, holder_count, generation)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Force-release every holder of a lock on `key`, regardless of owner.
|
||||
///
|
||||
/// Returns the number of owners that were released. Used by the admin
|
||||
|
||||
@@ -257,7 +257,7 @@ impl std::fmt::Display for ObjectKey {
|
||||
}
|
||||
|
||||
/// Lock type for object operations
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum LockMode {
|
||||
/// Shared lock for read operations
|
||||
Shared,
|
||||
|
||||
@@ -90,6 +90,8 @@ pub struct LockLeaseInfo {
|
||||
pub owner: String,
|
||||
/// Original acquisition time. Refreshes do not change this value.
|
||||
pub acquired_at: SystemTime,
|
||||
/// Opaque guard identity used to reject stale diagnostic snapshots.
|
||||
pub guard_id: Option<u64>,
|
||||
/// Remaining lease duration derived from the monotonic lease deadline.
|
||||
pub remaining_ttl: Duration,
|
||||
}
|
||||
|
||||
@@ -368,9 +368,10 @@ impl AdminClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// Cluster-aggregated background heal status.
|
||||
/// Cluster-aggregated background heal status. The route is registered
|
||||
/// POST-only on the server, so this must not go out as a GET.
|
||||
pub async fn background_heal_status(&self) -> Result<BackgroundHealStatus, AdminClientError> {
|
||||
self.get_json("/v3/background-heal/status").await
|
||||
self.post_json("/v3/background-heal/status", &[], Vec::new()).await
|
||||
}
|
||||
|
||||
/// Data scanner status (enabled state, freshness, runtime config).
|
||||
@@ -698,6 +699,21 @@ mod tests {
|
||||
assert!(!request.query.contains("clientToken"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_heal_status_posts_to_the_registered_route() {
|
||||
let body = r#"{"state":"idle","healQueueLength":0,"healActiveTasks":0,"clusterStatusComplete":true}"#;
|
||||
let server = TestServer::spawn(body, 200).await;
|
||||
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").unwrap();
|
||||
|
||||
let status = client.background_heal_status().await.expect("status decodes");
|
||||
assert_eq!(status.state, "idle");
|
||||
let request = server.recorded();
|
||||
// The server registers this route POST-only; a GET here answers 405.
|
||||
assert_eq!(request.method, "POST");
|
||||
assert_eq!(request.path, "/rustfs/admin/v3/background-heal/status");
|
||||
assert_eq!(request.query, "");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_error_status_maps_to_a_typed_error_with_body() {
|
||||
let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await;
|
||||
|
||||
@@ -19,7 +19,6 @@ use hyper::Uri;
|
||||
use crate::{trace::TraceType, utils::parse_duration};
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
#[allow(dead_code)]
|
||||
pub struct ServiceTraceOpts {
|
||||
s3: bool,
|
||||
internal: bool,
|
||||
@@ -41,7 +40,6 @@ pub struct ServiceTraceOpts {
|
||||
threshold: Duration,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl ServiceTraceOpts {
|
||||
pub fn trace_types(&self) -> TraceType {
|
||||
let mut tt = TraceType::default();
|
||||
|
||||
@@ -58,7 +58,7 @@ tracing = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
metrics-util = { version = "0.20", features = ["debugging"] }
|
||||
metrics-util = { workspace = true, features = ["debugging"] }
|
||||
# `rt-multi-thread` lets the concurrency stress tests run tasks on real worker
|
||||
# threads, so they exercise true parallelism on the shared singleflight/index
|
||||
# state rather than only cooperative interleaving.
|
||||
|
||||
@@ -163,4 +163,5 @@ libc = { workspace = true }
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
log = "0.4"
|
||||
log.workspace = true
|
||||
metrics-util = { workspace = true, features = ["debugging"] }
|
||||
|
||||
@@ -54,12 +54,37 @@ pub(crate) struct IlmActionTaskStats {
|
||||
pub(crate) value: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct IlmQueueTaskStats {
|
||||
pub(crate) action: String,
|
||||
pub(crate) state: String,
|
||||
pub(crate) value: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct IlmTaskEventStats {
|
||||
pub(crate) action: String,
|
||||
pub(crate) result: String,
|
||||
pub(crate) value: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct IlmBackpressureStats {
|
||||
pub(crate) action: String,
|
||||
pub(crate) reason: String,
|
||||
pub(crate) value: u64,
|
||||
}
|
||||
|
||||
/// ILM statistics with runtime-local node identity and bounded action/state details.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct IlmRuntimeStats {
|
||||
pub(crate) server: String,
|
||||
pub(crate) stats: IlmStats,
|
||||
pub(crate) action_tasks: Vec<IlmActionTaskStats>,
|
||||
pub(crate) queue_tasks: Vec<IlmQueueTaskStats>,
|
||||
pub(crate) task_events: Vec<IlmTaskEventStats>,
|
||||
pub(crate) backpressure: Vec<IlmBackpressureStats>,
|
||||
pub(crate) versions_scanned: u64,
|
||||
}
|
||||
|
||||
fn is_live_action_task_state(state: &str) -> bool {
|
||||
@@ -112,6 +137,30 @@ pub(crate) fn collect_ilm_runtime_metrics(stats: &IlmRuntimeStats) -> Vec<Promet
|
||||
}),
|
||||
);
|
||||
|
||||
metrics.extend(stats.queue_tasks.iter().map(|task| {
|
||||
PrometheusMetric::from_descriptor(&ILM_TASKS_MD, task.value as f64)
|
||||
.with_label_owned(SERVER_LABEL, stats.server.clone())
|
||||
.with_label_owned(ACTION_LABEL, task.action.clone())
|
||||
.with_label_owned(QUEUE_STATE_LABEL, task.state.clone())
|
||||
}));
|
||||
metrics.extend(stats.task_events.iter().map(|event| {
|
||||
PrometheusMetric::from_descriptor(&ILM_TASK_EVENTS_MD, event.value as f64)
|
||||
.with_label_owned(SERVER_LABEL, stats.server.clone())
|
||||
.with_label_owned(ACTION_LABEL, event.action.clone())
|
||||
.with_label_owned(RESULT_LABEL, event.result.clone())
|
||||
}));
|
||||
metrics.extend(stats.backpressure.iter().map(|event| {
|
||||
PrometheusMetric::from_descriptor(&ILM_QUEUE_BACKPRESSURE_MD, event.value as f64)
|
||||
.with_label_owned(SERVER_LABEL, stats.server.clone())
|
||||
.with_label_owned(ACTION_LABEL, event.action.clone())
|
||||
.with_label_owned(REASON_LABEL, event.reason.clone())
|
||||
}));
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&ILM_VERSIONS_SCANNED_BY_SERVER_MD, stats.versions_scanned as f64)
|
||||
.with_label_owned(SERVER_LABEL, stats.server.clone())
|
||||
.with_label_owned(SOURCE_LABEL, "lifecycle".to_string()),
|
||||
);
|
||||
|
||||
metrics
|
||||
}
|
||||
|
||||
@@ -135,6 +184,22 @@ mod tests {
|
||||
let runtime_stats = IlmRuntimeStats {
|
||||
server: "node1:9000".to_string(),
|
||||
stats,
|
||||
queue_tasks: vec![IlmQueueTaskStats {
|
||||
action: "transition".to_string(),
|
||||
state: "pending".to_string(),
|
||||
value: 8,
|
||||
}],
|
||||
task_events: vec![IlmTaskEventStats {
|
||||
action: "transition".to_string(),
|
||||
result: "completed".to_string(),
|
||||
value: 7,
|
||||
}],
|
||||
backpressure: vec![IlmBackpressureStats {
|
||||
action: "transition".to_string(),
|
||||
reason: "queue_full".to_string(),
|
||||
value: 2,
|
||||
}],
|
||||
versions_scanned: 1000000,
|
||||
action_tasks: vec![
|
||||
IlmActionTaskStats {
|
||||
action: "expiry".to_string(),
|
||||
@@ -156,7 +221,7 @@ mod tests {
|
||||
|
||||
let metrics = collect_ilm_runtime_metrics(&runtime_stats);
|
||||
|
||||
assert_eq!(metrics.len(), 11);
|
||||
assert_eq!(metrics.len(), 15);
|
||||
|
||||
let pending = metrics.iter().find(|m| m.value == 100.0);
|
||||
assert!(pending.is_some());
|
||||
@@ -178,6 +243,44 @@ mod tests {
|
||||
});
|
||||
assert!(transition_timeout.is_none());
|
||||
|
||||
let transition_queue = metrics.iter().find(|m| {
|
||||
m.name == ILM_TASKS_MD.get_full_metric_name()
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == ACTION_LABEL && value.as_ref() == "transition")
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == QUEUE_STATE_LABEL && value.as_ref() == "pending")
|
||||
});
|
||||
assert_eq!(transition_queue.map(|metric| metric.value), Some(8.0));
|
||||
|
||||
let completed = metrics.iter().find(|m| {
|
||||
m.name == ILM_TASK_EVENTS_MD.get_full_metric_name()
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == RESULT_LABEL && value.as_ref() == "completed")
|
||||
});
|
||||
assert_eq!(completed.map(|metric| metric.value), Some(7.0));
|
||||
|
||||
let backpressure = metrics.iter().find(|m| {
|
||||
m.name == ILM_QUEUE_BACKPRESSURE_MD.get_full_metric_name()
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == REASON_LABEL && value.as_ref() == "queue_full")
|
||||
});
|
||||
assert_eq!(backpressure.map(|metric| metric.value), Some(2.0));
|
||||
|
||||
let version_detail = metrics.iter().find(|m| {
|
||||
m.name == ILM_VERSIONS_SCANNED_BY_SERVER_MD.get_full_metric_name()
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == SERVER_LABEL && value.as_ref() == "node1:9000")
|
||||
&& m.labels
|
||||
.iter()
|
||||
.any(|(name, value)| *name == SOURCE_LABEL && value.as_ref() == "lifecycle")
|
||||
});
|
||||
assert_eq!(version_detail.map(|metric| metric.value), Some(1000000.0));
|
||||
|
||||
let transition_active = metrics.iter().find(|m| {
|
||||
m.name == ILM_ACTION_TASKS_MD.get_full_metric_name()
|
||||
&& m.labels
|
||||
|
||||
@@ -59,9 +59,12 @@ pub use cluster_iam::{IamStats, collect_iam_metrics};
|
||||
pub use cluster_usage::{BucketUsageStats, ClusterUsageStats, collect_bucket_usage_metrics, collect_cluster_usage_metrics};
|
||||
pub use compression::{CompressionClusterStats, collect_compression_cluster_metrics};
|
||||
pub use dial9::{Dial9Stats, collect_current_dial9_metrics, collect_dial9_metrics, is_dial9_enabled};
|
||||
pub(crate) use ilm::{IlmActionTaskStats, IlmRuntimeStats, collect_ilm_runtime_metrics};
|
||||
pub(crate) use ilm::{
|
||||
IlmActionTaskStats, IlmBackpressureStats, IlmQueueTaskStats, IlmRuntimeStats, IlmTaskEventStats, collect_ilm_runtime_metrics,
|
||||
};
|
||||
pub use ilm::{IlmStats, collect_ilm_metrics};
|
||||
pub use node::{DiskStats, collect_node_metrics};
|
||||
pub(crate) use notification::collect_notification_runtime_metrics;
|
||||
pub use notification::{NotificationStats, collect_notification_metrics};
|
||||
pub(crate) use notification_target::{NotificationTargetRuntimeStats, collect_notification_target_runtime_metrics};
|
||||
pub use notification_target::{NotificationTargetStats, collect_notification_target_metrics};
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user