Compare commits

..

14 Commits

Author SHA1 Message Date
Quentin Dufour 85aca61860 [SKIP CI] WIP merkle todo encapsulation 2025-05-02 08:18:46 +02:00
Quentin Dufour 46ebfdba66 set config 2025-05-01 16:30:09 +02:00
Quentin Dufour ee8fa687ad add permits to merkle worker 2025-05-01 13:53:38 +02:00
Quentin Dufour fa457328c8 [SKIP CI] add forget semaphore, add them back not yet implemented 2025-05-01 10:05:04 +02:00
Quentin Dufour f34558af07 add a rpc in-flight limiter 2025-05-01 08:56:57 +02:00
Quentin Dufour d78e5f8a1b fix logic 2025-04-30 15:53:12 +02:00
Quentin Dufour 3172f875ae pass config 2025-04-30 14:41:16 +02:00
Quentin Dufour 11a6417d11 try to better track queue len evol 2025-04-30 09:01:50 +02:00
Quentin Dufour b0a9e007bd try another approach to backpressure 2025-04-30 08:49:44 +02:00
Quentin Dufour 904548d1d1 allow up to 30sec 2025-04-30 08:38:28 +02:00
Quentin Dufour 6cc79bc696 react slower 2025-04-30 08:37:35 +02:00
Quentin Dufour 60b3d28f93 add an opentelemetry metric 2025-04-30 07:24:20 +02:00
Quentin Dufour 7fddf0af9c first implementation 2025-04-29 10:50:47 +02:00
Quentin Dufour 78882f4040 add a backpressure system 2025-04-29 09:38:47 +02:00
87 changed files with 825 additions and 2110 deletions
-8
View File
@@ -1,6 +1,3 @@
labels:
nix: "enabled"
when:
event:
- push
@@ -31,11 +28,6 @@ steps:
commands:
- nix-build -j4 --attr flakePackages.tests-sqlite
- name: unit + func tests (fjall)
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --attr flakePackages.tests-fjall
- name: integration tests
image: nixpkgs/nix:nixos-22.05
commands:
-3
View File
@@ -1,6 +1,3 @@
labels:
nix: "enabled"
when:
event:
- deployment
-3
View File
@@ -1,6 +1,3 @@
labels:
nix: "enabled"
when:
event:
- deployment
Generated
+68 -234
View File
@@ -687,12 +687,6 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d2c12f985c78475a6b8d629afd0c360260ef34cfef52efccdcfd31972f81c2e"
[[package]]
name = "byteview"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
[[package]]
name = "cc"
version = "1.2.16"
@@ -804,12 +798,6 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
[[package]]
name = "compare"
version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7"
[[package]]
name = "core-foundation"
version = "0.9.4"
@@ -879,18 +867,9 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
version = "0.5.15"
version = "0.5.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471"
dependencies = [
"crossbeam-utils",
]
@@ -904,16 +883,6 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-skiplist"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
@@ -985,20 +954,6 @@ dependencies = [
"num_cpus",
]
[[package]]
name = "dashmap"
version = "6.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
dependencies = [
"cfg-if",
"crossbeam-utils",
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core 0.9.10",
]
[[package]]
name = "deranged"
version = "0.4.0"
@@ -1041,12 +996,6 @@ dependencies = [
"syn 2.0.100",
]
[[package]]
name = "double-ended-peekable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
[[package]]
name = "dyn-clone"
version = "1.0.19"
@@ -1068,18 +1017,6 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "enum_dispatch"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd"
dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.100",
]
[[package]]
name = "env_logger"
version = "0.10.2"
@@ -1147,23 +1084,6 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "fjall"
version = "2.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc"
dependencies = [
"byteorder",
"byteview",
"dashmap 6.1.0",
"log",
"lsm-tree",
"path-absolutize",
"std-semaphore",
"tempfile",
"xxhash-rust",
]
[[package]]
name = "fnv"
version = "1.0.7"
@@ -1280,7 +1200,7 @@ dependencies = [
[[package]]
name = "garage"
version = "1.3.0"
version = "1.1.0"
dependencies = [
"assert-json-diff",
"async-trait",
@@ -1330,13 +1250,12 @@ dependencies = [
"timeago",
"tokio",
"tracing",
"tracing-journald",
"tracing-subscriber",
]
[[package]]
name = "garage_api_admin"
version = "1.3.0"
version = "1.1.0"
dependencies = [
"argon2",
"async-trait",
@@ -1362,7 +1281,7 @@ dependencies = [
[[package]]
name = "garage_api_common"
version = "1.3.0"
version = "1.1.0"
dependencies = [
"base64 0.21.7",
"bytes",
@@ -1381,6 +1300,7 @@ dependencies = [
"http-body-util",
"hyper 1.6.0",
"hyper-util",
"idna 0.5.0",
"md-5",
"nom",
"opentelemetry",
@@ -1396,7 +1316,7 @@ dependencies = [
[[package]]
name = "garage_api_k2v"
version = "1.3.0"
version = "1.1.0"
dependencies = [
"base64 0.21.7",
"err-derive",
@@ -1419,7 +1339,7 @@ dependencies = [
[[package]]
name = "garage_api_s3"
version = "1.3.0"
version = "1.1.0"
dependencies = [
"aes-gcm",
"async-compression",
@@ -1464,7 +1384,7 @@ dependencies = [
[[package]]
name = "garage_block"
version = "1.3.0"
version = "1.1.0"
dependencies = [
"arc-swap",
"async-compression",
@@ -1489,13 +1409,11 @@ dependencies = [
[[package]]
name = "garage_db"
version = "1.3.0"
version = "1.1.0"
dependencies = [
"err-derive",
"fjall",
"heed",
"mktemp",
"parking_lot 0.12.3",
"r2d2",
"r2d2_sqlite",
"rusqlite",
@@ -1504,7 +1422,7 @@ dependencies = [
[[package]]
name = "garage_model"
version = "1.3.0"
version = "1.1.0"
dependencies = [
"async-trait",
"base64 0.21.7",
@@ -1531,7 +1449,7 @@ dependencies = [
[[package]]
name = "garage_net"
version = "1.3.0"
version = "1.1.0"
dependencies = [
"arc-swap",
"bytes",
@@ -1552,11 +1470,12 @@ dependencies = [
"tokio",
"tokio-stream",
"tokio-util 0.7.14",
"tracing",
]
[[package]]
name = "garage_rpc"
version = "1.3.0"
version = "1.1.0"
dependencies = [
"arc-swap",
"async-trait",
@@ -1588,7 +1507,7 @@ dependencies = [
[[package]]
name = "garage_table"
version = "1.3.0"
version = "1.1.0"
dependencies = [
"arc-swap",
"async-trait",
@@ -1609,7 +1528,7 @@ dependencies = [
[[package]]
name = "garage_util"
version = "1.3.0"
version = "1.1.0"
dependencies = [
"arc-swap",
"async-trait",
@@ -1641,7 +1560,7 @@ dependencies = [
[[package]]
name = "garage_web"
version = "1.3.0"
version = "1.1.0"
dependencies = [
"err-derive",
"garage_api_common",
@@ -1737,12 +1656,6 @@ dependencies = [
"syn 2.0.100",
]
[[package]]
name = "guardian"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f"
[[package]]
name = "h2"
version = "0.3.26"
@@ -1810,11 +1723,11 @@ dependencies = [
[[package]]
name = "hashlink"
version = "0.10.0"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
dependencies = [
"hashbrown 0.15.2",
"hashbrown 0.14.5",
]
[[package]]
@@ -2258,6 +2171,16 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "idna"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
dependencies = [
"unicode-bidi",
"unicode-normalization",
]
[[package]]
name = "idna"
version = "1.0.3"
@@ -2317,15 +2240,6 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "interval-heap"
version = "0.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6"
dependencies = [
"compare",
]
[[package]]
name = "ipnet"
version = "2.11.0"
@@ -2619,9 +2533,9 @@ dependencies = [
[[package]]
name = "libsqlite3-sys"
version = "0.35.0"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f"
checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"
dependencies = [
"cc",
"pkg-config",
@@ -2682,36 +2596,6 @@ dependencies = [
"hashbrown 0.15.2",
]
[[package]]
name = "lsm-tree"
version = "2.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab73c02eadb3dc12c0024e5b61d6284e6d59064e67e74fbad77856caa56f62c7"
dependencies = [
"byteorder",
"crossbeam-skiplist",
"double-ended-peekable",
"enum_dispatch",
"guardian",
"interval-heap",
"log",
"lz4_flex",
"path-absolutize",
"quick_cache",
"rustc-hash",
"self_cell",
"tempfile",
"value-log",
"varint-rs",
"xxhash-rust",
]
[[package]]
name = "lz4_flex"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
[[package]]
name = "matchers"
version = "0.1.0"
@@ -2966,7 +2850,7 @@ checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8"
dependencies = [
"async-trait",
"crossbeam-channel",
"dashmap 4.0.2",
"dashmap",
"fnv",
"futures-channel",
"futures-executor",
@@ -3127,24 +3011,6 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "path-absolutize"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5"
dependencies = [
"path-dedot",
]
[[package]]
name = "path-dedot"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397"
dependencies = [
"once_cell",
]
[[package]]
name = "pem"
version = "3.0.5"
@@ -3440,16 +3306,6 @@ dependencies = [
"serde",
]
[[package]]
name = "quick_cache"
version = "0.6.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786"
dependencies = [
"equivalent",
"hashbrown 0.15.2",
]
[[package]]
name = "quote"
version = "1.0.40"
@@ -3478,9 +3334,9 @@ dependencies = [
[[package]]
name = "r2d2_sqlite"
version = "0.31.0"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63417e83dc891797eea3ad379f52a5986da4bca0d6ef28baf4d14034dd111b0c"
checksum = "6a982edf65c129796dba72f8775b292ef482b40d035e827a9825b3bc07ccc5f2"
dependencies = [
"r2d2",
"rusqlite",
@@ -3669,9 +3525,9 @@ checksum = "3cd14fd5e3b777a7422cca79358c57a8f6e3a703d9ac187448d0daf220c2407f"
[[package]]
name = "rusqlite"
version = "0.37.0"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f"
checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae"
dependencies = [
"bitflags 2.9.0",
"fallible-iterator",
@@ -3687,12 +3543,6 @@ version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
[[package]]
name = "rustc-hash"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
[[package]]
name = "rustc_version"
version = "0.4.1"
@@ -3935,12 +3785,6 @@ dependencies = [
"libc",
]
[[package]]
name = "self_cell"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749"
[[package]]
name = "semver"
version = "1.0.26"
@@ -4154,12 +3998,6 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "std-semaphore"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e"
[[package]]
name = "strsim"
version = "0.11.1"
@@ -4415,6 +4253,21 @@ dependencies = [
"zerovec",
]
[[package]]
name = "tinyvec"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.44.1"
@@ -4688,17 +4541,6 @@ dependencies = [
"tracing",
]
[[package]]
name = "tracing-journald"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc0b4143302cf1022dac868d521e36e8b27691f72c84b3311750d5188ebba657"
dependencies = [
"libc",
"tracing-core",
"tracing-subscriber",
]
[[package]]
name = "tracing-log"
version = "0.2.0"
@@ -4746,12 +4588,27 @@ version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
[[package]]
name = "unicode-bidi"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
[[package]]
name = "unicode-ident"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
[[package]]
name = "unicode-normalization"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
dependencies = [
"tinyvec",
]
[[package]]
name = "unicode-segmentation"
version = "1.12.0"
@@ -4799,7 +4656,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
dependencies = [
"form_urlencoded",
"idna",
"idna 1.0.3",
"percent-encoding",
]
@@ -4837,29 +4694,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
[[package]]
name = "value-log"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c"
dependencies = [
"byteorder",
"byteview",
"interval-heap",
"log",
"path-absolutize",
"rustc-hash",
"tempfile",
"varint-rs",
"xxhash-rust",
]
[[package]]
name = "varint-rs"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]]
name = "vcpkg"
version = "0.2.15"
+15 -17
View File
@@ -24,18 +24,18 @@ default-members = ["src/garage"]
# Internal Garage crates
format_table = { version = "0.1.1", path = "src/format-table" }
garage_api_common = { version = "1.3.0", path = "src/api/common" }
garage_api_admin = { version = "1.3.0", path = "src/api/admin" }
garage_api_s3 = { version = "1.3.0", path = "src/api/s3" }
garage_api_k2v = { version = "1.3.0", path = "src/api/k2v" }
garage_block = { version = "1.3.0", path = "src/block" }
garage_db = { version = "1.3.0", path = "src/db", default-features = false }
garage_model = { version = "1.3.0", path = "src/model", default-features = false }
garage_net = { version = "1.3.0", path = "src/net" }
garage_rpc = { version = "1.3.0", path = "src/rpc" }
garage_table = { version = "1.3.0", path = "src/table" }
garage_util = { version = "1.3.0", path = "src/util" }
garage_web = { version = "1.3.0", path = "src/web" }
garage_api_common = { version = "1.1.0", path = "src/api/common" }
garage_api_admin = { version = "1.1.0", path = "src/api/admin" }
garage_api_s3 = { version = "1.1.0", path = "src/api/s3" }
garage_api_k2v = { version = "1.1.0", path = "src/api/k2v" }
garage_block = { version = "1.1.0", path = "src/block" }
garage_db = { version = "1.1.0", path = "src/db", default-features = false }
garage_model = { version = "1.1.0", path = "src/model", default-features = false }
garage_net = { version = "1.1.0", path = "src/net" }
garage_rpc = { version = "1.1.0", path = "src/rpc" }
garage_table = { version = "1.1.0", path = "src/table" }
garage_util = { version = "1.1.0", path = "src/util" }
garage_web = { version = "1.1.0", path = "src/web" }
k2v-client = { version = "0.0.4", path = "src/k2v-client" }
# External crates from crates.io
@@ -58,6 +58,7 @@ git-version = "0.3.4"
hex = "0.4"
hexdump = "0.1"
hmac = "0.12"
idna = "0.5"
itertools = "0.12"
ipnet = "2.9.0"
lazy_static = "1.4"
@@ -65,7 +66,6 @@ md-5 = "0.10"
mktemp = "0.5"
nix = { version = "0.29", default-features = false, features = ["fs"] }
nom = "7.1"
parking_lot = "0.12"
parse_duration = "2.1"
pin-project = "1.0.12"
pnet_datalink = "0.34"
@@ -84,14 +84,12 @@ pretty_env_logger = "0.5"
structopt = { version = "0.3", default-features = false }
syslog-tracing = "0.3"
tracing = "0.1"
tracing-journald = "0.3.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
heed = { version = "0.11", default-features = false, features = ["lmdb"] }
rusqlite = "0.37"
rusqlite = "0.31.0"
r2d2 = "0.8"
r2d2_sqlite = "0.31"
fjall = "2.4"
r2d2_sqlite = "0.24"
async-compression = { version = "0.4", features = ["tokio", "zstd"] }
zstd = { version = "0.13", default-features = false }
+4 -4
View File
@@ -12,7 +12,7 @@ In this section, we cover the following web applications:
| [Mastodon](#mastodon) | ✅ | Natively supported |
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
| [ejabberd](#ejabberd) | ✅ | `mod_s3_upload` |
| [Pixelfed](#pixelfed) | | Natively supported |
| [Pixelfed](#pixelfed) | | Not yet tested |
| [Pleroma](#pleroma) | ❓ | Not yet tested |
| [Lemmy](#lemmy) | ✅ | Supported with pict-rs |
| [Funkwhale](#funkwhale) | ❓ | Not yet tested |
@@ -191,10 +191,10 @@ garage key create peertube-key
Keep the Key ID and the Secret key in a pad, they will be needed later.
We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists).
We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
```bash
garage bucket create peertube-videos
garage bucket create peertube-playlists
garage bucket create peertube-playlist
```
Now we allow our key to read and write on these buckets:
@@ -253,7 +253,7 @@ object_storage:
proxify_private_files: false
streaming_playlists:
bucket_name: 'peertube-playlists'
bucket_name: 'peertube-playlist'
# Keep it empty for our example
prefix: ''
-46
View File
@@ -161,49 +161,3 @@ kopia repository validate-provider
You can then run all the standard kopia commands: `kopia snapshot create`, `kopia mount`...
Everything should work out-of-the-box.
## Plakar
Create your key and bucket on Garage server:
```bash
garage key create my-plakar-key
garage bucket create plakar-backups
garage bucket allow plakar-backups --read --write --key my-plakar-key
```
On Plakar server, add your Garage as a storage location:
```bash
plakar store add garageS3 s3://my-garage.tld/plakar-backups \
region=garage # Or as you've specified in garage.toml \
access_key=<Key ID from "garage key info my-plakar-key"> \
secret_access_key=<Secret key from "garage key info my-plakar-key">
```
Then create the repository.
```bash
plakar at @garageS3 create -plaintext # Unencrypted
# or
plakar at @garageS3 create #encrypted
```
If you encrypt your backups (Plakar default), you will need to define a strong passphrase. Do not forget to save your password safely. It will be needed to decrypt your backups.
After the repository has been created, check that everything works as expected (that might give an empty result as no file has been added yet, but no error message):
```bash
plakar at @garageS3 check
```
Now that everything is configure, you can use Garage as your backups storage. For instance sync it with a local backup storage:
```bash
$ plakar at ~/backups sync to @garageS3
```
Or list the S3 storage content:
```bash
$ plakar at @garageS3 ls
```
More information in Plakar documentation: https://www.plakar.io/docs/main/quickstart/
+12 -24
View File
@@ -8,18 +8,18 @@ have published Ansible roles. We list them and compare them below.
## Comparison of Ansible roles
| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster ansible-role-garage](#eddster-ansible-role-garage) |
|------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------|
| **Runtime** | Systemd | Docker | Systemd |
| **Target OS** | Any Linux | Any Linux | Any Linux |
| **Architecture** | amd64, arm64, i686 | amd64, arm64 | arm64, arm, 386, amd64 |
| **Additional software** | None | Traefik | Ngnix and Keepalived (optional) |
| **Automatic node connection** | ❌ | ✅ | ✅ |
| **Layout management** | ❌ | ✅ | ✅ |
| **Manage buckets & keys** | ❌ | ✅ (basic) | ✅ |
| **Allow custom Garage config** | ✅ | ❌ | ❌ |
| **Facilitate Garage upgrades** | ✅ | ❌ | ✅ |
| **Multiple instances on one host** | ✅ | ✅ | ❌ |
| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) |
|------------------------------------|---------------------------------------------|---------------------------------------------------------------|
| **Runtime** | Systemd | Docker |
| **Target OS** | Any Linux | Any Linux |
| **Architecture** | amd64, arm64, i686 | amd64, arm64 |
| **Additional software** | None | Traefik |
| **Automatic node connection** | ❌ | ✅ |
| **Layout management** | ❌ | ✅ |
| **Manage buckets & keys** | ❌ | ✅ (basic) |
| **Allow custom Garage config** | ✅ | ❌ |
| **Facilitate Garage upgrades** | ✅ | ❌ |
| **Multiple instances on one host** | ✅ | ✅ |
## zorun/ansible-role-garage
@@ -49,15 +49,3 @@ structured DNS names, etc).
As a result, this role makes it easier to start with Garage on Ansible,
but is less flexible.
## eddster2309/ansible-role-garage
[Source code](https://github.com/eddster2309/ansible-role-garage), [Ansible galaxy](https://galaxy.ansible.com/ui/standalone/roles/eddster2309/garage/)
This role is a opinionated but customisable role using the official Garage
static binaries and only requires Systemd. As such it should work on any
Linux based host. It includes all the nesscary configuration to
automatically setup a clustered Garage deployment. Most Garage
configuration options are exposed through Ansible variables so while you
can't provide a custom config you can get very close. It can optionally
installed a HA nginx deployment with Keepalived.
-7
View File
@@ -26,13 +26,6 @@ Or deploy with custom values:
helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml
```
If you want to manage the CustomRessourceDefinition used by garage for its `kubernetes_discovery` outside of the helm chart, add `garage.kubernetesSkipCrd: true` to your custom values and use the kustomization before deploying the helm chart:
```bash
kubectl apply -k ../k8s/crd
helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml
```
After deploying, cluster layout must be configured manually as described in [Creating a cluster layout](@/documentation/quick-start/_index.md#creating-a-cluster-layout). Use the following command to access garage CLI:
```bash
+5 -5
View File
@@ -96,14 +96,14 @@ to store 2 TB of data in total.
## Get a Docker image
Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
We encourage you to use a fixed tag (eg. `v1.3.0`) and not the `latest` tag.
For this example, we will use the latest published version at the time of the writing which is `v1.3.0` but it's up to you
We encourage you to use a fixed tag (eg. `v1.1.0`) and not the `latest` tag.
For this example, we will use the latest published version at the time of the writing which is `v1.1.0` but it's up to you
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
For example:
```
sudo docker pull dxflrs/garage:v1.3.0
sudo docker pull dxflrs/garage:v1.1.0
```
## Deploying and configuring Garage
@@ -171,7 +171,7 @@ docker run \
-v /etc/garage.toml:/etc/garage.toml \
-v /var/lib/garage/meta:/var/lib/garage/meta \
-v /var/lib/garage/data:/var/lib/garage/data \
dxflrs/garage:v1.3.0
dxflrs/garage:v1.1.0
```
With this command line, Garage should be started automatically at each boot.
@@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
version: "3"
services:
garage:
image: dxflrs/garage:v1.3.0
image: dxflrs/garage:v1.1.0
network_mode: "host"
restart: unless-stopped
volumes:
-1
View File
@@ -28,7 +28,6 @@ StateDirectory=garage
DynamicUser=true
ProtectHome=true
NoNewPrivileges=true
LimitNOFILE=42000
[Install]
WantedBy=multi-user.target
+5 -6
View File
@@ -132,7 +132,7 @@ docker run \
-v /path/to/garage.toml:/etc/garage.toml \
-v /path/to/garage/meta:/var/lib/garage/meta \
-v /path/to/garage/data:/var/lib/garage/data \
dxflrs/garage:v1.3.0
dxflrs/garage:v1.1.0
```
Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903`
@@ -182,12 +182,11 @@ ID Hostname Address Tag Zone Capacit
## Creating a cluster layout
Creating a cluster layout for a Garage deployment means informing Garage
of the disk space available on each node of the cluster, `-c`,
as well as the name of the zone (e.g. datacenter), `-z`, each machine is located in.
of the disk space available on each node of the cluster
as well as the zone (e.g. datacenter) each machine is located in.
For our test deployment, we are have only one node with zone named `dc1` and a
capacity of `1G`, though the capacity is ignored for a single node deployment
and can be changed later when adding new nodes.
For our test deployment, we are using only one node. The way in which we configure
it does not matter, you can simply write:
```bash
garage layout assign -z dc1 -c 1G <node_id>
+9 -57
View File
@@ -24,7 +24,6 @@ db_engine = "lmdb"
block_size = "1M"
block_ram_buffer_max = "256MiB"
block_max_concurrent_reads = 16
lmdb_map_size = "1T"
@@ -47,7 +46,6 @@ bootstrap_peers = [
"212fd62eeaca72c122b45a7f4fa0f55e012aa5e24ac384a72a3016413fa724ff@[fc00:F::1]:3901",
]
allow_punycode = false
[consul_discovery]
api = "catalog"
@@ -94,31 +92,29 @@ The following gives details about each available configuration option.
[Environment variables](#env_variables).
Top-level configuration options, in alphabetical order:
[`allow_punycode`](#allow_punycode),
Top-level configuration options:
[`allow_world_readable_secrets`](#allow_world_readable_secrets),
[`block_max_concurrent_reads`](`block_max_concurrent_reads),
[`block_ram_buffer_max`](#block_ram_buffer_max),
[`block_size`](#block_size),
[`bootstrap_peers`](#bootstrap_peers),
[`compression_level`](#compression_level),
[`consistency_mode`](#consistency_mode),
[`data_dir`](#data_dir),
[`data_fsync`](#data_fsync),
[`db_engine`](#db_engine),
[`disable_scrub`](#disable_scrub),
[`use_local_tz`](#use_local_tz),
[`lmdb_map_size`](#lmdb_map_size),
[`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval),
[`metadata_dir`](#metadata_dir),
[`metadata_fsync`](#metadata_fsync),
[`metadata_snapshots_dir`](#metadata_snapshots_dir),
[`replication_factor`](#replication_factor),
[`consistency_mode`](#consistency_mode),
[`rpc_bind_addr`](#rpc_bind_addr),
[`rpc_bind_outgoing`](#rpc_bind_outgoing),
[`rpc_public_addr`](#rpc_public_addr),
[`rpc_public_addr_subnet`](#rpc_public_addr_subnet)
[`rpc_secret`/`rpc_secret_file`](#rpc_secret),
[`use_local_tz`](#use_local_tz).
[`rpc_secret`/`rpc_secret_file`](#rpc_secret).
The `[consul_discovery]` section:
[`api`](#consul_api),
@@ -155,17 +151,13 @@ The `[admin]` section:
### Environment variables {#env_variables}
The following configuration parameters must be specified as environment variables,
they do not exist in the configuration file:
The following configuration parameter must be specified as an environment
variable, it does not exist in the configuration file:
- `GARAGE_LOG_TO_SYSLOG` (since `v0.9.4`): set this to `1` or `true` to make the
Garage daemon send its logs to `syslog` (using the libc `syslog` function)
instead of printing to stderr.
- `GARAGE_LOG_TO_JOURNALD` (since `v1.2.0`): set this to `1` or `true` to make the
Garage daemon send its logs to `journald` (using the native protocol of `systemd-journald`)
instead of printing to stderr.
The following environment variables can be used to override the corresponding
values in the configuration file:
@@ -177,7 +169,7 @@ values in the configuration file:
### Top-level configuration options
#### `replication_factor` (since `v1.0.0`) {#replication_factor}
#### `replication_factor` {#replication_factor}
The replication factor can be any positive integer smaller or equal the node count in your cluster.
The chosen replication factor has a big impact on the cluster's failure tolerancy and performance characteristics.
@@ -225,7 +217,7 @@ is in progress. In theory, no data should be lost as rebalancing is a
routine operation for Garage, although we cannot guarantee you that everything
will go right in such an extreme scenario.
#### `consistency_mode` (since `v1.0.0`) {#consistency_mode}
#### `consistency_mode` {#consistency_mode}
The consistency mode setting determines the read and write behaviour of your cluster.
@@ -335,7 +327,6 @@ Since `v0.8.0`, Garage can use alternative storage backends as follows:
| --------- | ----------------- | ------------- |
| [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
| [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
| [Fjall](https://github.com/fjall-rs/fjall) (**experimental support** since `v1.3.0`) | `"fjall"` | `<metadata_dir>/db.fjall/` |
| [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` |
Sled was supported until Garage v0.9.x, and was removed in Garage v1.0.
@@ -372,14 +363,6 @@ LMDB works very well, but is known to have the following limitations:
so it is not the best choice for high-performance storage clusters,
but it should work fine in many cases.
- Fjall: a storage engine based on LSM trees, which theoretically allow for
higher write throughput than other storage engines that are based on B-trees.
Using Fjall could potentially improve Garage's performance significantly in
write-heavy workloads. **Support for Fjall is experimental at this point**,
we have added it to Garage for evaluation purposes only. **Do not use it for
production-critical workloads.**
It is possible to convert Garage's metadata directory from one format to another
using the `garage convert-db` command, which should be used as follows:
@@ -417,7 +400,6 @@ Here is how this option impacts the different database engines:
|----------|------------------------------------|-------------------------------|
| Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` |
| LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` |
| Fjall | default options | not supported |
Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).
@@ -524,29 +506,6 @@ node.
The default value is 256MiB.
#### `block_max_concurrent_reads` (since `v1.3.0` / `v2.1.0`) {#block_max_concurrent_reads}
The maximum number of blocks (individual files in the data directory) open
simultaneously for reading.
Reducing this number does not limit the number of data blocks that can be
transferred through the network simultaneously. This mechanism was just added
as a backpressure mechanism for HDD read speed: it helps avoid a situation
where too many requests are coming in and Garage is reading too many block
files simultaneously, thus not making timely progress on any of the reads.
When a request to read a data block comes in through the network, the requests
awaits for one of the `block_max_concurrent_reads` slots to be available
(internally implemented using a Semaphore object). Once it acquired a read
slot, it reads the entire block file to RAM and frees the slot as soon as the
block file is finished reading. Only after the slot is released will the
block's data start being transferred over the network. If the request fails to
acquire a reading slot wihtin 15 seconds, it fails with a timeout error.
Timeout events can be monitored through the `block_read_semaphore_timeouts`
metric in Prometheus: a non-zero number of such events indicates an I/O
bottleneck on HDD read speed.
#### `lmdb_map_size` {#lmdb_map_size}
This parameters can be used to set the map size used by LMDB,
@@ -645,7 +604,7 @@ be obtained by running `garage node id` and then included directly in the
key will be returned by `garage node id` and you will have to add the IP
yourself.
#### `allow_world_readable_secrets` or `GARAGE_ALLOW_WORLD_READABLE_SECRETS` (env) {#allow_world_readable_secrets}
### `allow_world_readable_secrets` or `GARAGE_ALLOW_WORLD_READABLE_SECRETS` (env) {#allow_world_readable_secrets}
Garage checks the permissions of your secret files to make sure they're not
world-readable. In some cases, the check might fail and consider your files as
@@ -657,13 +616,6 @@ permission verification.
Alternatively, you can set the `GARAGE_ALLOW_WORLD_READABLE_SECRETS`
environment variable to `true` to bypass the permissions check.
#### `allow_punycode` {#allow_punycode}
Allow creating buckets with names containing punycode. When used for buckets served
as websites, this allows using almost any unicode character in the domain name.
Default to `false`.
### The `[consul_discovery]` section
Garage supports discovering other nodes of the cluster using Consul. For this
@@ -23,6 +23,7 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the
- 2022-05-25 - Many Ceph S3 endpoints are not documented but implemented. Following a notification from the Ceph community, we added them.
## High-level features
| Feature | Garage | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) |
@@ -33,7 +34,6 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the
| [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) | ✅ Implemented | ❌| ✅| ✅ | ✅ |
| [Presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html) | ✅ Implemented | ❌| ✅ | ✅ | ✅(❓) |
| [SSE-C encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) | ✅ Implemented | ❓ | ✅ | ❌ | ✅ |
| [Bucket versioning](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html) | ❌ Missing | ✅ | ✅ | ❌ | ✅ |
*Note:* OpenIO does not says if it supports presigned URLs. Because it is part
of signature v4 and they claim they support it without additional precisions,
+1 -1
View File
@@ -70,7 +70,7 @@ Example response body:
```json
{
"node": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df",
"garageVersion": "v1.3.0",
"garageVersion": "v1.1.0",
"garageFeatures": [
"k2v",
"lmdb",
-3
View File
@@ -53,9 +53,6 @@
tests-sqlite = testWith {
GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite";
};
tests-fjall = testWith {
GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall";
};
};
# ---- developpment shell, for making native builds only ----
+1 -2
View File
@@ -68,13 +68,12 @@ let
rootFeatures = if features != null then
features
else
([ "bundled-libs" "lmdb" "sqlite" "fjall" "k2v" ] ++ (lib.optionals release [
([ "bundled-libs" "lmdb" "sqlite" "k2v" ] ++ (lib.optionals release [
"consul-discovery"
"kubernetes-discovery"
"metrics"
"telemetry-otlp"
"syslog"
"journald"
]));
featuresStr = lib.concatStringsSep "," rootFeatures;
+3 -3
View File
@@ -2,8 +2,8 @@ apiVersion: v2
name: garage
description: S3-compatible object store for small self-hosted geo-distributed deployments
type: application
version: 0.7.2
appVersion: "v1.3.0"
version: 0.7.0
appVersion: "v1.1.0"
home: https://garagehq.deuxfleurs.fr/
icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg
@@ -15,4 +15,4 @@ keywords:
sources:
- https://git.deuxfleurs.fr/Deuxfleurs/garage.git
maintainers: []
maintainers: []
+1 -1
View File
@@ -1,6 +1,6 @@
# garage
![Version: 0.7.2](https://img.shields.io/badge/Version-0.7.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.3.0](https://img.shields.io/badge/AppVersion-v1.3.0-informational?style=flat-square)
![Version: 0.7.0](https://img.shields.io/badge/Version-0.7.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.1.0](https://img.shields.io/badge/AppVersion-v1.1.0-informational?style=flat-square)
S3-compatible object store for small self-hosted geo-distributed deployments
@@ -1,43 +0,0 @@
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: garagenodes.deuxfleurs.fr
spec:
conversion:
strategy: None
group: deuxfleurs.fr
names:
kind: GarageNode
listKind: GarageNodeList
plural: garagenodes
singular: garagenode
scope: Namespaced
versions:
- name: v1
schema:
openAPIV3Schema:
description: Auto-generated derived type for Node via `CustomResource`
properties:
spec:
properties:
address:
format: ip
type: string
hostname:
type: string
port:
format: uint16
minimum: 0
type: integer
required:
- address
- hostname
- port
type: object
required:
- spec
title: GarageNode
type: object
served: true
storage: true
subresources: {}
-5
View File
@@ -1,5 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- garagenodes.deuxfleurs.fr.yaml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_admin"
version = "1.3.0"
version = "1.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+3 -3
View File
@@ -277,7 +277,7 @@ pub async fn handle_create_bucket(
let helper = garage.locked_helper().await;
if let Some(ga) = &req.global_alias {
if !is_valid_bucket_name(ga, garage.config.allow_punycode) {
if !is_valid_bucket_name(ga) {
return Err(Error::bad_request(format!(
"{}: {}",
ga, INVALID_BUCKET_NAME_MESSAGE
@@ -292,7 +292,7 @@ pub async fn handle_create_bucket(
}
if let Some(la) = &req.local_alias {
if !is_valid_bucket_name(&la.alias, garage.config.allow_punycode) {
if !is_valid_bucket_name(&la.alias) {
return Err(Error::bad_request(format!(
"{}: {}",
la.alias, INVALID_BUCKET_NAME_MESSAGE
@@ -382,7 +382,7 @@ pub async fn handle_delete_bucket(
for ((key_id, alias), _, active) in state.local_aliases.items().iter() {
if *active {
helper
.purge_local_bucket_alias(bucket.id, key_id, alias)
.unset_local_bucket_alias(bucket.id, key_id, alias)
.await?;
}
}
+2 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_common"
version = "1.3.0"
version = "1.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@@ -28,6 +28,7 @@ err-derive.workspace = true
hex.workspace = true
hmac.workspace = true
md-5.workspace = true
idna.workspace = true
tracing.workspace = true
nom.workspace = true
pin-project.workspace = true
+11 -22
View File
@@ -58,12 +58,6 @@ pub trait ApiHandler: Send + Sync + 'static {
req: Request<IncomingBody>,
endpoint: Self::Endpoint,
) -> impl Future<Output = Result<Response<BoxBody<Self::Error>>, Self::Error>> + Send;
/// Returns the key id used to authenticate this request. The ID returned must be safe to
/// log.
fn key_id_from_request(&self, _req: &Request<IncomingBody>) -> Option<String> {
None
}
}
pub struct ApiServer<A: ApiHandler> {
@@ -148,20 +142,19 @@ impl<A: ApiHandler> ApiServer<A> {
) -> Result<Response<BoxBody<A::Error>>, http::Error> {
let uri = req.uri().clone();
let source = if let Ok(forwarded_for_ip_addr) =
if let Ok(forwarded_for_ip_addr) =
forwarded_headers::handle_forwarded_for_headers(req.headers())
{
format!("{forwarded_for_ip_addr} (via {addr})")
info!(
"{} (via {}) {} {}",
forwarded_for_ip_addr,
addr,
req.method(),
uri
);
} else {
format!("{addr}")
};
// we only do this to log the access key, so we can discard any error
let key = self
.api_handler
.key_id_from_request(&req)
.map(|k| format!("(key {k}) "))
.unwrap_or_default();
info!("{source} {key}{} {uri}", req.method());
info!("{} {} {}", addr, req.method(), uri);
}
debug!("{:?}", req);
let tracer = opentelemetry::global::tracer("garage");
@@ -350,11 +343,7 @@ where
while !*must_exit.borrow() {
let (stream, client_addr) = tokio::select! {
acc = listener.accept() => match acc {
Ok(r) => r,
Err(e) if e.kind() == std::io::ErrorKind::ConnectionAborted => continue,
Err(e) => return Err(e.into()),
},
acc = listener.accept() => acc?,
_ = must_exit.changed() => continue,
};
+2 -1
View File
@@ -8,6 +8,7 @@ use hyper::{
body::{Body, Bytes},
Request, Response,
};
use idna::domain_to_unicode;
use serde::{Deserialize, Serialize};
use garage_model::bucket_table::BucketParams;
@@ -96,7 +97,7 @@ pub fn authority_to_host(authority: &str) -> Result<String, Error> {
authority
))),
};
authority.map(|h| h.to_ascii_lowercase())
authority.map(|h| domain_to_unicode(h).0)
}
/// Extract the bucket name and the key name from an HTTP path and possibly a bucket provided in
+2 -2
View File
@@ -417,7 +417,7 @@ pub async fn verify_v4(
// ============ Authorization header, or X-Amz-* query params =========
pub struct Authorization {
pub key_id: String,
key_id: String,
scope: String,
signed_headers: String,
signature: String,
@@ -426,7 +426,7 @@ pub struct Authorization {
}
impl Authorization {
pub fn parse_header(headers: &HeaderMap) -> Result<Self, Error> {
fn parse_header(headers: &HeaderMap) -> Result<Self, Error> {
let authorization = headers
.get(AUTHORIZATION)
.ok_or_bad_request("Missing authorization header")?
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_k2v"
version = "1.3.0"
version = "1.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
-6
View File
@@ -176,12 +176,6 @@ impl ApiHandler for K2VApiServer {
Ok(resp_ok)
}
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
garage_api_common::signature::payload::Authorization::parse_header(req.headers())
.map(|auth| auth.key_id)
.ok()
}
}
impl ApiEndpoint for K2VApiEndpoint {
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_s3"
version = "1.3.0"
version = "1.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
-7
View File
@@ -226,7 +226,6 @@ impl ApiHandler for S3ApiServer {
Endpoint::DeleteBucket {} => handle_delete_bucket(ctx).await,
Endpoint::GetBucketLocation {} => handle_get_bucket_location(ctx),
Endpoint::GetBucketVersioning {} => handle_get_bucket_versioning(),
Endpoint::GetBucketAcl {} => handle_get_bucket_acl(ctx),
Endpoint::ListObjects {
delimiter,
encoding_type,
@@ -343,12 +342,6 @@ impl ApiHandler for S3ApiServer {
Ok(resp_ok)
}
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
garage_api_common::signature::payload::Authorization::parse_header(req.headers())
.map(|auth| auth.key_id)
.ok()
}
}
impl ApiEndpoint for S3ApiEndpoint {
+4 -62
View File
@@ -5,7 +5,7 @@ use hyper::{Request, Response, StatusCode};
use garage_model::bucket_alias_table::*;
use garage_model::bucket_table::Bucket;
use garage_model::garage::Garage;
use garage_model::key_table::{Key, KeyParams};
use garage_model::key_table::Key;
use garage_model::permission::BucketKeyPerm;
use garage_table::util::*;
use garage_util::crdt::*;
@@ -44,55 +44,6 @@ pub fn handle_get_bucket_versioning() -> Result<Response<ResBody>, Error> {
.body(string_body(xml))?)
}
pub fn handle_get_bucket_acl(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
let ReqCtx {
bucket_id, api_key, ..
} = ctx;
let key_p = api_key.params().ok_or_internal_error(
"Key should not be in deleted state at this point (in handle_get_bucket_acl)",
)?;
let mut grants: Vec<s3_xml::Grant> = vec![];
let kp = api_key.bucket_permissions(&bucket_id);
if kp.allow_owner {
grants.push(s3_xml::Grant {
grantee: create_grantee(&key_p, &api_key),
permission: s3_xml::Value("FULL_CONTROL".to_string()),
});
} else {
if kp.allow_read {
grants.push(s3_xml::Grant {
grantee: create_grantee(&key_p, &api_key),
permission: s3_xml::Value("READ".to_string()),
});
grants.push(s3_xml::Grant {
grantee: create_grantee(&key_p, &api_key),
permission: s3_xml::Value("READ_ACP".to_string()),
});
}
if kp.allow_write {
grants.push(s3_xml::Grant {
grantee: create_grantee(&key_p, &api_key),
permission: s3_xml::Value("WRITE".to_string()),
});
}
}
let access_control_policy = s3_xml::AccessControlPolicy {
xmlns: (),
owner: None,
acl: s3_xml::AccessControlList { entries: grants },
};
let xml = s3_xml::to_xml_with_header(&access_control_policy)?;
trace!("xml: {}", xml);
Ok(Response::builder()
.header("Content-Type", "application/xml")
.body(string_body(xml))?)
}
pub async fn handle_list_buckets(
garage: &Garage,
api_key: &Key,
@@ -221,7 +172,7 @@ pub async fn handle_create_bucket(
}
// Create the bucket!
if !is_valid_bucket_name(&bucket_name, garage.config.allow_punycode) {
if !is_valid_bucket_name(&bucket_name) {
return Err(Error::bad_request(format!(
"{}: {}",
bucket_name, INVALID_BUCKET_NAME_MESSAGE
@@ -290,11 +241,11 @@ pub async fn handle_delete_bucket(ctx: ReqCtx) -> Result<Response<ResBody>, Erro
// 1. delete bucket alias
if is_local_alias {
helper
.purge_local_bucket_alias(*bucket_id, &api_key.key_id, bucket_name)
.unset_local_bucket_alias(*bucket_id, &api_key.key_id, bucket_name)
.await?;
} else {
helper
.purge_global_bucket_alias(*bucket_id, bucket_name)
.unset_global_bucket_alias(*bucket_id, bucket_name)
.await?;
}
@@ -360,15 +311,6 @@ fn parse_create_bucket_xml(xml_bytes: &[u8]) -> Option<Option<String>> {
Some(ret)
}
fn create_grantee(key_params: &KeyParams, api_key: &Key) -> s3_xml::Grantee {
s3_xml::Grantee {
xmlns_xsi: (),
typ: "CanonicalUser".to_string(),
display_name: Some(s3_xml::Value(key_params.name.get().to_string())),
id: Some(s3_xml::Value(api_key.key_id.to_string())),
}
}
#[cfg(test)]
mod tests {
use super::*;
+29 -62
View File
@@ -26,10 +26,9 @@ use garage_api_common::signature::checksum::*;
use crate::api_server::{ReqBody, ResBody};
use crate::encryption::EncryptionParams;
use crate::error::*;
use crate::get::{check_version_not_deleted, full_object_byte_stream, PreconditionHeaders};
use crate::get::{full_object_byte_stream, PreconditionHeaders};
use crate::multipart;
use crate::put::{extract_metadata_headers, save_stream, ChecksumMode, SaveStreamResult};
use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION;
use crate::xml::{self as s3_xml, xmlns_tag};
pub const X_AMZ_COPY_SOURCE_IF_MATCH: HeaderName =
@@ -85,18 +84,7 @@ pub async fn handle_copy(
Some(v) if v == hyper::header::HeaderValue::from_static("REPLACE") => {
extract_metadata_headers(req.headers())?
}
_ => {
// The x-amz-website-redirect-location header is not copied, instead
// it is replaced by the value from the request (or removed if no
// value was specified)
let is_redirect =
|(key, _): &(String, String)| key == X_AMZ_WEBSITE_REDIRECT_LOCATION.as_str();
let mut headers: Vec<_> = source_object_meta_inner.headers.clone();
headers.retain(|h| !is_redirect(h));
let new_headers = extract_metadata_headers(req.headers())?;
headers.extend(new_headers.into_iter().filter(is_redirect));
headers
}
_ => source_object_meta_inner.into_owned().headers,
},
checksum: source_checksum,
};
@@ -237,7 +225,6 @@ async fn handle_copy_metaonly(
.get(&source_version.uuid, &EmptyKey)
.await?;
let source_version = source_version.ok_or(Error::NoSuchKey)?;
check_version_not_deleted(&source_version)?;
// Write an "uploading" marker in Object table
// This holds a reference to the object in the Version table
@@ -429,7 +416,6 @@ pub async fn handle_upload_part_copy(
.get(&source_object_version.uuid, &EmptyKey)
.await?
.ok_or(Error::NoSuchKey)?;
check_version_not_deleted(&source_version)?;
// We want to reuse blocks from the source version as much as possible.
// However, we still need to get the data from these blocks
@@ -561,7 +547,6 @@ pub async fn handle_upload_part_copy(
let mut current_offset = 0;
let mut next_block = defragmenter.next().await?;
let mut blocks_to_dup = dest_version.clone();
// TODO this could be optimized similarly to read_and_put_blocks
// low priority because uploadpartcopy is rarely used
@@ -591,7 +576,8 @@ pub async fn handle_upload_part_copy(
.unwrap()?;
checksummer = checksummer_updated;
let (version_block_key, version_block) = (
dest_version.blocks.clear();
dest_version.blocks.put(
VersionBlockKey {
part_number,
offset: current_offset,
@@ -603,56 +589,37 @@ pub async fn handle_upload_part_copy(
);
current_offset += data_len;
let next = if let Some(final_data) = data_to_upload {
dest_version.blocks.clear();
dest_version.blocks.put(version_block_key, version_block);
let block_ref = BlockRef {
block: final_hash,
version: dest_version_id,
deleted: false.into(),
};
let (_, _, _, next) = futures::try_join!(
// Thing 1: if the block is not exactly a block that existed before,
// we need to insert that data as a new block.
garage.block_manager.rpc_put_block(
final_hash,
final_data,
dest_encryption.is_encrypted(),
None
),
// Thing 2: we need to insert the block in the version
garage.version_table.insert(&dest_version),
// Thing 3: we need to add a block reference
garage.block_ref_table.insert(&block_ref),
// Thing 4: we need to read the next block
defragmenter.next(),
)?;
next
} else {
blocks_to_dup.blocks.put(version_block_key, version_block);
defragmenter.next().await?
let block_ref = BlockRef {
block: final_hash,
version: dest_version_id,
deleted: false.into(),
};
let (_, _, _, next) = futures::try_join!(
// Thing 1: if the block is not exactly a block that existed before,
// we need to insert that data as a new block.
async {
if let Some(final_data) = data_to_upload {
garage
.block_manager
.rpc_put_block(final_hash, final_data, dest_encryption.is_encrypted(), None)
.await
} else {
Ok(())
}
},
// Thing 2: we need to insert the block in the version
garage.version_table.insert(&dest_version),
// Thing 3: we need to add a block reference
garage.block_ref_table.insert(&block_ref),
// Thing 4: we need to read the next block
defragmenter.next(),
)?;
next_block = next;
}
assert_eq!(current_offset, source_range.length);
// Put the duplicated blocks into the version & block_refs tables
let block_refs_to_put = blocks_to_dup
.blocks
.items()
.iter()
.map(|b| BlockRef {
block: b.1.hash,
version: dest_version_id,
deleted: false.into(),
})
.collect::<Vec<_>>();
futures::try_join!(
garage.version_table.insert(&blocks_to_dup),
garage.block_ref_table.insert_many(&block_refs_to_put[..]),
)?;
let checksums = checksummer.finalize();
let etag = dest_encryption.etag_from_md5(&checksums.md5);
let checksum = checksums.extract(dest_object_checksum_algorithm);
+2 -30
View File
@@ -19,13 +19,12 @@ use garage_net::stream::ByteStream;
use garage_rpc::rpc_helper::OrderTag;
use garage_table::EmptyKey;
use garage_util::data::*;
use garage_util::error::{Error as UtilError, OkOrMessage};
use garage_util::error::OkOrMessage;
use garage_model::garage::Garage;
use garage_model::s3::object_table::*;
use garage_model::s3::version_table::*;
use garage_api_common::common_error::CommonError;
use garage_api_common::helpers::*;
use garage_api_common::signature::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE};
@@ -216,7 +215,6 @@ pub async fn handle_head_without_ctx(
.get(&object_version.uuid, &EmptyKey)
.await?
.ok_or(Error::NoSuchKey)?;
check_version_not_deleted(&version)?;
let (part_offset, part_end) =
calculate_part_bounds(&version, pn).ok_or(Error::InvalidPart)?;
@@ -367,21 +365,6 @@ pub async fn handle_get_without_ctx(
}
}
pub(crate) fn check_version_not_deleted(version: &Version) -> Result<(), Error> {
if version.deleted.get() {
// the version was deleted between when the object_table was consulted
// and now, this could mean the object was deleted, or overriden.
// Rather than say the key doesn't exist, return a transient error
// to signal the client to try again.
return Err(CommonError::InternalError(UtilError::Message(
"conflict/inconsistency between object and version state, version is deleted"
.to_string(),
))
.into());
}
Ok(())
}
async fn handle_get_full(
garage: Arc<Garage>,
version: &ObjectVersion,
@@ -448,7 +431,6 @@ pub fn full_object_byte_stream(
.ok_or_message("channel closed")?;
let version = version_fut.await.unwrap()?.ok_or(Error::NoSuchKey)?;
check_version_not_deleted(&version)?;
for (i, (_, vb)) in version.blocks.items().iter().enumerate().skip(1) {
let stream_block_i = encryption
.get_block(&garage, &vb.hash, Some(order_stream.order(i as u64)))
@@ -464,14 +446,6 @@ pub fn full_object_byte_stream(
{
Ok(()) => (),
Err(e) => {
// TODO i think this is a bad idea, we should log
// an error and stop there. If the error happens to
// be exactly the size of what hasn't been streamed
// yet, the client will see the request as a
// success
// instead truncating the output notify the client
// something happened with their download, so that
// they can retry it
let _ = tx.send(error_stream_item(e)).await;
}
}
@@ -523,7 +497,7 @@ async fn handle_get_range(
.get(&version.uuid, &EmptyKey)
.await?
.ok_or(Error::NoSuchKey)?;
check_version_not_deleted(&version)?;
let body =
body_from_blocks_range(garage, encryption, version.blocks.items(), begin, end);
Ok(resp_builder.body(body)?)
@@ -574,8 +548,6 @@ async fn handle_get_part(
.await?
.ok_or(Error::NoSuchKey)?;
check_version_not_deleted(&version)?;
let (begin, end) =
calculate_part_bounds(&version, part_number).ok_or(Error::InvalidPart)?;
+1 -1
View File
@@ -27,7 +27,7 @@ pub async fn handle_get_lifecycle(ctx: ReqCtx) -> Result<Response<ResBody>, Erro
.body(string_body(xml))?)
} else {
Ok(Response::builder()
.status(StatusCode::NOT_FOUND)
.status(StatusCode::NO_CONTENT)
.body(empty_body())?)
}
}
-77
View File
@@ -13,10 +13,6 @@ pub fn xmlns_tag<S: Serializer>(_v: &(), s: S) -> Result<S::Ok, S::Error> {
s.serialize_str("http://s3.amazonaws.com/doc/2006-03-01/")
}
pub fn xmlns_xsi_tag<S: Serializer>(_v: &(), s: S) -> Result<S::Ok, S::Error> {
s.serialize_str("http://www.w3.org/2001/XMLSchema-instance")
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
pub struct Value(#[serde(rename = "$value")] pub String);
@@ -323,42 +319,6 @@ pub struct PostObject {
pub etag: Value,
}
#[derive(Debug, Serialize, PartialEq, Eq)]
pub struct Grantee {
#[serde(rename = "xmlns:xsi", serialize_with = "xmlns_xsi_tag")]
pub xmlns_xsi: (),
#[serde(rename = "xsi:type")]
pub typ: String,
#[serde(rename = "DisplayName")]
pub display_name: Option<Value>,
#[serde(rename = "ID")]
pub id: Option<Value>,
}
#[derive(Debug, Serialize, PartialEq, Eq)]
pub struct Grant {
#[serde(rename = "Grantee")]
pub grantee: Grantee,
#[serde(rename = "Permission")]
pub permission: Value,
}
#[derive(Debug, Serialize, PartialEq, Eq)]
pub struct AccessControlList {
#[serde(rename = "Grant")]
pub entries: Vec<Grant>,
}
#[derive(Debug, Serialize, PartialEq, Eq)]
pub struct AccessControlPolicy {
#[serde(serialize_with = "xmlns_tag")]
pub xmlns: (),
#[serde(rename = "Owner")]
pub owner: Option<Owner>,
#[serde(rename = "AccessControlList")]
pub acl: AccessControlList,
}
#[cfg(test)]
mod tests {
use super::*;
@@ -467,43 +427,6 @@ mod tests {
Ok(())
}
#[test]
fn get_bucket_acl_result() -> Result<(), ApiError> {
let grant = Grant {
grantee: Grantee {
xmlns_xsi: (),
typ: "CanonicalUser".to_string(),
display_name: Some(Value("owner_name".to_string())),
id: Some(Value("qsdfjklm".to_string())),
},
permission: Value("FULL_CONTROL".to_string()),
};
let get_bucket_acl = AccessControlPolicy {
xmlns: (),
owner: None,
acl: AccessControlList {
entries: vec![grant],
},
};
assert_eq!(
to_xml_with_header(&get_bucket_acl)?,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<AccessControlPolicy xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
<AccessControlList>\
<Grant>\
<Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"CanonicalUser\">\
<DisplayName>owner_name</DisplayName>\
<ID>qsdfjklm</ID>\
</Grantee>\
<Permission>FULL_CONTROL</Permission>\
</Grant>\
</AccessControlList>\
</AccessControlPolicy>"
);
Ok(())
}
#[test]
fn delete_result() -> Result<(), ApiError> {
let delete_result = DeleteResult {
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_block"
version = "1.3.0"
version = "1.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+8 -19
View File
@@ -17,6 +17,7 @@ use opentelemetry::{
Context,
};
use garage_net::endpoint::RpcInFlightLimiter;
use garage_net::stream::{read_stream_to_end, stream_asyncread, ByteStream};
use garage_db as db;
@@ -50,8 +51,6 @@ pub const INLINE_THRESHOLD: usize = 3072;
// to delete the block locally.
pub(crate) const BLOCK_GC_DELAY: Duration = Duration::from_secs(600);
const BLOCK_READ_SEMAPHORE_TIMEOUT: Duration = Duration::from_secs(15);
/// RPC messages used to share blocks of data between nodes
#[derive(Debug, Serialize, Deserialize)]
pub enum BlockRpc {
@@ -89,7 +88,6 @@ pub struct BlockManager {
disable_scrub: bool,
mutation_lock: Vec<Mutex<BlockManagerLocked>>,
read_semaphore: Semaphore,
pub rc: BlockRc,
pub resync: BlockResyncManager,
@@ -179,8 +177,6 @@ impl BlockManager {
.iter()
.map(|_| Mutex::new(BlockManagerLocked()))
.collect::<Vec<_>>(),
read_semaphore: Semaphore::new(config.block_max_concurrent_reads),
rc,
resync,
system,
@@ -300,6 +296,7 @@ impl BlockManager {
&node_id,
BlockRpc::GetBlock(*hash, order_tag),
priority,
RpcInFlightLimiter::TableWrite,
);
tokio::select! {
res = rpc => {
@@ -413,8 +410,8 @@ impl BlockManager {
}
/// Get number of items in the refcount table
pub fn rc_approximate_len(&self) -> Result<usize, Error> {
Ok(self.rc.rc_table.approximate_len()?)
pub fn rc_len(&self) -> Result<usize, Error> {
Ok(self.rc.rc_table.len()?)
}
/// Send command to start/stop/manager scrub worker
@@ -432,7 +429,7 @@ impl BlockManager {
/// List all resync errors
pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> {
let mut blocks = Vec::with_capacity(self.resync.errors.approximate_len()?);
let mut blocks = Vec::with_capacity(self.resync.errors.len()?);
for ent in self.resync.errors.iter()? {
let (hash, cnt) = ent?;
let cnt = ErrorCounter::decode(&cnt);
@@ -562,6 +559,9 @@ impl BlockManager {
match self.find_block(hash).await {
Some(p) => self.read_block_from(hash, &p).await,
None => {
// Not found but maybe we should have had it ??
self.resync
.put_to_resync(hash, 2 * self.system.rpc_helper().rpc_timeout())?;
return Err(Error::Message(format!(
"block {:?} not found on node",
hash
@@ -583,15 +583,6 @@ impl BlockManager {
) -> Result<DataBlock, Error> {
let (header, path) = block_path.as_parts_ref();
let permit = tokio::select! {
sem = self.read_semaphore.acquire() => sem.ok_or_message("acquire read semaphore")?,
_ = tokio::time::sleep(BLOCK_READ_SEMAPHORE_TIMEOUT) => {
self.metrics.block_read_semaphore_timeouts.add(1);
debug!("read block {:?}: read_semaphore acquire timeout", hash);
return Err(Error::Message("read block: read_semaphore acquire timeout".into()));
}
};
let mut f = fs::File::open(&path).await?;
let mut data = vec![];
f.read_to_end(&mut data).await?;
@@ -616,8 +607,6 @@ impl BlockManager {
return Err(Error::CorruptData(*hash));
}
drop(permit);
Ok(data)
}
+3 -9
View File
@@ -22,7 +22,6 @@ pub struct BlockManagerMetrics {
pub(crate) bytes_read: BoundCounter<u64>,
pub(crate) block_read_duration: BoundValueRecorder<f64>,
pub(crate) block_read_semaphore_timeouts: BoundCounter<u64>,
pub(crate) bytes_written: BoundCounter<u64>,
pub(crate) block_write_duration: BoundValueRecorder<f64>,
pub(crate) delete_counter: BoundCounter<u64>,
@@ -51,7 +50,7 @@ impl BlockManagerMetrics {
.init(),
_rc_size: meter
.u64_value_observer("block.rc_size", move |observer| {
if let Ok(value) = rc_tree.approximate_len() {
if let Ok(value) = rc_tree.len() {
observer.observe(value as u64, &[])
}
})
@@ -59,7 +58,7 @@ impl BlockManagerMetrics {
.init(),
_resync_queue_len: meter
.u64_value_observer("block.resync_queue_length", move |observer| {
if let Ok(value) = resync_queue.approximate_len() {
if let Ok(value) = resync_queue.len() {
observer.observe(value as u64, &[]);
}
})
@@ -69,7 +68,7 @@ impl BlockManagerMetrics {
.init(),
_resync_errored_blocks: meter
.u64_value_observer("block.resync_errored_blocks", move |observer| {
if let Ok(value) = resync_errors.approximate_len() {
if let Ok(value) = resync_errors.len() {
observer.observe(value as u64, &[]);
}
})
@@ -120,11 +119,6 @@ impl BlockManagerMetrics {
.with_description("Duration of block read operations")
.init()
.bind(&[]),
block_read_semaphore_timeouts: meter
.u64_counter("block.read_semaphore_timeouts")
.with_description("Number of block reads that failed due to semaphore acquire timeout")
.init()
.bind(&[]),
bytes_written: meter
.u64_counter("block.bytes_written")
.with_description("Number of bytes written to disk")
+6 -16
View File
@@ -106,13 +106,13 @@ impl BlockResyncManager {
}
/// Get length of resync queue
pub fn queue_approximate_len(&self) -> Result<usize, Error> {
Ok(self.queue.approximate_len()?)
pub fn queue_len(&self) -> Result<usize, Error> {
Ok(self.queue.len()?)
}
/// Get number of blocks that have an error
pub fn errors_approximate_len(&self) -> Result<usize, Error> {
Ok(self.errors.approximate_len()?)
pub fn errors_len(&self) -> Result<usize, Error> {
Ok(self.errors.len()?)
}
/// Clear the error counter for a block and put it in queue immediately
@@ -133,14 +133,6 @@ impl BlockResyncManager {
)))
}
/// Clear the entire resync queue and list of errored blocks
/// Corresponds to `garage repair clear-resync-queue`
pub fn clear_resync_queue(&self) -> Result<(), Error> {
self.queue.clear()?;
self.errors.clear()?;
Ok(())
}
pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
let notify = self.notify.clone();
vars.register_rw(
@@ -556,11 +548,9 @@ impl Worker for ResyncWorker {
}
WorkerStatus {
queue_length: Some(self.manager.resync.queue_approximate_len().unwrap_or(0) as u64),
queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64),
tranquility: Some(tranquility),
persistent_errors: Some(
self.manager.resync.errors_approximate_len().unwrap_or(0) as u64
),
persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64),
..Default::default()
}
}
+1 -6
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_db"
version = "1.3.0"
version = "1.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@@ -16,14 +16,10 @@ err-derive.workspace = true
tracing.workspace = true
heed = { workspace = true, optional = true }
rusqlite = { workspace = true, optional = true, features = ["backup"] }
r2d2 = { workspace = true, optional = true }
r2d2_sqlite = { workspace = true, optional = true }
fjall = { workspace = true, optional = true }
parking_lot = { workspace = true, optional = true }
[dev-dependencies]
mktemp.workspace = true
@@ -31,5 +27,4 @@ mktemp.workspace = true
default = [ "lmdb", "sqlite" ]
bundled-libs = [ "rusqlite?/bundled" ]
lmdb = [ "heed" ]
fjall = [ "dep:fjall", "dep:parking_lot" ]
sqlite = [ "rusqlite", "r2d2", "r2d2_sqlite" ]
-453
View File
@@ -1,453 +0,0 @@
use core::ops::Bound;
use std::path::PathBuf;
use std::sync::Arc;
use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard};
use fjall::{
PartitionCreateOptions, PersistMode, TransactionalKeyspace, TransactionalPartitionHandle,
WriteTransaction,
};
use crate::{
open::{Engine, OpenOpt},
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
TxResult, TxValueIter, Value, ValueIter,
};
pub use fjall;
// --
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
info!("Opening Fjall database at: {}", path.display());
if opt.fsync {
return Err(Error(
"metadata_fsync is not supported with the Fjall database engine".into(),
));
}
let mut config = fjall::Config::new(path);
if let Some(block_cache_size) = opt.fjall_block_cache_size {
config = config.cache_size(block_cache_size as u64);
}
let keyspace = config.open_transactional()?;
Ok(FjallDb::init(keyspace))
}
// -- err
impl From<fjall::Error> for Error {
fn from(e: fjall::Error) -> Error {
Error(format!("fjall: {}", e).into())
}
}
impl From<fjall::LsmError> for Error {
fn from(e: fjall::LsmError) -> Error {
Error(format!("fjall lsm_tree: {}", e).into())
}
}
impl From<fjall::Error> for TxOpError {
fn from(e: fjall::Error) -> TxOpError {
TxOpError(e.into())
}
}
// -- db
pub struct FjallDb {
keyspace: TransactionalKeyspace,
trees: RwLock<Vec<(String, TransactionalPartitionHandle)>>,
}
type ByteRefRangeBound<'r> = (Bound<&'r [u8]>, Bound<&'r [u8]>);
impl FjallDb {
pub fn init(keyspace: TransactionalKeyspace) -> Db {
let s = Self {
keyspace,
trees: RwLock::new(Vec::new()),
};
Db(Arc::new(s))
}
fn get_tree(
&self,
i: usize,
) -> Result<MappedRwLockReadGuard<'_, TransactionalPartitionHandle>> {
RwLockReadGuard::try_map(self.trees.read(), |trees: &Vec<_>| {
trees.get(i).map(|tup| &tup.1)
})
.map_err(|_| Error("invalid tree id".into()))
}
}
impl IDb for FjallDb {
fn engine(&self) -> String {
"Fjall (EXPERIMENTAL!)".into()
}
fn open_tree(&self, name: &str) -> Result<usize> {
let mut trees = self.trees.write();
let safe_name = encode_name(name)?;
if let Some(i) = trees.iter().position(|(name, _)| *name == safe_name) {
Ok(i)
} else {
let tree = self
.keyspace
.open_partition(&safe_name, PartitionCreateOptions::default())?;
let i = trees.len();
trees.push((safe_name, tree));
Ok(i)
}
}
fn list_trees(&self) -> Result<Vec<String>> {
Ok(self
.keyspace
.list_partitions()
.iter()
.map(|n| decode_name(&n))
.collect::<Result<Vec<_>>>()?)
}
fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
std::fs::create_dir_all(base_path)?;
let path = Engine::Fjall.db_path(base_path);
let source_state = self.keyspace.read_tx();
let copy_keyspace = fjall::Config::new(path).open()?;
for partition_name in self.keyspace.list_partitions() {
let source_partition = self
.keyspace
.open_partition(&partition_name, PartitionCreateOptions::default())?;
let copy_partition =
copy_keyspace.open_partition(&partition_name, PartitionCreateOptions::default())?;
for entry in source_state.iter(&source_partition) {
let (key, value) = entry?;
copy_partition.insert(key, value)?;
}
}
copy_keyspace.persist(PersistMode::SyncAll)?;
Ok(())
}
// ----
fn get(&self, tree_idx: usize, key: &[u8]) -> Result<Option<Value>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
let val = tx.get(&tree, key)?;
match val {
None => Ok(None),
Some(v) => Ok(Some(v.to_vec())),
}
}
fn approximate_len(&self, tree_idx: usize) -> Result<usize> {
let tree = self.get_tree(tree_idx)?;
Ok(tree.approximate_len())
}
fn is_empty(&self, tree_idx: usize) -> Result<bool> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(tx.is_empty(&tree)?)
}
fn insert(&self, tree_idx: usize, key: &[u8], value: &[u8]) -> Result<()> {
let tree = self.get_tree(tree_idx)?;
let mut tx = self.keyspace.write_tx();
tx.insert(&tree, key, value);
tx.commit()?;
Ok(())
}
fn remove(&self, tree_idx: usize, key: &[u8]) -> Result<()> {
let tree = self.get_tree(tree_idx)?;
let mut tx = self.keyspace.write_tx();
tx.remove(&tree, key);
tx.commit()?;
Ok(())
}
fn clear(&self, tree_idx: usize) -> Result<()> {
let mut trees = self.trees.write();
if tree_idx >= trees.len() {
return Err(Error("invalid tree id".into()));
}
let (name, tree) = trees.remove(tree_idx);
self.keyspace.delete_partition(tree)?;
let tree = self
.keyspace
.open_partition(&name, PartitionCreateOptions::default())?;
trees.insert(tree_idx, (name, tree));
Ok(())
}
fn iter(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(tx.iter(&tree).map(iterator_remap)))
}
fn iter_rev(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(tx.iter(&tree).rev().map(iterator_remap)))
}
fn range<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(
tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high))
.map(iterator_remap),
))
}
fn range_rev<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(
tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high))
.rev()
.map(iterator_remap),
))
}
// ----
fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()> {
let trees = self.trees.read();
let mut tx = FjallTx {
trees: &trees[..],
tx: self.keyspace.write_tx(),
};
let res = f.try_on(&mut tx);
match res {
TxFnResult::Ok(on_commit) => {
tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?;
Ok(on_commit)
}
TxFnResult::Abort => {
tx.tx.rollback();
Err(TxError::Abort(()))
}
TxFnResult::DbErr => {
tx.tx.rollback();
Err(TxError::Db(Error(
"(this message will be discarded)".into(),
)))
}
}
}
}
// ----
struct FjallTx<'a> {
trees: &'a [(String, TransactionalPartitionHandle)],
tx: WriteTransaction<'a>,
}
impl<'a> FjallTx<'a> {
fn get_tree(&self, i: usize) -> TxOpResult<&TransactionalPartitionHandle> {
self.trees.get(i).map(|tup| &tup.1).ok_or_else(|| {
TxOpError(Error(
"invalid tree id (it might have been openned after the transaction started)".into(),
))
})
}
}
impl<'a> ITx for FjallTx<'a> {
fn get(&self, tree_idx: usize, key: &[u8]) -> TxOpResult<Option<Value>> {
let tree = self.get_tree(tree_idx)?;
match self.tx.get(tree, key)? {
Some(v) => Ok(Some(v.to_vec())),
None => Ok(None),
}
}
fn len(&self, tree_idx: usize) -> TxOpResult<usize> {
let tree = self.get_tree(tree_idx)?;
Ok(self.tx.len(tree)? as usize)
}
fn insert(&mut self, tree_idx: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> {
let tree = self.get_tree(tree_idx)?.clone();
self.tx.insert(&tree, key, value);
Ok(())
}
fn remove(&mut self, tree_idx: usize, key: &[u8]) -> TxOpResult<()> {
let tree = self.get_tree(tree_idx)?.clone();
self.tx.remove(&tree, key);
Ok(())
}
fn clear(&mut self, _tree_idx: usize) -> TxOpResult<()> {
unimplemented!("LSM tree clearing in cross-partition transaction is not supported")
}
fn iter(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?.clone();
Ok(Box::new(self.tx.iter(&tree).map(iterator_remap_tx)))
}
fn iter_rev(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?.clone();
Ok(Box::new(self.tx.iter(&tree).rev().map(iterator_remap_tx)))
}
fn range<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let low = clone_bound(low);
let high = clone_bound(high);
Ok(Box::new(
self.tx
.range::<Vec<u8>, ByteVecRangeBounds>(&tree, (low, high))
.map(iterator_remap_tx),
))
}
fn range_rev<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let low = clone_bound(low);
let high = clone_bound(high);
Ok(Box::new(
self.tx
.range::<Vec<u8>, ByteVecRangeBounds>(&tree, (low, high))
.rev()
.map(iterator_remap_tx),
))
}
}
// -- maps fjall's (k, v) to ours
fn iterator_remap(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> Result<(Value, Value)> {
r.map(|(k, v)| (k.to_vec(), v.to_vec()))
.map_err(|e| e.into())
}
fn iterator_remap_tx(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> TxOpResult<(Value, Value)> {
r.map(|(k, v)| (k.to_vec(), v.to_vec()))
.map_err(|e| e.into())
}
// -- utils to deal with Garage's tightness on Bound lifetimes
type ByteVecBound = Bound<Vec<u8>>;
type ByteVecRangeBounds = (ByteVecBound, ByteVecBound);
fn clone_bound(bound: Bound<&[u8]>) -> ByteVecBound {
let value = match bound {
Bound::Excluded(v) | Bound::Included(v) => v.to_vec(),
Bound::Unbounded => vec![],
};
match bound {
Bound::Included(_) => Bound::Included(value),
Bound::Excluded(_) => Bound::Excluded(value),
Bound::Unbounded => Bound::Unbounded,
}
}
// -- utils to encode table names --
fn encode_name(s: &str) -> Result<String> {
let base = 'A' as u32;
let mut ret = String::with_capacity(s.len() + 10);
for c in s.chars() {
if c.is_alphanumeric() || c == '_' || c == '-' || c == '#' {
ret.push(c);
} else if c <= u8::MAX as char {
ret.push('$');
let c_hi = c as u32 / 16;
let c_lo = c as u32 % 16;
ret.push(char::from_u32(base + c_hi).unwrap());
ret.push(char::from_u32(base + c_lo).unwrap());
} else {
return Err(Error(
format!("table name {} could not be safely encoded", s).into(),
));
}
}
Ok(ret)
}
fn decode_name(s: &str) -> Result<String> {
use std::convert::TryFrom;
let errfn = || Error(format!("encoded table name {} is invalid", s).into());
let c_map = |c: char| {
let c = c as u32;
let base = 'A' as u32;
if (base..base + 16).contains(&c) {
Some(c - base)
} else {
None
}
};
let mut ret = String::with_capacity(s.len());
let mut it = s.chars();
while let Some(c) = it.next() {
if c == '$' {
let c_hi = it.next().and_then(c_map).ok_or_else(errfn)?;
let c_lo = it.next().and_then(c_map).ok_or_else(errfn)?;
let c_dec = char::try_from(c_hi * 16 + c_lo).map_err(|_| errfn())?;
ret.push(c_dec);
} else {
ret.push(c);
}
}
Ok(ret)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_encdec_name() {
for name in [
"testname",
"test_name",
"test name",
"test$name",
"test:name@help.me$get/this**right",
] {
let encname = encode_name(name).unwrap();
assert!(!encname.contains(' '));
assert!(!encname.contains('.'));
assert!(!encname.contains('*'));
assert_eq!(*name, decode_name(&encname).unwrap());
}
}
}
+29 -48
View File
@@ -1,8 +1,6 @@
#[macro_use]
extern crate tracing;
#[cfg(feature = "fjall")]
pub mod fjall_adapter;
#[cfg(feature = "lmdb")]
pub mod lmdb_adapter;
#[cfg(feature = "sqlite")]
@@ -106,44 +104,32 @@ impl Db {
result: Cell::new(None),
};
let tx_res = self.0.transaction(&f);
let fn_res = f.result.into_inner();
let ret = f
.result
.into_inner()
.expect("Transaction did not store result");
match (tx_res, fn_res) {
(Ok(on_commit), Some(Ok(value))) => {
// Transaction succeeded
// TxFn stored the value to return to the user in fn_res
// tx_res contains the on_commit list of callbacks, run them now
on_commit.into_iter().for_each(|f| f());
Ok(value)
}
(Err(TxError::Abort(())), Some(Err(TxError::Abort(e)))) => {
// Transaction was aborted by user code
// The abort error value is stored in fn_res
Err(TxError::Abort(e))
}
(Err(TxError::Db(_tx_e)), Some(Err(TxError::Db(fn_e)))) => {
// Transaction encountered a DB error in user code
// The error value encountered is the one in fn_res,
// tx_res contains only a dummy error message
Err(TxError::Db(fn_e))
}
(Err(TxError::Db(tx_e)), None) => {
// Transaction encounterred a DB error when initializing the transaction,
// before user code was called
Err(TxError::Db(tx_e))
}
(Err(TxError::Db(tx_e)), Some(Ok(_))) => {
// Transaction encounterred a DB error when commiting the transaction,
// after user code was called
Err(TxError::Db(tx_e))
}
(tx_res, fn_res) => {
panic!(
"unexpected error case: tx_res={:?}, fn_res={:?}",
tx_res.map(|_| "..."),
fn_res.map(|x| x.map(|_| "...").map_err(|_| "..."))
);
}
match tx_res {
Ok(on_commit) => match ret {
Ok(value) => {
on_commit.into_iter().for_each(|f| f());
Ok(value)
}
_ => unreachable!(),
},
Err(TxError::Abort(())) => match ret {
Err(TxError::Abort(e)) => Err(TxError::Abort(e)),
_ => unreachable!(),
},
Err(TxError::Db(e2)) => match ret {
// Ok was stored -> the error occurred when finalizing
// transaction
Ok(_) => Err(TxError::Db(e2)),
// An error was already stored: that's the one we want to
// return
Err(TxError::Db(e)) => Err(TxError::Db(e)),
_ => unreachable!(),
},
}
}
@@ -166,7 +152,7 @@ impl Db {
let tree_names = other.list_trees()?;
for name in tree_names {
let tree = self.open_tree(&name)?;
if !tree.is_empty()? {
if tree.len()? > 0 {
return Err(Error(format!("tree {} already contains data", name).into()));
}
@@ -208,12 +194,8 @@ impl Tree {
self.0.get(self.1, key.as_ref())
}
#[inline]
pub fn approximate_len(&self) -> Result<usize> {
self.0.approximate_len(self.1)
}
#[inline]
pub fn is_empty(&self) -> Result<bool> {
self.0.is_empty(self.1)
pub fn len(&self) -> Result<usize> {
self.0.len(self.1)
}
#[inline]
@@ -351,8 +333,7 @@ pub(crate) trait IDb: Send + Sync {
fn snapshot(&self, path: &PathBuf) -> Result<()>;
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
fn approximate_len(&self, tree: usize) -> Result<usize>;
fn is_empty(&self, tree: usize) -> Result<bool>;
fn len(&self, tree: usize) -> Result<usize>;
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>;
fn remove(&self, tree: usize, key: &[u8]) -> Result<()>;
+29 -94
View File
@@ -1,8 +1,8 @@
use core::ops::Bound;
use core::ptr::NonNull;
use std::collections::HashMap;
use std::convert::TryInto;
use std::marker::PhantomPinned;
use std::path::PathBuf;
use std::pin::Pin;
use std::sync::{Arc, RwLock};
@@ -11,55 +11,12 @@ use heed::types::ByteSlice;
use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database};
use crate::{
open::{Engine, OpenOpt},
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
TxResult, TxValueIter, Value, ValueIter,
};
pub use heed;
// ---- top-level open function
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
info!("Opening LMDB database at: {}", path.display());
if let Err(e) = std::fs::create_dir_all(&path) {
return Err(Error(
format!("Unable to create LMDB data directory: {}", e).into(),
));
}
let map_size = match opt.lmdb_map_size {
None => recommended_map_size(),
Some(v) => v - (v % 4096),
};
let mut env_builder = heed::EnvOpenOptions::new();
env_builder.max_dbs(100);
env_builder.map_size(map_size);
env_builder.max_readers(2048);
unsafe {
env_builder.flag(heed::flags::Flags::MdbNoRdAhead);
env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
if !opt.fsync {
env_builder.flag(heed::flags::Flags::MdbNoSync);
}
}
match env_builder.open(&path) {
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
return Err(Error(
"OutOfMemory error while trying to open LMDB database. This can happen \
if your operating system is not allowing you to use sufficient virtual \
memory address space. Please check that no limit is set (ulimit -v). \
You may also try to set a smaller `lmdb_map_size` configuration parameter. \
On 32-bit machines, you should probably switch to another database engine."
.into(),
))
}
Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
Ok(db) => Ok(LmdbDb::init(db)),
}
}
// -- err
impl From<heed::Error> for Error {
@@ -147,9 +104,10 @@ impl IDb for LmdbDb {
Ok(ret2)
}
fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
std::fs::create_dir_all(base_path)?;
let path = Engine::Lmdb.db_path(base_path);
fn snapshot(&self, to: &PathBuf) -> Result<()> {
std::fs::create_dir_all(to)?;
let mut path = to.clone();
path.push("data.mdb");
self.db
.copy_to_path(path, heed::CompactionOption::Enabled)?;
Ok(())
@@ -168,16 +126,11 @@ impl IDb for LmdbDb {
}
}
fn approximate_len(&self, tree: usize) -> Result<usize> {
fn len(&self, tree: usize) -> Result<usize> {
let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?;
Ok(tree.len(&tx)?.try_into().unwrap())
}
fn is_empty(&self, tree: usize) -> Result<bool> {
let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?;
Ok(tree.is_empty(&tx)?)
}
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
let tree = self.get_tree(tree)?;
@@ -206,15 +159,13 @@ impl IDb for LmdbDb {
fn iter(&self, tree: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?;
// Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) }
TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?))
}
fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?;
// Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?)) }
TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?))
}
fn range<'r>(
@@ -225,8 +176,7 @@ impl IDb for LmdbDb {
) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?;
// Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?)) }
TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?))
}
fn range_rev<'r>(
&self,
@@ -236,8 +186,7 @@ impl IDb for LmdbDb {
) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?;
// Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?)) }
TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?))
}
// ----
@@ -367,41 +316,28 @@ where
{
tx: RoTxn<'a>,
iter: Option<I>,
_pin: PhantomPinned,
}
impl<'a, I> TxAndIterator<'a, I>
where
I: Iterator<Item = IteratorItem<'a>> + 'a,
{
fn iter(self: Pin<&mut Self>) -> &mut Option<I> {
// Safety: iter is not structural
unsafe { &mut self.get_unchecked_mut().iter }
}
/// Safety: iterfun must not store its argument anywhere but in its result.
unsafe fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>>
fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>>
where
F: FnOnce(&'a RoTxn<'a>) -> Result<I>,
{
let res = TxAndIterator {
tx,
iter: None,
_pin: PhantomPinned,
};
let res = TxAndIterator { tx, iter: None };
let mut boxed = Box::pin(res);
let tx_lifetime_overextended: &'a RoTxn<'a> = {
let tx = &boxed.tx;
// Safety: Artificially extending the lifetime because
// this reference will only be stored and accessed from the
// returned ValueIter which guarantees that it is destroyed
// before the tx it is pointing to.
unsafe { &*&raw const *tx }
};
let iter = iterfun(&tx_lifetime_overextended)?;
// This unsafe allows us to bypass lifetime checks
let tx = unsafe { NonNull::from(&boxed.tx).as_ref() };
let iter = iterfun(tx)?;
*boxed.as_mut().iter() = Some(iter);
let mut_ref = Pin::as_mut(&mut boxed);
// This unsafe allows us to write in a field of the pinned struct
unsafe {
Pin::get_unchecked_mut(mut_ref).iter = Some(iter);
}
Ok(Box::new(TxAndIteratorPin(boxed)))
}
@@ -412,10 +348,8 @@ where
I: Iterator<Item = IteratorItem<'a>> + 'a,
{
fn drop(&mut self) {
// Safety: `new_unchecked` is okay because we know this value is never
// used again after being dropped.
let this = unsafe { Pin::new_unchecked(self) };
drop(this.iter().take());
// ensure the iterator is dropped before the RoTxn it references
drop(self.iter.take());
}
}
@@ -431,12 +365,13 @@ where
fn next(&mut self) -> Option<Self::Item> {
let mut_ref = Pin::as_mut(&mut self.0);
let next = mut_ref.iter().as_mut()?.next()?;
let res = match next {
Err(e) => Err(e.into()),
Ok((k, v)) => Ok((k.to_vec(), v.to_vec())),
};
Some(res)
// This unsafe allows us to mutably access the iterator field
let next = unsafe { Pin::get_unchecked_mut(mut_ref).iter.as_mut()?.next() };
match next {
None => None,
Some(Err(e)) => Some(Err(e.into())),
Some(Ok((k, v))) => Some(Ok((k.to_vec(), v.to_vec()))),
}
}
}
+44 -28
View File
@@ -11,7 +11,6 @@ use crate::{Db, Error, Result};
pub enum Engine {
Lmdb,
Sqlite,
Fjall,
}
impl Engine {
@@ -20,26 +19,8 @@ impl Engine {
match self {
Self::Lmdb => "lmdb",
Self::Sqlite => "sqlite",
Self::Fjall => "fjall",
}
}
/// Return engine-specific DB path from base path
pub fn db_path(&self, base_path: &PathBuf) -> PathBuf {
let mut ret = base_path.clone();
match self {
Self::Lmdb => {
ret.push("db.lmdb");
}
Self::Sqlite => {
ret.push("db.sqlite");
}
Self::Fjall => {
ret.push("db.fjall");
}
}
ret
}
}
impl std::fmt::Display for Engine {
@@ -55,11 +36,10 @@ impl std::str::FromStr for Engine {
match text {
"lmdb" | "heed" => Ok(Self::Lmdb),
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
"fjall" => Ok(Self::Fjall),
"sled" => Err(Error("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into())),
kind => Err(Error(
format!(
"Invalid DB engine: {} (options are: lmdb, sqlite, fjall)",
"Invalid DB engine: {} (options are: lmdb, sqlite)",
kind
)
.into(),
@@ -71,7 +51,6 @@ impl std::str::FromStr for Engine {
pub struct OpenOpt {
pub fsync: bool,
pub lmdb_map_size: Option<usize>,
pub fjall_block_cache_size: Option<usize>,
}
impl Default for OpenOpt {
@@ -79,7 +58,6 @@ impl Default for OpenOpt {
Self {
fsync: false,
lmdb_map_size: None,
fjall_block_cache_size: None,
}
}
}
@@ -88,15 +66,53 @@ pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
match engine {
// ---- Sqlite DB ----
#[cfg(feature = "sqlite")]
Engine::Sqlite => crate::sqlite_adapter::open_db(path, opt),
Engine::Sqlite => {
info!("Opening Sqlite database at: {}", path.display());
let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
Ok(crate::sqlite_adapter::SqliteDb::new(manager, opt.fsync)?)
}
// ---- LMDB DB ----
#[cfg(feature = "lmdb")]
Engine::Lmdb => crate::lmdb_adapter::open_db(path, opt),
Engine::Lmdb => {
info!("Opening LMDB database at: {}", path.display());
if let Err(e) = std::fs::create_dir_all(&path) {
return Err(Error(
format!("Unable to create LMDB data directory: {}", e).into(),
));
}
// ---- Fjall DB ----
#[cfg(feature = "fjall")]
Engine::Fjall => crate::fjall_adapter::open_db(path, opt),
let map_size = match opt.lmdb_map_size {
None => crate::lmdb_adapter::recommended_map_size(),
Some(v) => v - (v % 4096),
};
let mut env_builder = heed::EnvOpenOptions::new();
env_builder.max_dbs(100);
env_builder.map_size(map_size);
env_builder.max_readers(2048);
unsafe {
env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoRdAhead);
env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
if !opt.fsync {
env_builder.flag(heed::flags::Flags::MdbNoSync);
}
}
match env_builder.open(&path) {
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
return Err(Error(
"OutOfMemory error while trying to open LMDB database. This can happen \
if your operating system is not allowing you to use sufficient virtual \
memory address space. Please check that no limit is set (ulimit -v). \
You may also try to set a smaller `lmdb_map_size` configuration parameter. \
On 32-bit machines, you should probably switch to another database engine."
.into(),
))
}
Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
Ok(db) => Ok(crate::lmdb_adapter::LmdbDb::init(db)),
}
}
// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
// attribute is added so that we won't have to change this match in case stop building
+12 -28
View File
@@ -11,23 +11,12 @@ use r2d2_sqlite::SqliteConnectionManager;
use rusqlite::{params, Rows, Statement, Transaction};
use crate::{
open::{Engine, OpenOpt},
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
TxResult, TxValueIter, Value, ValueIter,
};
pub use rusqlite;
// ---- top-level open function
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
info!("Opening Sqlite database at: {}", path.display());
let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
Ok(SqliteDb::new(manager, opt.fsync)?)
}
// ----
type Connection = r2d2::PooledConnection<SqliteConnectionManager>;
// --- err
@@ -150,18 +139,17 @@ impl IDb for SqliteDb {
Ok(trees)
}
fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
std::fs::create_dir_all(base_path)?;
let path = Engine::Sqlite
.db_path(&base_path)
.into_os_string()
.into_string()
.map_err(|_| Error("invalid sqlite path string".into()))?;
info!("Start sqlite VACUUM INTO `{}`", path);
self.db.get()?.execute("VACUUM INTO ?1", params![path])?;
info!("Finished sqlite VACUUM INTO `{}`", path);
fn snapshot(&self, to: &PathBuf) -> Result<()> {
fn progress(p: rusqlite::backup::Progress) {
let percent = (p.pagecount - p.remaining) * 100 / p.pagecount;
info!("Sqlite snapshot progress: {}%", percent);
}
std::fs::create_dir_all(to)?;
let mut path = to.clone();
path.push("db.sqlite");
self.db
.get()?
.backup(rusqlite::DatabaseName::Main, path, Some(progress))?;
Ok(())
}
@@ -172,7 +160,7 @@ impl IDb for SqliteDb {
self.internal_get(&self.db.get()?, &tree, key)
}
fn approximate_len(&self, tree: usize) -> Result<usize> {
fn len(&self, tree: usize) -> Result<usize> {
let tree = self.get_tree(tree)?;
let db = self.db.get()?;
@@ -184,10 +172,6 @@ impl IDb for SqliteDb {
}
}
fn is_empty(&self, tree: usize) -> Result<bool> {
Ok(self.approximate_len(tree)? == 0)
}
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
let tree = self.get_tree(tree)?;
let db = self.db.get()?;
+2 -14
View File
@@ -1,7 +1,7 @@
use crate::*;
fn test_suite(db: Db) {
let tree = db.open_tree("tree:this_is_a_tree").unwrap();
let tree = db.open_tree("tree").unwrap();
let ka: &[u8] = &b"test"[..];
let kb: &[u8] = &b"zwello"[..];
@@ -14,7 +14,7 @@ fn test_suite(db: Db) {
assert!(tree.insert(ka, va).is_ok());
assert_eq!(tree.get(ka).unwrap().unwrap(), va);
assert_eq!(tree.iter().unwrap().count(), 1);
assert_eq!(tree.len().unwrap(), 1);
// ---- test transaction logic ----
@@ -148,15 +148,3 @@ fn test_sqlite_db() {
let db = SqliteDb::new(manager, false).unwrap();
test_suite(db);
}
#[test]
#[cfg(feature = "fjall")]
fn test_fjall_db() {
use crate::fjall_adapter::{fjall, FjallDb};
let path = mktemp::Temp::new_dir().unwrap();
let config = fjall::Config::new(path).temporary(true);
let keyspace = config.open_transactional().unwrap();
let db = FjallDb::init(keyspace);
test_suite(db);
}
+1 -5
View File
@@ -1,6 +1,6 @@
[package]
name = "garage"
version = "1.3.0"
version = "1.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@@ -57,7 +57,6 @@ opentelemetry.workspace = true
opentelemetry-prometheus = { workspace = true, optional = true }
opentelemetry-otlp = { workspace = true, optional = true }
syslog-tracing = { workspace = true, optional = true }
tracing-journald = { workspace = true, optional = true }
[dev-dependencies]
garage_api_common.workspace = true
@@ -91,7 +90,6 @@ k2v = [ "garage_util/k2v", "garage_api_k2v" ]
# Database engines
lmdb = [ "garage_model/lmdb" ]
sqlite = [ "garage_model/sqlite" ]
fjall = [ "garage_model/fjall" ]
# Automatic registration and discovery via Consul API
consul-discovery = [ "garage_rpc/consul-discovery" ]
@@ -103,8 +101,6 @@ metrics = [ "garage_api_admin/metrics", "opentelemetry-prometheus" ]
telemetry-otlp = [ "opentelemetry-otlp" ]
# Logging to syslog
syslog = [ "syslog-tracing" ]
# Logging to journald
journald = [ "tracing-journald" ]
# NOTE: bundled-libs and system-libs should be treat as mutually exclusive;
# exactly one of them should be enabled.
+1 -9
View File
@@ -101,7 +101,6 @@ impl AdminRpcHandler {
let mut obj_dels = 0;
let mut mpu_dels = 0;
let mut ver_dels = 0;
let mut br_dels = 0;
for hash in blocks {
let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
@@ -132,19 +131,12 @@ impl AdminRpcHandler {
ver_dels += 1;
}
}
if !br.deleted.get() {
let mut br = br;
br.deleted.set();
self.garage.block_ref_table.insert(&br).await?;
br_dels += 1;
}
}
}
Ok(AdminRpc::Ok(format!(
"Purged {} blocks: marked {} block refs, {} versions, {} objects and {} multipart uploads as deleted",
"Purged {} blocks, {} versions, {} objects, {} multipart uploads",
blocks.len(),
br_dels,
ver_dels,
obj_dels,
mpu_dels,
+1 -1
View File
@@ -126,7 +126,7 @@ impl AdminRpcHandler {
#[allow(clippy::ptr_arg)]
async fn handle_create_bucket(&self, name: &String) -> Result<AdminRpc, Error> {
if !is_valid_bucket_name(name, self.garage.config.allow_punycode) {
if !is_valid_bucket_name(name) {
return Err(Error::BadRequest(format!(
"{}: {}",
name, INVALID_BUCKET_NAME_MESSAGE
+19 -13
View File
@@ -13,6 +13,8 @@ use serde::{Deserialize, Serialize};
use format_table::format_table_to_string;
use garage_net::endpoint::RpcInFlightLimiter;
use garage_util::background::BackgroundRunner;
use garage_util::data::*;
use garage_util::error::Error as GarageError;
@@ -118,6 +120,7 @@ impl AdminRpcHandler {
&node,
AdminRpc::LaunchRepair(opt_to_send.clone()),
PRIO_NORMAL,
RpcInFlightLimiter::NoLimit,
)
.await;
if !matches!(resp, Ok(Ok(_))) {
@@ -164,7 +167,12 @@ impl AdminRpcHandler {
let node_id = (*node).into();
match self
.endpoint
.call(&node_id, AdminRpc::Stats(opt), PRIO_NORMAL)
.call(
&node_id,
AdminRpc::Stats(opt),
PRIO_NORMAL,
RpcInFlightLimiter::NoLimit,
)
.await
{
Ok(Ok(AdminRpc::Ok(s))) => writeln!(&mut ret, "{}", s).unwrap(),
@@ -219,7 +227,7 @@ impl AdminRpcHandler {
// Gather block manager statistics
writeln!(&mut ret, "\nBlock manager stats:").unwrap();
let rc_len = self.garage.block_manager.rc_approximate_len()?.to_string();
let rc_len = self.garage.block_manager.rc_len()?.to_string();
writeln!(
&mut ret,
@@ -230,13 +238,13 @@ impl AdminRpcHandler {
writeln!(
&mut ret,
" resync queue length: {}",
self.garage.block_manager.resync.queue_approximate_len()?
self.garage.block_manager.resync.queue_len()?
)
.unwrap();
writeln!(
&mut ret,
" blocks with resync errors: {}",
self.garage.block_manager.resync.errors_approximate_len()?
self.garage.block_manager.resync.errors_len()?
)
.unwrap();
@@ -346,21 +354,16 @@ impl AdminRpcHandler {
F: TableSchema + 'static,
R: TableReplication + 'static,
{
let data_len = t
.data
.store
.approximate_len()
.map_err(GarageError::from)?
.to_string();
let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?.to_string();
let data_len = t.data.store.len().map_err(GarageError::from)?.to_string();
let mkl_len = t.merkle_updater.merkle_tree_len()?.to_string();
Ok(format!(
" {}\t{}\t{}\t{}\t{}",
F::TABLE_NAME,
data_len,
mkl_len,
t.merkle_updater.todo_approximate_len()?,
t.data.gc_todo_approximate_len()?
t.merkle_updater.todo_len()?,
t.data.gc_todo_len()?
))
}
@@ -412,6 +415,7 @@ impl AdminRpcHandler {
variable: variable.clone(),
}),
PRIO_NORMAL,
RpcInFlightLimiter::NoLimit,
)
.await??
{
@@ -461,6 +465,7 @@ impl AdminRpcHandler {
value: value.to_string(),
}),
PRIO_NORMAL,
RpcInFlightLimiter::NoLimit,
)
.await??
{
@@ -493,6 +498,7 @@ impl AdminRpcHandler {
&to,
AdminRpc::MetaOperation(MetaOperation::Snapshot { all: false }),
PRIO_NORMAL,
RpcInFlightLimiter::NoLimit,
)
.await?
}))
+17 -3
View File
@@ -2,6 +2,7 @@ use std::collections::{HashMap, HashSet};
use std::time::Duration;
use format_table::format_table;
use garage_net::endpoint::RpcInFlightLimiter;
use garage_util::error::*;
use garage_rpc::layout::*;
@@ -200,7 +201,12 @@ pub async fn cmd_connect(
args: ConnectNodeOpt,
) -> Result<(), Error> {
match rpc_cli
.call(&rpc_host, SystemRpc::Connect(args.node), PRIO_NORMAL)
.call(
&rpc_host,
SystemRpc::Connect(args.node),
PRIO_NORMAL,
RpcInFlightLimiter::NoLimit,
)
.await??
{
SystemRpc::Ok => {
@@ -216,7 +222,10 @@ pub async fn cmd_admin(
rpc_host: NodeID,
args: AdminRpc,
) -> Result<(), HelperError> {
match rpc_cli.call(&rpc_host, args, PRIO_NORMAL).await?? {
match rpc_cli
.call(&rpc_host, args, PRIO_NORMAL, RpcInFlightLimiter::NoLimit)
.await??
{
AdminRpc::Ok(msg) => {
println!("{}", msg);
}
@@ -271,7 +280,12 @@ pub async fn fetch_status(
rpc_host: NodeID,
) -> Result<Vec<KnownNodeInfo>, Error> {
match rpc_cli
.call(&rpc_host, SystemRpc::GetKnownNodes, PRIO_NORMAL)
.call(
&rpc_host,
SystemRpc::GetKnownNodes,
PRIO_NORMAL,
RpcInFlightLimiter::NoLimit,
)
.await??
{
SystemRpc::ReturnKnownNodes(nodes) => Ok(nodes),
+14 -2
View File
@@ -1,6 +1,7 @@
use bytesize::ByteSize;
use format_table::format_table;
use garage_net::endpoint::RpcInFlightLimiter;
use garage_util::crdt::Crdt;
use garage_util::error::*;
@@ -45,7 +46,12 @@ pub async fn cmd_assign_role(
args: AssignRoleOpt,
) -> Result<(), Error> {
let status = match rpc_cli
.call(&rpc_host, SystemRpc::GetKnownNodes, PRIO_NORMAL)
.call(
&rpc_host,
SystemRpc::GetKnownNodes,
PRIO_NORMAL,
RpcInFlightLimiter::NoLimit,
)
.await??
{
SystemRpc::ReturnKnownNodes(nodes) => nodes,
@@ -475,7 +481,12 @@ pub async fn fetch_layout(
rpc_host: NodeID,
) -> Result<LayoutHistory, Error> {
match rpc_cli
.call(&rpc_host, SystemRpc::PullClusterLayout, PRIO_NORMAL)
.call(
&rpc_host,
SystemRpc::PullClusterLayout,
PRIO_NORMAL,
RpcInFlightLimiter::NoLimit,
)
.await??
{
SystemRpc::AdvertiseClusterLayout(t) => Ok(t),
@@ -493,6 +504,7 @@ pub async fn send_layout(
&rpc_host,
SystemRpc::AdvertiseClusterLayout(layout),
PRIO_NORMAL,
RpcInFlightLimiter::NoLimit,
)
.await??;
Ok(())
-7
View File
@@ -466,10 +466,6 @@ pub enum RepairWhat {
/// Repair (resync/rebalance) the set of stored blocks in the cluster
#[structopt(name = "blocks", version = garage_version())]
Blocks,
/// Clear the block resync queue. The list of blocks in errored state
/// is cleared as well. You MUST run `garage repair blocks` after invoking this.
#[structopt(name = "clear-resync-queue", version = garage_version())]
ClearResyncQueue,
/// Repropagate object deletions to the version table
#[structopt(name = "versions", version = garage_version())]
Versions,
@@ -482,9 +478,6 @@ pub enum RepairWhat {
/// Recalculate block reference counters
#[structopt(name = "block-rc", version = garage_version())]
BlockRc,
/// Fix inconsistency in bucket aliases (WARNING: EXPERIMENTAL)
#[structopt(name = "aliases", version = garage_version())]
Aliases,
/// Verify integrity of all blocks on disc
#[structopt(name = "scrub", version = garage_version())]
Scrub {
+1 -38
View File
@@ -208,43 +208,6 @@ fn init_logging(opt: &Opt) {
}
}
if std::env::var("GARAGE_LOG_TO_JOURNALD")
.map(|x| x == "1" || x == "true")
.unwrap_or(false)
{
#[cfg(feature = "journald")]
{
use tracing_journald::{Priority, PriorityMappings};
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
let registry = tracing_subscriber::registry()
.with(tracing_subscriber::fmt::layer().with_writer(std::io::sink))
.with(env_filter);
match tracing_journald::layer() {
Ok(layer) => {
registry
.with(layer.with_priority_mappings(PriorityMappings {
info: Priority::Informational,
debug: Priority::Debug,
..PriorityMappings::new()
}))
.init();
}
Err(e) => {
eprintln!("Couldn't connect to journald: {}.", e);
std::process::exit(1);
}
}
return;
}
#[cfg(not(feature = "journald"))]
{
eprintln!("Journald support is not enabled in this build.");
std::process::exit(1);
}
}
tracing_subscriber::fmt()
.with_writer(std::io::stderr)
.with_env_filter(env_filter)
@@ -281,7 +244,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
// Generate a temporary keypair for our RPC client
let (_pk, sk) = sodiumoxide::crypto::sign::ed25519::gen_keypair();
let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk, None);
let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk, None, None);
// Find and parse the address of the target host
let (id, addr, is_default_addr) = if let Some(h) = opt.rpc_host {
-9
View File
@@ -88,15 +88,6 @@ pub async fn launch_online_repair(
garage.block_manager.clone(),
));
}
RepairWhat::Aliases => {
info!("Repairing bucket aliases (foreground)");
garage.locked_helper().await.repair_aliases().await?;
}
RepairWhat::ClearResyncQueue => {
let garage = garage.clone();
tokio::task::spawn_blocking(move || garage.block_manager.resync.clear_resync_queue())
.await??
}
}
Ok(())
}
+4 -15
View File
@@ -183,21 +183,10 @@ fn watch_shutdown_signal() -> watch::Receiver<bool> {
let mut sigterm =
signal(SignalKind::terminate()).expect("Failed to install SIGTERM handler");
let mut sighup = signal(SignalKind::hangup()).expect("Failed to install SIGHUP handler");
loop {
tokio::select! {
_ = sigint.recv() => {
info!("Received SIGINT, shutting down.");
break
}
_ = sigterm.recv() => {
info!("Received SIGTERM, shutting down.");
break
}
_ = sighup.recv() => {
info!("Received SIGHUP, reload not supported.");
continue
}
}
tokio::select! {
_ = sigint.recv() => info!("Received SIGINT, shutting down."),
_ = sigterm.recv() => info!("Received SIGTERM, shutting down."),
_ = sighup.recv() => info!("Received SIGHUP, shutting down."),
}
send_cancel.send(true).unwrap();
});
-2
View File
@@ -63,8 +63,6 @@ rpc_bind_addr = "127.0.0.1:{rpc_port}"
rpc_public_addr = "127.0.0.1:{rpc_port}"
rpc_secret = "{secret}"
allow_punycode = true
[s3_api]
s3_region = "{region}"
api_bind_addr = "127.0.0.1:{s3_port}"
-115
View File
@@ -533,118 +533,3 @@ async fn test_website_check_domain() {
})
);
}
#[tokio::test]
async fn test_website_puny() {
const BCKT_NAME: &str = "xn--pda.eu";
let ctx = common::context();
let bucket = ctx.create_bucket(BCKT_NAME);
let data = ByteStream::from_static(BODY);
ctx.client
.put_object()
.bucket(&bucket)
.key("index.html")
.body(data)
.send()
.await
.unwrap();
let client = Client::builder(TokioExecutor::new()).build_http();
let req = |suffix| {
Request::builder()
.method("GET")
.uri(format!("http://127.0.0.1:{}/", ctx.garage.web_port))
.header("Host", format!("{}{}", BCKT_NAME, suffix))
.body(Body::new(Bytes::new()))
.unwrap()
};
ctx.garage
.command()
.args(["bucket", "website", "--allow", BCKT_NAME])
.quiet()
.expect_success_status("Could not allow website on bucket");
let mut resp = client.request(req("")).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
assert_eq!(
resp.into_body().collect().await.unwrap().to_bytes(),
BODY.as_ref()
);
resp = client.request(req(".web.garage")).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
assert_eq!(
resp.into_body().collect().await.unwrap().to_bytes(),
BODY.as_ref()
);
for bname in [
BCKT_NAME.to_string(),
format!("{BCKT_NAME}.web.garage"),
format!("{BCKT_NAME}.s3.garage"),
] {
let admin_req = || {
Request::builder()
.method("GET")
.uri(format!(
"http://127.0.0.1:{0}/check?domain={1}",
ctx.garage.admin_port, bname
))
.body(Body::new(Bytes::new()))
.unwrap()
};
let admin_resp = client.request(admin_req()).await.unwrap();
assert_eq!(admin_resp.status(), StatusCode::OK);
assert_eq!(
admin_resp.into_body().collect().await.unwrap().to_bytes(),
format!("Domain '{bname}' is managed by Garage").as_bytes()
);
}
}
#[tokio::test]
async fn test_website_object_not_found() {
const BCKT_NAME: &str = "not-found";
let ctx = common::context();
let _bucket = ctx.create_bucket(BCKT_NAME);
let client = Client::builder(TokioExecutor::new()).build_http();
let req = |suffix| {
Request::builder()
.method("GET")
.uri(format!("http://127.0.0.1:{}/", ctx.garage.web_port))
.header("Host", format!("{}{}", BCKT_NAME, suffix))
.body(Body::new(Bytes::new()))
.unwrap()
};
ctx.garage
.command()
.args(["bucket", "website", "--allow", BCKT_NAME])
.quiet()
.expect_success_status("Could not allow website on bucket");
let resp = client.request(req("")).await.unwrap();
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
// the error we return by default are *not* xml
assert_eq!(
resp.headers().get(http::header::CONTENT_TYPE).unwrap(),
"text/html; charset=utf-8"
);
let result = String::from_utf8(
resp.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.to_vec(),
)
.unwrap();
assert!(result.contains("not found"));
}
-10
View File
@@ -72,16 +72,6 @@ impl K2vClient {
.enable_http2()
.build();
let client = HttpClient::builder(TokioExecutor::new()).build(connector);
Self::new_with_client(config, client)
}
/// Create a new K2V client with an external client.
/// Useful for example if you plan on creating many clients but you want to mutualize the
/// underlying thread pools & co.
pub fn new_with_client(
config: K2vClientConfig,
client: HttpClient<HttpsConnector<HttpConnector>, Body>,
) -> Result<Self, Error> {
let user_agent: std::borrow::Cow<str> = match &config.user_agent {
Some(ua) => ua.into(),
None => format!("k2v/{}", env!("CARGO_PKG_VERSION")).into(),
+1 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_model"
version = "1.3.0"
version = "1.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@@ -44,4 +44,3 @@ default = [ "lmdb", "sqlite" ]
k2v = [ "garage_util/k2v" ]
lmdb = [ "garage_db/lmdb" ]
sqlite = [ "garage_db/sqlite" ]
fjall = [ "garage_db/fjall" ]
+10 -8
View File
@@ -22,10 +22,14 @@ mod v08 {
pub use v08::*;
impl BucketAlias {
pub fn new(name: String, ts: u64, bucket_id: Option<Uuid>) -> Self {
BucketAlias {
name,
state: crdt::Lww::raw(ts, bucket_id),
pub fn new(name: String, ts: u64, bucket_id: Option<Uuid>) -> Option<Self> {
if !is_valid_bucket_name(&name) {
None
} else {
Some(BucketAlias {
name,
state: crdt::Lww::raw(ts, bucket_id),
})
}
}
@@ -76,7 +80,7 @@ impl TableSchema for BucketAliasTable {
/// In the case of Garage, bucket names must not be hex-encoded
/// 32 byte string, which is excluded thanks to the
/// maximum length of 63 bytes given in the spec.
pub fn is_valid_bucket_name(n: &str, puny: bool) -> bool {
pub fn is_valid_bucket_name(n: &str) -> bool {
// Bucket names must be between 3 and 63 characters
n.len() >= 3 && n.len() <= 63
// Bucket names must be composed of lowercase letters, numbers,
@@ -88,9 +92,7 @@ pub fn is_valid_bucket_name(n: &str, puny: bool) -> bool {
// Bucket names must not be formatted as an IP address
&& n.parse::<std::net::IpAddr>().is_err()
// Bucket names must not start with "xn--"
&& (!n.starts_with("xn--") || puny)
// We are a bit stricter, to properly restrict punycode in all labels
&& (!n.contains(".xn--") || puny)
&& !n.starts_with("xn--")
// Bucket names must not end with "-s3alias"
&& !n.ends_with("-s3alias")
}
+41 -12
View File
@@ -116,17 +116,21 @@ impl Garage {
info!("Opening database...");
let db_engine = db::Engine::from_str(&config.db_engine)
.ok_or_message("Invalid `db_engine` value in configuration file")?;
let db_path = db_engine.db_path(&config.metadata_dir);
let mut db_path = config.metadata_dir.clone();
match db_engine {
db::Engine::Sqlite => {
db_path.push("db.sqlite");
}
db::Engine::Lmdb => {
db_path.push("db.lmdb");
}
}
let db_opt = db::OpenOpt {
fsync: config.metadata_fsync,
lmdb_map_size: match config.lmdb_map_size {
v if v == usize::default() => None,
v => Some(v),
},
fjall_block_cache_size: match config.fjall_block_cache_size {
v if v == usize::default() => None,
v => Some(v),
},
};
let db = db::open_db(&db_path, db_engine, &db_opt)
.ok_or_message("Unable to open metadata db")?;
@@ -171,7 +175,13 @@ impl Garage {
// ---- admin tables ----
info!("Initialize bucket_table...");
let bucket_table = Table::new(BucketTable, control_rep_param.clone(), system.clone(), &db);
let bucket_table = Table::new(
BucketTable,
control_rep_param.clone(),
system.clone(),
&db,
&config.experimental.merkle_backpressure,
);
info!("Initialize bucket_alias_table...");
let bucket_alias_table = Table::new(
@@ -179,9 +189,16 @@ impl Garage {
control_rep_param.clone(),
system.clone(),
&db,
&config.experimental.merkle_backpressure,
);
info!("Initialize key_table_table...");
let key_table = Table::new(KeyTable, control_rep_param, system.clone(), &db);
let key_table = Table::new(
KeyTable,
control_rep_param,
system.clone(),
&db,
&config.experimental.merkle_backpressure,
);
// ---- S3 tables ----
info!("Initialize block_ref_table...");
@@ -192,6 +209,7 @@ impl Garage {
meta_rep_param.clone(),
system.clone(),
&db,
&config.experimental.merkle_backpressure,
);
info!("Initialize version_table...");
@@ -202,10 +220,12 @@ impl Garage {
meta_rep_param.clone(),
system.clone(),
&db,
&config.experimental.merkle_backpressure,
);
info!("Initialize multipart upload counter table...");
let mpu_counter_table = IndexCounter::new(system.clone(), meta_rep_param.clone(), &db);
let mpu_counter_table =
IndexCounter::new(system.clone(), meta_rep_param.clone(), &db, &config);
info!("Initialize multipart upload table...");
let mpu_table = Table::new(
@@ -216,10 +236,12 @@ impl Garage {
meta_rep_param.clone(),
system.clone(),
&db,
&config.experimental.merkle_backpressure,
);
info!("Initialize object counter table...");
let object_counter_table = IndexCounter::new(system.clone(), meta_rep_param.clone(), &db);
let object_counter_table =
IndexCounter::new(system.clone(), meta_rep_param.clone(), &db, &config);
info!("Initialize object_table...");
#[allow(clippy::redundant_clone)]
@@ -232,6 +254,7 @@ impl Garage {
meta_rep_param.clone(),
system.clone(),
&db,
&config.experimental.merkle_backpressure,
);
info!("Load lifecycle worker state...");
@@ -241,7 +264,7 @@ impl Garage {
// ---- K2V ----
#[cfg(feature = "k2v")]
let k2v = GarageK2V::new(system.clone(), &db, meta_rep_param);
let k2v = GarageK2V::new(system.clone(), &db, meta_rep_param, &config);
// ---- setup block refcount recalculation ----
// this function can be used to fix inconsistencies in the RC table
@@ -331,9 +354,14 @@ impl Garage {
#[cfg(feature = "k2v")]
impl GarageK2V {
fn new(system: Arc<System>, db: &db::Db, meta_rep_param: TableShardedReplication) -> Self {
fn new(
system: Arc<System>,
db: &db::Db,
meta_rep_param: TableShardedReplication,
config: &Config,
) -> Self {
info!("Initialize K2V counter table...");
let counter_table = IndexCounter::new(system.clone(), meta_rep_param.clone(), db);
let counter_table = IndexCounter::new(system.clone(), meta_rep_param.clone(), db, config);
info!("Initialize K2V subscription manager...");
let subscriptions = Arc::new(SubscriptionManager::new());
@@ -347,6 +375,7 @@ impl GarageK2V {
meta_rep_param,
system.clone(),
db,
&config.experimental.merkle_backpressure,
);
info!("Initialize K2V RPC handler...");
+42 -278
View File
@@ -1,7 +1,3 @@
use std::collections::{HashMap, HashSet};
use garage_db as db;
use garage_util::crdt::*;
use garage_util::data::*;
use garage_util::error::{Error as GarageError, OkOrMessage};
@@ -51,10 +47,6 @@ impl<'a> LockedHelper<'a> {
KeyHelper(self.0)
}
// ================================================
// global bucket aliases
// ================================================
/// Sets a new alias for a bucket in global namespace.
/// This function fails if:
/// - alias name is not valid according to S3 spec
@@ -65,7 +57,7 @@ impl<'a> LockedHelper<'a> {
bucket_id: Uuid,
alias_name: &String,
) -> Result<(), Error> {
if !is_valid_bucket_name(alias_name, self.0.config.allow_punycode) {
if !is_valid_bucket_name(alias_name) {
return Err(Error::InvalidBucketName(alias_name.to_string()));
}
@@ -96,7 +88,8 @@ impl<'a> LockedHelper<'a> {
// writes are now done and all writes use timestamp alias_ts
let alias = match alias {
None => BucketAlias::new(alias_name.clone(), alias_ts, Some(bucket_id)),
None => BucketAlias::new(alias_name.clone(), alias_ts, Some(bucket_id))
.ok_or_else(|| Error::InvalidBucketName(alias_name.clone()))?,
Some(mut a) => {
a.state = Lww::raw(alias_ts, Some(bucket_id));
a
@@ -187,14 +180,13 @@ impl<'a> LockedHelper<'a> {
.ok_or_else(|| Error::NoSuchBucket(alias_name.to_string()))?;
// Checks ok, remove alias
let alias_ts = increment_logical_clock_2(
alias.state.timestamp(),
bucket
.state
.as_option()
.map(|p| p.aliases.get_timestamp(alias_name))
.unwrap_or(0),
);
let alias_ts = match bucket.state.as_option() {
Some(bucket_state) => increment_logical_clock_2(
alias.state.timestamp(),
bucket_state.aliases.get_timestamp(alias_name),
),
None => increment_logical_clock(alias.state.timestamp()),
};
// ---- timestamp-ensured causality barrier ----
// writes are now done and all writes use timestamp alias_ts
@@ -212,10 +204,6 @@ impl<'a> LockedHelper<'a> {
Ok(())
}
// ================================================
// local bucket aliases
// ================================================
/// Sets a new alias for a bucket in the local namespace of a key.
/// This function fails if:
/// - alias name is not valid according to S3 spec
@@ -228,12 +216,14 @@ impl<'a> LockedHelper<'a> {
key_id: &String,
alias_name: &String,
) -> Result<(), Error> {
if !is_valid_bucket_name(alias_name, self.0.config.allow_punycode) {
let key_helper = KeyHelper(self.0);
if !is_valid_bucket_name(alias_name) {
return Err(Error::InvalidBucketName(alias_name.to_string()));
}
let mut bucket = self.bucket().get_existing_bucket(bucket_id).await?;
let mut key = self.key().get_existing_key(key_id).await?;
let mut key = key_helper.get_existing_key(key_id).await?;
let key_param = key.state.as_option_mut().unwrap();
@@ -282,13 +272,23 @@ impl<'a> LockedHelper<'a> {
key_id: &String,
alias_name: &String,
) -> Result<(), Error> {
let mut bucket = self.bucket().get_existing_bucket(bucket_id).await?;
let mut key = self.key().get_existing_key(key_id).await?;
let key_helper = KeyHelper(self.0);
let mut bucket = self.bucket().get_existing_bucket(bucket_id).await?;
let mut key = key_helper.get_existing_key(key_id).await?;
let key_p = key.state.as_option().unwrap();
let bucket_p = bucket.state.as_option_mut().unwrap();
if key_p.local_aliases.get(alias_name).cloned().flatten() != Some(bucket_id) {
if key
.state
.as_option()
.unwrap()
.local_aliases
.get(alias_name)
.cloned()
.flatten()
!= Some(bucket_id)
{
return Err(GarageError::Message(format!(
"Bucket {:?} does not have alias {} in namespace of key {}",
bucket_id, alias_name, key_id
@@ -305,17 +305,17 @@ impl<'a> LockedHelper<'a> {
.local_aliases
.items()
.iter()
.any(|((k, n), _, active)| (*k != key.key_id || n != alias_name) && *active);
.any(|((k, n), _, active)| *k == key.key_id && n == alias_name && *active);
if !has_other_global_aliases && !has_other_local_aliases {
return Err(Error::BadRequest(format!("Bucket {} doesn't have other aliases, please delete it instead of just unaliasing.", alias_name)));
}
// Checks ok, remove alias
let key_param = key.state.as_option_mut().unwrap();
let bucket_p_local_alias_key = (key.key_id.clone(), alias_name.clone());
let alias_ts = increment_logical_clock_2(
key_p.local_aliases.get_timestamp(alias_name),
key_param.local_aliases.get_timestamp(alias_name),
bucket_p
.local_aliases
.get_timestamp(&bucket_p_local_alias_key),
@@ -324,8 +324,7 @@ impl<'a> LockedHelper<'a> {
// ---- timestamp-ensured causality barrier ----
// writes are now done and all writes use timestamp alias_ts
key.state.as_option_mut().unwrap().local_aliases =
LwwMap::raw_item(alias_name.clone(), alias_ts, None);
key_param.local_aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, None);
self.0.key_table.insert(&key).await?;
bucket_p.local_aliases = LwwMap::raw_item(bucket_p_local_alias_key, alias_ts, false);
@@ -334,68 +333,21 @@ impl<'a> LockedHelper<'a> {
Ok(())
}
/// Ensures a bucket does not have a certain local alias.
/// Contrarily to unset_local_bucket_alias, this does not
/// fail on any condition other than:
/// - bucket cannot be found (its fine if it is in deleted state)
/// - key cannot be found (its fine if alias in key points to nothing
/// or to another bucket)
pub async fn purge_local_bucket_alias(
&self,
bucket_id: Uuid,
key_id: &String,
alias_name: &String,
) -> Result<(), Error> {
let mut bucket = self.bucket().get_internal_bucket(bucket_id).await?;
let mut key = self.key().get_internal_key(key_id).await?;
let bucket_p_local_alias_key = (key.key_id.clone(), alias_name.clone());
let alias_ts = increment_logical_clock_2(
key.state
.as_option()
.map(|p| p.local_aliases.get_timestamp(alias_name))
.unwrap_or(0),
bucket
.state
.as_option()
.map(|p| p.local_aliases.get_timestamp(&bucket_p_local_alias_key))
.unwrap_or(0),
);
// ---- timestamp-ensured causality barrier ----
// writes are now done and all writes use timestamp alias_ts
if let Some(kp) = key.state.as_option_mut() {
kp.local_aliases = LwwMap::raw_item(alias_name.clone(), alias_ts, None);
self.0.key_table.insert(&key).await?;
}
if let Some(bp) = bucket.state.as_option_mut() {
bp.local_aliases = LwwMap::raw_item(bucket_p_local_alias_key, alias_ts, false);
self.0.bucket_table.insert(&bucket).await?;
}
Ok(())
}
// ================================================
// permissions
// ================================================
/// Sets permissions for a key on a bucket.
/// This function fails if:
/// - bucket or key cannot be found at all (its ok if they are in deleted state)
/// - bucket or key is in deleted state and we are trying to set
/// permissions other than "deny all"
/// - bucket or key is in deleted state and we are trying to set permissions other than "deny
/// all"
pub async fn set_bucket_key_permissions(
&self,
bucket_id: Uuid,
key_id: &String,
mut perm: BucketKeyPerm,
) -> Result<(), Error> {
let key_helper = KeyHelper(self.0);
let mut bucket = self.bucket().get_internal_bucket(bucket_id).await?;
let mut key = self.key().get_internal_key(key_id).await?;
let mut key = key_helper.get_internal_key(key_id).await?;
if let Some(bstate) = bucket.state.as_option() {
if let Some(kp) = bstate.authorized_keys.get(key_id) {
@@ -432,20 +384,21 @@ impl<'a> LockedHelper<'a> {
Ok(())
}
// ================================================
// keys
// ================================================
// ----
/// Deletes an API access key
pub async fn delete_key(&self, key: &mut Key) -> Result<(), Error> {
let state = key.state.as_option_mut().unwrap();
// --- done checking, now commit ---
// (the step at unset_local_bucket_alias will fail if a bucket
// does not have another alias, the deletion will be
// interrupted in the middle if that happens)
// 1. Delete local aliases
for (alias, _, to) in state.local_aliases.items().iter() {
if let Some(bucket_id) = to {
self.purge_local_bucket_alias(*bucket_id, &key.key_id, alias)
self.unset_local_bucket_alias(*bucket_id, &key.key_id, alias)
.await?;
}
}
@@ -462,193 +415,4 @@ impl<'a> LockedHelper<'a> {
Ok(())
}
// ================================================
// repair procedure
// ================================================
pub async fn repair_aliases(&self) -> Result<(), GarageError> {
self.0.db.transaction(|tx| {
info!("--- begin repair_aliases transaction ----");
// 1. List all non-deleted buckets, so that we can fix bad aliases
let mut all_buckets: HashSet<Uuid> = HashSet::new();
for item in tx.range::<&[u8], _>(&self.0.bucket_table.data.store, ..)? {
let bucket = self
.0
.bucket_table
.data
.decode_entry(&(item?.1))
.map_err(db::TxError::Abort)?;
if !bucket.is_deleted() {
all_buckets.insert(bucket.id);
}
}
info!("number of buckets: {}", all_buckets.len());
// 2. List all aliases declared in bucket_alias_table and key_table
// Take note of aliases that point to non-existing buckets
let mut global_aliases: HashMap<String, Uuid> = HashMap::new();
{
let mut delete_global = vec![];
for item in tx.range::<&[u8], _>(&self.0.bucket_alias_table.data.store, ..)? {
let mut alias = self
.0
.bucket_alias_table
.data
.decode_entry(&(item?.1))
.map_err(db::TxError::Abort)?;
if let Some(id) = alias.state.get() {
if all_buckets.contains(id) {
// keep aliases
global_aliases.insert(alias.name().to_string(), *id);
} else {
// delete alias
warn!(
"global alias: remove {} -> {:?} (bucket is deleted)",
alias.name(),
id
);
alias.state.update(None);
delete_global.push(alias);
}
}
}
info!("number of global aliases: {}", global_aliases.len());
info!("global alias table: {} entries fixed", delete_global.len());
for ga in delete_global {
debug!("Enqueue update to global alias table: {:?}", ga);
self.0.bucket_alias_table.queue_insert(tx, &ga)?;
}
}
let mut local_aliases: HashMap<(String, String), Uuid> = HashMap::new();
{
let mut delete_local = vec![];
for item in tx.range::<&[u8], _>(&self.0.key_table.data.store, ..)? {
let mut key = self
.0
.key_table
.data
.decode_entry(&(item?.1))
.map_err(db::TxError::Abort)?;
let Some(p) = key.state.as_option_mut() else {
continue;
};
let mut has_changes = false;
for (name, _, to) in p.local_aliases.items().to_vec() {
if let Some(id) = to {
if all_buckets.contains(&id) {
local_aliases.insert((key.key_id.clone(), name), id);
} else {
warn!(
"local alias: remove ({}, {}) -> {:?} (bucket is deleted)",
key.key_id, name, id
);
p.local_aliases.update_in_place(name, None);
has_changes = true;
}
}
}
if has_changes {
delete_local.push(key);
}
}
info!("number of local aliases: {}", local_aliases.len());
info!("key table: {} entries fixed", delete_local.len());
for la in delete_local {
debug!("Enqueue update to key table: {:?}", la);
self.0.key_table.queue_insert(tx, &la)?;
}
}
// 4. Reverse the alias maps to determine the aliases per-bucket
let mut bucket_global: HashMap<Uuid, Vec<String>> = HashMap::new();
let mut bucket_local: HashMap<Uuid, Vec<(String, String)>> = HashMap::new();
for (name, bucket) in global_aliases {
bucket_global.entry(bucket).or_default().push(name);
}
for ((key, name), bucket) in local_aliases {
bucket_local.entry(bucket).or_default().push((key, name));
}
// 5. Fix the bucket table to ensure consistency
let mut bucket_updates = vec![];
for item in tx.range::<&[u8], _>(&self.0.bucket_table.data.store, ..)? {
let bucket = self
.0
.bucket_table
.data
.decode_entry(&(item?.1))
.map_err(db::TxError::Abort)?;
let mut bucket2 = bucket.clone();
let Some(param) = bucket2.state.as_option_mut() else {
continue;
};
// fix global aliases
{
let ga = bucket_global.remove(&bucket.id).unwrap_or_default();
for (name, _, active) in param.aliases.items().to_vec() {
if active && !ga.contains(&name) {
warn!("bucket {:?}: remove global alias {}", bucket.id, name);
param.aliases.update_in_place(name, false);
}
}
for name in ga {
if param.aliases.get(&name).copied() != Some(true) {
warn!("bucket {:?}: add global alias {}", bucket.id, name);
param.aliases.update_in_place(name, true);
}
}
}
// fix local aliases
{
let la = bucket_local.remove(&bucket.id).unwrap_or_default();
for (pair, _, active) in param.local_aliases.items().to_vec() {
if active && !la.contains(&pair) {
warn!("bucket {:?}: remove local alias {:?}", bucket.id, pair);
param.local_aliases.update_in_place(pair, false);
}
}
for pair in la {
if param.local_aliases.get(&pair).copied() != Some(true) {
warn!("bucket {:?}: add local alias {:?}", bucket.id, pair);
param.local_aliases.update_in_place(pair, true);
}
}
}
if bucket2 != bucket {
bucket_updates.push(bucket2);
}
}
info!("bucket table: {} entries fixed", bucket_updates.len());
for b in bucket_updates {
debug!("Enqueue update to bucket table: {:?}", b);
self.0.bucket_table.queue_insert(tx, &b)?;
}
info!("--- end repair_aliases transaction ----");
Ok(())
})?;
info!("repair_aliases is done");
Ok(())
}
}
+3
View File
@@ -10,6 +10,7 @@ use garage_db as db;
use garage_rpc::layout::LayoutHelper;
use garage_rpc::system::System;
use garage_util::background::BackgroundRunner;
use garage_util::config::Config;
use garage_util::data::*;
use garage_util::error::*;
use garage_util::migrate::Migrate;
@@ -173,6 +174,7 @@ impl<T: CountedItem> IndexCounter<T> {
system: Arc<System>,
replication: TableShardedReplication,
db: &db::Db,
config: &Config,
) -> Arc<Self> {
Arc::new(Self {
this_node: system.id,
@@ -186,6 +188,7 @@ impl<T: CountedItem> IndexCounter<T> {
replication,
system,
db,
&config.experimental.merkle_backpressure,
),
})
}
+3 -3
View File
@@ -121,13 +121,13 @@ impl Worker for LifecycleWorker {
mpu_aborted,
..
} => {
let n_objects = self.garage.object_table.data.store.approximate_len().ok();
let n_objects = self.garage.object_table.data.store.len().ok();
let progress = match n_objects {
Some(total) if total > 0 => format!(
None => "...".to_string(),
Some(total) => format!(
"~{:.2}%",
100. * std::cmp::min(*counter, total) as f32 / total as f32
),
_ => "...".to_string(),
};
WorkerStatus {
progress: Some(progress),
+2 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_net"
version = "1.3.0"
version = "1.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@@ -39,6 +39,7 @@ kuska-handshake.workspace = true
opentelemetry = { workspace = true, optional = true }
opentelemetry-contrib = { workspace = true, optional = true }
tracing.workspace = true
[dev-dependencies]
pretty_env_logger.workspace = true
+22 -1
View File
@@ -4,6 +4,7 @@ use std::pin::Pin;
use std::sync::atomic::{self, AtomicU32};
use std::sync::{Arc, Mutex};
use std::task::Poll;
use tracing::*;
use arc_swap::ArcSwapOption;
use bytes::Bytes;
@@ -14,7 +15,7 @@ use futures::Stream;
use kuska_handshake::async_std::{handshake_client, BoxStream};
use tokio::net::TcpStream;
use tokio::select;
use tokio::sync::{mpsc, oneshot, watch};
use tokio::sync::{mpsc, oneshot, watch, Semaphore};
use tokio_util::compat::*;
#[cfg(feature = "telemetry")]
@@ -25,6 +26,7 @@ use opentelemetry::{
#[cfg(feature = "telemetry")]
use opentelemetry_contrib::trace::propagator::binary::*;
use crate::endpoint::RpcInFlightLimiter;
use crate::error::*;
use crate::message::*;
use crate::netapp::*;
@@ -41,6 +43,7 @@ pub(crate) struct ClientConn {
next_query_number: AtomicU32,
inflight: Mutex<HashMap<RequestID, oneshot::Sender<ByteStream>>>,
rpc_table_write_inflight_limiter: Option<Semaphore>,
}
impl ClientConn {
@@ -98,8 +101,14 @@ impl ClientConn {
next_query_number: AtomicU32::from(RequestID::default()),
query_send: ArcSwapOption::new(Some(Arc::new(query_send))),
inflight: Mutex::new(HashMap::new()),
rpc_table_write_inflight_limiter: netapp.max_in_flight_table_write.map(Semaphore::new),
});
info!(
"Created conn with table write limit set to {}",
netapp.max_in_flight_table_write.unwrap_or(0)
);
netapp.connected_as_client(peer_id, conn.clone());
let debug_name = format!("CLI {}", hex::encode(&peer_id[..8]));
@@ -144,10 +153,21 @@ impl ClientConn {
req: Req<T>,
path: &str,
prio: RequestPriority,
limiter: RpcInFlightLimiter,
) -> Result<Resp<T>, Error>
where
T: Message,
{
let _permit = match (limiter, &self.rpc_table_write_inflight_limiter) {
(RpcInFlightLimiter::TableWrite, Some(sem)) => {
info!(
"Available RPC table write slots: {}",
sem.available_permits()
);
Some(sem.acquire().await.unwrap())
}
_ => None,
};
let query_send = self.query_send.load_full().ok_or(Error::ConnectionClosed)?;
let id = self
@@ -212,6 +232,7 @@ impl ClientConn {
let stream = Box::pin(canceller.for_stream(stream));
let resp_enc = RespEnc::decode(stream).await?;
drop(_permit);
debug!("client: got response to request {} (path {})", id, path);
Resp::from_enc(resp_enc)
}
+17 -2
View File
@@ -57,6 +57,13 @@ where
}
}
#[derive(Debug, Copy, Clone, Default)]
pub enum RpcInFlightLimiter {
#[default]
NoLimit,
TableWrite,
}
// ----
/// This struct represents an endpoint for message of type `M`.
@@ -114,6 +121,7 @@ where
target: &NodeID,
req: T,
prio: RequestPriority,
limiter: RpcInFlightLimiter,
) -> Result<Resp<M>, Error>
where
T: IntoReq<M>,
@@ -136,7 +144,10 @@ where
"Not connected: {}",
hex::encode(&target[..8])
))),
Some(c) => c.call(req.into_req()?, self.path.as_str(), prio).await,
Some(c) => {
c.call(req.into_req()?, self.path.as_str(), prio, limiter)
.await
}
}
}
}
@@ -149,8 +160,12 @@ where
target: &NodeID,
req: M,
prio: RequestPriority,
limiter: RpcInFlightLimiter,
) -> Result<<M as Message>::Response, Error> {
Ok(self.call_streaming(target, req, prio).await?.into_msg())
Ok(self
.call_streaming(target, req, prio, limiter)
.await?
.into_msg())
}
}
+5
View File
@@ -74,6 +74,8 @@ pub struct NetApp {
pub id: NodeID,
/// Private key associated with our peer ID
pub privkey: ed25519::SecretKey,
/// Config related to netapp
pub(crate) max_in_flight_table_write: Option<usize>,
pub(crate) server_conns: RwLock<HashMap<NodeID, Arc<ServerConn>>>,
pub(crate) client_conns: RwLock<HashMap<NodeID, Arc<ClientConn>>>,
@@ -101,6 +103,7 @@ impl NetApp {
netid: auth::Key,
privkey: ed25519::SecretKey,
bind_outgoing_to: Option<IpAddr>,
max_in_flight_table_write: Option<usize>,
) -> Arc<Self> {
let mut version_tag = [0u8; 16];
version_tag[0..8].copy_from_slice(&u64::to_be_bytes(NETAPP_VERSION_TAG)[..]);
@@ -114,6 +117,7 @@ impl NetApp {
netid,
id,
privkey,
max_in_flight_table_write,
server_conns: RwLock::new(HashMap::new()),
client_conns: RwLock::new(HashMap::new()),
endpoints: RwLock::new(HashMap::new()),
@@ -427,6 +431,7 @@ impl NetApp {
server_port,
},
PRIO_NORMAL,
RpcInFlightLimiter::NoLimit,
)
.await
.map(|_| ())
+7 -2
View File
@@ -406,7 +406,7 @@ impl PeeringManager {
ping_time
);
let ping_response = select! {
r = self.ping_endpoint.call(&id, ping_msg, PRIO_HIGH) => r,
r = self.ping_endpoint.call(&id, ping_msg, PRIO_HIGH, RpcInFlightLimiter::NoLimit) => r,
_ = tokio::time::sleep(ping_timeout) => Err(Error::Message("Ping timeout".into())),
};
@@ -458,7 +458,12 @@ impl PeeringManager {
let pex_message = PeerListMessage { list: peer_list };
match self
.peer_list_endpoint
.call(id, pex_message, PRIO_BACKGROUND)
.call(
id,
pex_message,
PRIO_BACKGROUND,
RpcInFlightLimiter::NoLimit,
)
.await
{
Err(e) => warn!("Error doing peer exchange: {}", e),
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_rpc"
version = "1.3.0"
version = "1.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+8 -6
View File
@@ -229,11 +229,13 @@ impl LayoutManager {
}
/// Save cluster layout data to disk
async fn save_cluster_layout(&self) {
async fn save_cluster_layout(&self) -> Result<(), Error> {
let layout = self.layout.read().unwrap().inner().clone();
if let Err(e) = self.persist_cluster_layout.save_async(&layout).await {
error!("Failed to save cluster_layout: {}", e);
}
self.persist_cluster_layout
.save_async(&layout)
.await
.expect("Cannot save current cluster layout");
Ok(())
}
fn broadcast_update(self: &Arc<Self>, rpc: SystemRpc) {
@@ -311,7 +313,7 @@ impl LayoutManager {
self.change_notify.notify_waiters();
self.broadcast_update(SystemRpc::AdvertiseClusterLayout(new_layout));
self.save_cluster_layout().await;
self.save_cluster_layout().await?;
}
Ok(SystemRpc::Ok)
@@ -326,7 +328,7 @@ impl LayoutManager {
if let Some(new_trackers) = self.merge_layout_trackers(trackers) {
self.change_notify.notify_waiters();
self.broadcast_update(SystemRpc::AdvertiseClusterLayoutTrackers(new_trackers));
self.save_cluster_layout().await;
self.save_cluster_layout().await?;
}
Ok(SystemRpc::Ok)
+12 -1
View File
@@ -6,6 +6,7 @@ use std::time::Duration;
use futures::future::join_all;
use futures::stream::futures_unordered::FuturesUnordered;
use futures::stream::StreamExt;
use garage_net::endpoint::RpcInFlightLimiter;
use tokio::select;
use opentelemetry::KeyValue;
@@ -44,6 +45,8 @@ pub struct RequestStrategy<T> {
rs_timeout: Timeout,
/// Data to drop when everything completes
rs_drop_on_complete: T,
/// RPC In Flight Limiter
rs_inflight_limiter: RpcInFlightLimiter,
}
#[derive(Copy, Clone)]
@@ -61,6 +64,7 @@ impl Clone for RequestStrategy<()> {
rs_priority: self.rs_priority,
rs_timeout: self.rs_timeout,
rs_drop_on_complete: (),
rs_inflight_limiter: self.rs_inflight_limiter,
}
}
}
@@ -74,6 +78,7 @@ impl RequestStrategy<()> {
rs_priority: prio,
rs_timeout: Timeout::Default,
rs_drop_on_complete: (),
rs_inflight_limiter: RpcInFlightLimiter::NoLimit,
}
}
/// Add an item to be dropped on completion
@@ -84,6 +89,7 @@ impl RequestStrategy<()> {
rs_priority: self.rs_priority,
rs_timeout: self.rs_timeout,
rs_drop_on_complete: drop_on_complete,
rs_inflight_limiter: RpcInFlightLimiter::NoLimit,
}
}
}
@@ -109,6 +115,10 @@ impl<T> RequestStrategy<T> {
self.rs_timeout = Timeout::Custom(timeout);
self
}
pub fn with_write_limiter(mut self) -> Self {
self.rs_inflight_limiter = RpcInFlightLimiter::TableWrite;
self
}
/// Extract drop_on_complete item
fn extract_drop_on_complete(self) -> (RequestStrategy<()>, T) {
(
@@ -118,6 +128,7 @@ impl<T> RequestStrategy<T> {
rs_priority: self.rs_priority,
rs_timeout: self.rs_timeout,
rs_drop_on_complete: (),
rs_inflight_limiter: self.rs_inflight_limiter,
},
self.rs_drop_on_complete,
)
@@ -185,7 +196,7 @@ impl RpcHelper {
let node_id = to.into();
let rpc_call = endpoint
.call_streaming(&node_id, msg, strat.rs_priority)
.call_streaming(&node_id, msg, strat.rs_priority, strat.rs_inflight_limiter)
.with_context(Context::current_with_span(span))
.record_duration(&self.0.metrics.rpc_duration, &metric_tags);
+12 -2
View File
@@ -21,7 +21,7 @@ use garage_net::{NetApp, NetworkKey, NodeID, NodeKey};
#[cfg(feature = "kubernetes-discovery")]
use garage_util::config::KubernetesDiscoveryConfig;
use garage_util::config::{Config, DataDirEnum};
use garage_util::config::{Config, DataDirEnum, RpcInFlightLimiterEnum};
use garage_util::data::*;
use garage_util::error::*;
use garage_util::persister::Persister;
@@ -256,7 +256,17 @@ impl System {
let bind_outgoing_to = Some(config)
.filter(|x| x.rpc_bind_outgoing)
.map(|x| x.rpc_bind_addr.ip());
let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, node_key, bind_outgoing_to);
let maybe_max_table_write = match &config.experimental.rpc_in_flight_limiters {
RpcInFlightLimiterEnum::None => None,
RpcInFlightLimiterEnum::FixedSize(v) => Some(v.max_table_write),
};
let netapp = NetApp::new(
GARAGE_VERSION_TAG,
network_key,
node_key,
bind_outgoing_to,
maybe_max_table_write,
);
let system_endpoint = netapp.endpoint(SYSTEM_RPC_PATH.into());
// ---- setup netapp public listener and full mesh peering strategy ----
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_table"
version = "1.3.0"
version = "1.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+135 -37
View File
@@ -3,10 +3,12 @@ use std::convert::TryInto;
use std::sync::Arc;
use serde_bytes::ByteBuf;
use tokio::sync::Notify;
use tokio::sync::SemaphorePermit;
use tokio::sync::{Notify, Semaphore};
use garage_db as db;
use garage_util::config::MerkleBackpressureEnum;
use garage_util::data::*;
use garage_util::error::*;
use garage_util::migrate::Migrate;
@@ -20,6 +22,67 @@ use crate::replication::*;
use crate::schema::*;
use crate::util::*;
pub(crate) struct MerkleTodo {
merkle_todo: db::Tree,
merkle_todo_notify: Notify,
merkle_todo_bounded_queue: Option<Arc<Semaphore>>,
}
impl Clone for MerkleTodo {
fn clone(&self) -> Self {
Self {
merkle_todo: self.merkle_todo.clone(),
merkle_todo_notify: Notify::new(),
merkle_todo_bounded_queue: self.merkle_todo_bounded_queue.clone(),
}
}
}
impl MerkleTodo {
fn new<F: TableSchema>(db: &db::Db, config: &MerkleBackpressureEnum) -> Self {
let merkle_todo = db
.open_tree(format!("{}:merkle_todo", F::TABLE_NAME))
.expect("Unable to open DB Merkle TODO tree");
let merkle_todo_bounded_queue = match config {
MerkleBackpressureEnum::None => None,
MerkleBackpressureEnum::FixedQueue(p) => {
Some(Arc::new(Semaphore::new(p.max_queue_size)))
}
};
Self {
merkle_todo,
merkle_todo_notify: Notify::new(),
merkle_todo_bounded_queue,
}
}
pub(crate) fn len(&self) -> Result<usize, db::Error> {
self.merkle_todo.len()
}
pub(crate) async fn with_db<F: FnOnce(&db::Tree, SemaphorePermit)>(&self, f: F) {
let bounded = self
.merkle_todo_bounded_queue
.clone()
.unwrap_or(Arc::new(Semaphore::new(1)));
let permit = bounded.acquire().await.unwrap();
f(&self.merkle_todo, permit);
}
pub(crate) fn appended(&self, permit: SemaphorePermit) {
permit.forget();
self.merkle_todo_notify.notify_one();
}
pub(crate) fn processed(&self) {
let bounded = self
.merkle_todo_bounded_queue
.clone()
.unwrap_or(Arc::new(Semaphore::new(1)));
bounded.add_permits(1);
}
}
pub struct TableData<F: TableSchema, R: TableReplication> {
system: Arc<System>,
@@ -29,8 +92,7 @@ pub struct TableData<F: TableSchema, R: TableReplication> {
pub store: db::Tree,
pub(crate) merkle_tree: db::Tree,
pub(crate) merkle_todo: db::Tree,
pub(crate) merkle_todo_notify: Notify,
pub(crate) merkle_todo: MerkleTodo,
pub(crate) insert_queue: db::Tree,
pub(crate) insert_queue_notify: Arc<Notify>,
@@ -38,10 +100,18 @@ pub struct TableData<F: TableSchema, R: TableReplication> {
pub(crate) gc_todo: db::Tree,
pub(crate) metrics: TableMetrics,
pub(crate) config: MerkleBackpressureEnum,
}
impl<F: TableSchema, R: TableReplication> TableData<F, R> {
pub fn new(system: Arc<System>, instance: F, replication: R, db: &db::Db) -> Arc<Self> {
pub fn new(
system: Arc<System>,
instance: F,
replication: R,
db: &db::Db,
config: &MerkleBackpressureEnum,
) -> Arc<Self> {
let store = db
.open_tree(format!("{}:table", F::TABLE_NAME))
.expect("Unable to open DB tree");
@@ -49,9 +119,8 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
let merkle_tree = db
.open_tree(format!("{}:merkle_tree", F::TABLE_NAME))
.expect("Unable to open DB Merkle tree tree");
let merkle_todo = db
.open_tree(format!("{}:merkle_todo", F::TABLE_NAME))
.expect("Unable to open DB Merkle TODO tree");
let merkle_todo = MerkleTodo::new::<F>(db, config);
let insert_queue = db
.open_tree(format!("{}:insert_queue", F::TABLE_NAME))
@@ -76,11 +145,11 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
store,
merkle_tree,
merkle_todo,
merkle_todo_notify: Notify::new(),
insert_queue,
insert_queue_notify: Arc::new(Notify::new()),
gc_todo,
metrics,
config: config.clone(),
})
}
@@ -167,6 +236,8 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
// - When an entry is modified or deleted, add it to the merkle updater's todo list.
// This has to be done atomically with the modification for the merkle updater
// to maintain consistency. The merkle updater must then be notified with todo_notify.
// Also to avoid overloading the merkle updater, you need to sleep a given amount of
// time to enable backpressure (ie. slow down clients).
// - When an entry is updated to be a tombstone, add it to the gc_todo tree
pub(crate) fn update_many<T: Borrow<ByteBuf>>(&self, entries: &[T]) -> Result<(), Error> {
@@ -201,6 +272,7 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
) -> Result<Option<F::E>, Error> {
let tree_key = self.tree_key(partition_key, sort_key);
// transaction begins
let changed = self.store.db().transaction(|tx| {
let (old_entry, old_bytes, new_entry) = match tx.get(&self.store, &tree_key)? {
Some(old_bytes) => {
@@ -238,31 +310,44 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
Ok(None)
}
})?;
// transaction ends
if let Some((new_entry, new_bytes_hash)) = changed {
self.metrics.internal_update_counter.add(1);
let is_tombstone = new_entry.is_tombstone();
self.merkle_todo_notify.notify_one();
if is_tombstone {
// We are only responsible for GC'ing this item if we are the
// "leader" of the partition, i.e. the first node in the
// set of nodes that replicates this partition.
// This avoids GC loops and does not change the termination properties
// of the GC algorithm, as in all cases GC is suspended if
// any node of the partition is unavailable.
let pk_hash = Hash::try_from(&tree_key[..32]).unwrap();
// TODO: this probably breaks when the layout changes
let nodes = self.replication.storage_nodes(&pk_hash);
if nodes.first() == Some(&self.system.id) {
GcTodoEntry::new(tree_key, new_bytes_hash).save(&self.gc_todo)?;
// early return if nothing changed
let (new_entry, new_bytes_hash) = match changed {
Some((e, b)) => (e, b),
None => {
let maybe_bound = self.merkle_todo_bounded_queue.clone();
if let Some(b) = &maybe_bound {
b.add_permits(1);
}
return Ok(None);
}
};
Ok(Some(new_entry))
} else {
Ok(None)
// Handle GC in case of tombstone
let is_tombstone = new_entry.is_tombstone();
if is_tombstone {
// We are only responsible for GC'ing this item if we are the
// "leader" of the partition, i.e. the first node in the
// set of nodes that replicates this partition.
// This avoids GC loops and does not change the termination properties
// of the GC algorithm, as in all cases GC is suspended if
// any node of the partition is unavailable.
let pk_hash = Hash::try_from(&tree_key[..32]).unwrap();
// TODO: this probably breaks when the layout changes
let nodes = self.replication.storage_nodes(&pk_hash);
if nodes.first() == Some(&self.system.id) {
GcTodoEntry::new(tree_key, new_bytes_hash).save(&self.gc_todo)?;
}
}
// Collect metrics
self.metrics.internal_update_counter.add(1);
// Synchronize with the Merkle Worker
self.merkle_todo_notify.notify_one(); // Wake-up it
Ok(Some(new_entry))
}
pub(crate) fn delete_if_equal(self: &Arc<Self>, k: &[u8], v: &[u8]) -> Result<bool, Error> {
@@ -282,10 +367,16 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
_ => Ok(false),
})?;
if removed {
self.metrics.internal_delete_counter.add(1);
self.merkle_todo_notify.notify_one();
if !removed {
let maybe_bound = self.merkle_todo_bounded_queue.clone();
if let Some(b) = &maybe_bound {
b.add_permits(1);
}
return Ok(false);
}
self.metrics.internal_delete_counter.add(1);
self.merkle_todo_notify.notify_one();
Ok(removed)
}
@@ -310,11 +401,18 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
_ => Ok(false),
})?;
if removed {
self.metrics.internal_delete_counter.add(1);
self.merkle_todo_notify.notify_one();
if !removed {
let maybe_bound = self.merkle_todo_bounded_queue.clone();
if let Some(b) = &maybe_bound {
b.add_permits(1);
}
return Ok(false);
}
Ok(removed)
self.metrics.internal_delete_counter.add(1);
self.merkle_todo_notify.notify_one();
Ok(true)
}
// ---- Insert queue functions ----
@@ -367,7 +465,7 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
}
}
pub fn gc_todo_approximate_len(&self) -> Result<usize, Error> {
Ok(self.gc_todo.approximate_len()?)
pub fn gc_todo_len(&self) -> Result<usize, Error> {
Ok(self.gc_todo.len()?)
}
}
+11 -4
View File
@@ -262,7 +262,8 @@ impl<F: TableSchema, R: TableReplication> TableGc<F, R> {
// GC has been successful for all of these entries.
// We now remove them all from our local table and from the GC todo list.
for item in items {
self.data
let _is_removed = self
.data
.delete_if_equal_hash(&item.key[..], item.value_hash)
.err_context("GC: local delete tombstones")?;
item.remove_if_equal(&self.data.gc_todo)
@@ -275,14 +276,21 @@ impl<F: TableSchema, R: TableReplication> TableGc<F, R> {
impl<F: TableSchema, R: TableReplication> EndpointHandler<GcRpc> for TableGc<F, R> {
async fn handle(self: &Arc<Self>, message: &GcRpc, _from: NodeID) -> Result<GcRpc, Error> {
let maybe_bounded = self.data.merkle_todo_bounded_queue.clone();
match message {
GcRpc::Update(items) => {
if let Some(b) = maybe_bounded {
b.acquire_many(items.len() as u32).await.unwrap().forget();
}
self.data.update_many(items)?;
Ok(GcRpc::Ok)
}
GcRpc::DeleteIfEqualHash(items) => {
if let Some(b) = maybe_bounded {
b.acquire_many(items.len() as u32).await.unwrap().forget();
}
for (key, vhash) in items.iter() {
self.data.delete_if_equal_hash(&key[..], *vhash)?;
let _is_removed = self.data.delete_if_equal_hash(&key[..], *vhash)?;
}
Ok(GcRpc::Ok)
}
@@ -313,7 +321,7 @@ impl<F: TableSchema, R: TableReplication> Worker for GcWorker<F, R> {
fn status(&self) -> WorkerStatus {
WorkerStatus {
queue_length: Some(self.gc.data.gc_todo_approximate_len().unwrap_or(0) as u64),
queue_length: Some(self.gc.data.gc_todo_len().unwrap_or(0) as u64),
..Default::default()
}
}
@@ -329,7 +337,6 @@ impl<F: TableSchema, R: TableReplication> Worker for GcWorker<F, R> {
}
async fn wait_for_work(&mut self) -> WorkerState {
tokio::time::sleep(self.wait_delay).await;
WorkerState::Busy
}
}
+20 -5
View File
@@ -9,6 +9,7 @@ use tokio::sync::watch;
use garage_db as db;
use garage_util::background::*;
use garage_util::config::MerkleBackpressureEnum;
use garage_util::data::*;
use garage_util::encode::{nonversioned_decode, nonversioned_encode};
use garage_util::error::Error;
@@ -70,6 +71,15 @@ impl<F: TableSchema, R: TableReplication> MerkleUpdater<F, R> {
pub(crate) fn new(data: Arc<TableData<F, R>>) -> Arc<Self> {
let empty_node_hash = blake2sum(&nonversioned_encode(&MerkleNode::Empty).unwrap()[..]);
// @FIXME: move in worker
match &data.config {
MerkleBackpressureEnum::None => info!("Merkle Backpressure is not activated"),
MerkleBackpressureEnum::FixedQueue(v) => info!(
"Merkle backpressure with a fixed queue size (qlen={}) is activated.",
v.max_queue_size
),
}
Arc::new(Self {
data,
empty_node_hash,
@@ -125,6 +135,11 @@ impl<F: TableSchema, R: TableReplication> MerkleUpdater<F, R> {
k
);
}
let maybe_bound = self.data.merkle_todo_bounded_queue.clone();
if let Some(b) = &maybe_bound {
b.add_permits(1);
}
Ok(())
}
@@ -287,12 +302,12 @@ impl<F: TableSchema, R: TableReplication> MerkleUpdater<F, R> {
MerkleNode::decode_opt(&ent)
}
pub fn merkle_tree_approximate_len(&self) -> Result<usize, Error> {
Ok(self.data.merkle_tree.approximate_len()?)
pub fn merkle_tree_len(&self) -> Result<usize, Error> {
Ok(self.data.merkle_tree.len()?)
}
pub fn todo_approximate_len(&self) -> Result<usize, Error> {
Ok(self.data.merkle_todo.approximate_len()?)
pub fn todo_len(&self) -> Result<usize, Error> {
Ok(self.data.merkle_todo.len()?)
}
}
@@ -306,7 +321,7 @@ impl<F: TableSchema, R: TableReplication> Worker for MerkleWorker<F, R> {
fn status(&self) -> WorkerStatus {
WorkerStatus {
queue_length: Some(self.0.todo_approximate_len().unwrap_or(0) as u64),
queue_length: Some(self.0.todo_len().unwrap_or(0) as u64),
..Default::default()
}
}
+23 -5
View File
@@ -1,12 +1,16 @@
use opentelemetry::{global, metrics::*, KeyValue};
use std::convert::TryInto;
use garage_db as db;
use crate::data::MerkleTodo;
/// TableMetrics reference all counter used for metrics
pub struct TableMetrics {
pub(crate) _table_size: ValueObserver<u64>,
pub(crate) _merkle_tree_size: ValueObserver<u64>,
pub(crate) _merkle_todo_len: ValueObserver<u64>,
pub(crate) _merkle_todo_bounded_queue_free: ValueObserver<u64>,
pub(crate) _gc_todo_len: ValueObserver<u64>,
pub(crate) get_request_counter: BoundCounter<u64>,
@@ -25,7 +29,7 @@ impl TableMetrics {
table_name: &'static str,
store: db::Tree,
merkle_tree: db::Tree,
merkle_todo: db::Tree,
merkle_todo: MerkleTodo,
gc_todo: db::Tree,
) -> Self {
let meter = global::meter(table_name);
@@ -34,7 +38,7 @@ impl TableMetrics {
.u64_value_observer(
"table.size",
move |observer| {
if let Ok(value) = store.approximate_len() {
if let Ok(value) = store.len() {
observer.observe(
value as u64,
&[KeyValue::new("table_name", table_name)],
@@ -48,7 +52,7 @@ impl TableMetrics {
.u64_value_observer(
"table.merkle_tree_size",
move |observer| {
if let Ok(value) = merkle_tree.approximate_len() {
if let Ok(value) = merkle_tree.len() {
observer.observe(
value as u64,
&[KeyValue::new("table_name", table_name)],
@@ -62,7 +66,7 @@ impl TableMetrics {
.u64_value_observer(
"table.merkle_updater_todo_queue_length",
move |observer| {
if let Ok(v) = merkle_todo.approximate_len() {
if let Ok(v) = merkle_todo.len() {
observer.observe(
v as u64,
&[KeyValue::new("table_name", table_name)],
@@ -72,11 +76,25 @@ impl TableMetrics {
)
.with_description("Merkle tree updater TODO queue length")
.init(),
_merkle_todo_bounded_queue_free: meter
.u64_value_observer(
"table.merkle_todo_bounded_queue_free",
move |observer| {
let maybe_bounded = merkle_todo_bounded_queue.clone();
let free: u64 = match &maybe_bounded {
Some(v) => v.available_permits().try_into().unwrap(),
None => 0,
};
observer.observe(free, &[KeyValue::new("table_name", table_name)])
}
)
.with_description("Merkle TODO queue free slots")
.init(),
_gc_todo_len: meter
.u64_value_observer(
"table.gc_todo_queue_length",
move |observer| {
if let Ok(value) = gc_todo.approximate_len() {
if let Ok(value) = gc_todo.len() {
observer.observe(
value as u64,
&[KeyValue::new("table_name", table_name)],
+1 -1
View File
@@ -27,7 +27,7 @@ impl<F: TableSchema, R: TableReplication> Worker for InsertQueueWorker<F, R> {
fn status(&self) -> WorkerStatus {
WorkerStatus {
queue_length: Some(self.0.data.insert_queue.approximate_len().unwrap_or(0) as u64),
queue_length: Some(self.0.data.insert_queue.len().unwrap_or(0) as u64),
..Default::default()
}
}
+7 -1
View File
@@ -244,8 +244,14 @@ impl<F: TableSchema, R: TableReplication> TableSyncer<F, R> {
// All remote nodes have written those items, now we can delete them locally
let mut not_removed = 0;
let maybe_bounded = self.data.merkle_todo_bounded_queue.clone();
if let Some(b) = maybe_bounded {
b.acquire_many(items.len() as u32).await.unwrap().forget();
}
for (k, v) in items.iter() {
if !self.data.delete_if_equal(&k[..], &v[..])? {
let removed = self.data.delete_if_equal(&k[..], &v[..])?;
if !removed {
not_removed += 1;
}
}
+15 -3
View File
@@ -14,6 +14,7 @@ use opentelemetry::{
use garage_db as db;
use garage_util::background::BackgroundRunner;
use garage_util::config::MerkleBackpressureEnum;
use garage_util::data::*;
use garage_util::error::Error;
use garage_util::metrics::RecordDuration;
@@ -68,12 +69,18 @@ impl<F: TableSchema> Rpc for TableRpc<F> {
impl<F: TableSchema, R: TableReplication> Table<F, R> {
// =============== PUBLIC INTERFACE FUNCTIONS (new, insert, get, etc) ===============
pub fn new(instance: F, replication: R, system: Arc<System>, db: &db::Db) -> Arc<Self> {
pub fn new(
instance: F,
replication: R,
system: Arc<System>,
db: &db::Db,
config: &MerkleBackpressureEnum,
) -> Arc<Self> {
let endpoint = system
.netapp
.endpoint(format!("garage_table/table.rs/Rpc:{}", F::TABLE_NAME));
let data = TableData::new(system.clone(), instance, replication, db);
let data = TableData::new(system.clone(), instance, replication, db, config);
let merkle_updater = MerkleUpdater::new(data.clone());
@@ -131,7 +138,8 @@ impl<F: TableSchema, R: TableReplication> Table<F, R> {
who.as_ref(),
rpc,
RequestStrategy::with_priority(PRIO_NORMAL)
.with_quorum(self.data.replication.write_quorum()),
.with_quorum(self.data.replication.write_quorum())
.with_write_limiter(),
)
.await?;
@@ -527,6 +535,10 @@ impl<F: TableSchema, R: TableReplication> EndpointHandler<TableRpc<F>> for Table
Ok(TableRpc::Update(values))
}
TableRpc::Update(pairs) => {
let maybe_bounded = self.data.merkle_todo_bounded_queue.clone();
if let Some(b) = maybe_bounded {
b.acquire_many(pairs.len() as u32).await.unwrap().forget();
}
self.data.update_many(pairs)?;
Ok(TableRpc::Ok)
}
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_util"
version = "1.3.0"
version = "1.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+45 -14
View File
@@ -75,10 +75,6 @@ pub struct Config {
)]
pub block_ram_buffer_max: usize,
/// Maximum number of concurrent reads of block files on disk
#[serde(default = "default_block_max_concurrent_reads")]
pub block_max_concurrent_reads: usize,
/// Skip the permission check of secret files. Useful when
/// POSIX ACLs (or more complex chmods) are used.
#[serde(default)]
@@ -126,10 +122,6 @@ pub struct Config {
#[serde(deserialize_with = "deserialize_capacity", default)]
pub lmdb_map_size: usize,
/// Fjall block cache size
#[serde(deserialize_with = "deserialize_capacity", default)]
pub fjall_block_cache_size: usize,
// -- APIs
/// Configuration for S3 api
pub s3_api: S3ApiConfig,
@@ -144,9 +136,9 @@ pub struct Config {
#[serde(default = "Default::default")]
pub admin: AdminConfig,
/// Allow punycode in bucket names
#[serde(default)]
pub allow_punycode: bool,
/// --- Experimental
#[serde(default = "Default::default")]
pub experimental: ExperimentalConfig,
}
/// Value for data_dir: either a single directory or a list of dirs with attributes
@@ -267,6 +259,40 @@ pub struct KubernetesDiscoveryConfig {
pub skip_crd: bool,
}
#[derive(Deserialize, Debug, Clone, Default)]
pub struct ExperimentalConfig {
pub merkle_backpressure: MerkleBackpressureEnum,
pub rpc_in_flight_limiters: RpcInFlightLimiterEnum,
}
#[derive(Deserialize, Debug, Clone, Default)]
#[serde(rename_all = "lowercase", tag = "kind")]
pub enum MerkleBackpressureEnum {
#[default]
None,
FixedQueue(MerkleFixedQueue),
}
#[derive(Deserialize, Debug, Clone, Default)]
#[serde(rename_all = "lowercase", tag = "kind")]
pub enum RpcInFlightLimiterEnum {
#[default]
None,
FixedSize(InFlightFixedSize),
}
#[derive(Deserialize, Debug, Clone, Default)]
pub struct InFlightFixedSize {
#[serde(default = "default_max_table_write")]
pub max_table_write: usize,
}
#[derive(Deserialize, Debug, Clone, Default)]
pub struct MerkleFixedQueue {
#[serde(default = "default_max_queue_size")]
pub max_queue_size: usize,
}
/// Read and parse configuration
pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
let config = std::fs::read_to_string(config_file)?;
@@ -284,9 +310,6 @@ fn default_block_size() -> usize {
fn default_block_ram_buffer_max() -> usize {
256 * 1024 * 1024
}
fn default_block_max_concurrent_reads() -> usize {
16
}
fn default_consistency_mode() -> String {
"consistent".into()
@@ -296,6 +319,14 @@ fn default_compression() -> Option<i32> {
Some(1)
}
fn default_max_table_write() -> usize {
64
}
fn default_max_queue_size() -> usize {
256
}
fn deserialize_compression<'de, D>(deserializer: D) -> Result<Option<i32>, D::Error>
where
D: de::Deserializer<'de>,
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_web"
version = "1.3.0"
version = "1.1.0"
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
edition = "2018"
license = "AGPL-3.0"
+2 -22
View File
@@ -397,30 +397,10 @@ fn error_to_res(e: Error) -> Response<BoxBody<Error>> {
// was a HEAD request or we couldn't get the error document)
// We do NOT enter this code path when returning the bucket's
// error document (this is handled in serve_file)
let mut body_str = format!(
r"<title>{http_code} {code_text}</title>
<h1>{http_code} {code_text}</h1>",
http_code = e.http_status_code().as_u16(),
code_text = e.http_status_code().canonical_reason().unwrap_or("Unknown"),
);
if let Error::ApiError(ref err) = e {
body_str.push_str(&format!(
r"
<ul>
<li>Code: {s3_code}</li>
<li>Message: {s3_message}.</li>
</ul>",
s3_code = err.aws_code(),
s3_message = err,
));
}
let mut http_error = Response::new(string_body(body_str));
let body = string_body(format!("{}\n", e));
let mut http_error = Response::new(body);
*http_error.status_mut() = e.http_status_code();
e.add_headers(http_error.headers_mut());
http_error.headers_mut().insert(
http::header::CONTENT_TYPE,
"text/html; charset=utf-8".parse().unwrap(),
);
http_error
}