Compare commits

...

43 Commits

Author SHA1 Message Date
Alex Auvolat 4b1fdbef55 bump version to v1.3.0 2025-09-14 21:36:33 +02:00
Alex Auvolat 0f1b488be0 fix rust warnings 2025-09-14 21:25:37 +02:00
Alex 0bbf63ee0e Merge pull request 'update rusqlite and snapshot using VACUUM INTO' (#1164) from update-rusqlite into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1164
2025-09-14 18:28:01 +00:00
Alex 879d941d7b Merge pull request 'add garage repair clear-resync-queue (fix #1151)' (#1165) from clear-resync-queue into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1165
2025-09-14 17:50:41 +00:00
Alex Auvolat d726cf0299 add garage repair clear-resync-queue (fix #1151) 2025-09-14 19:34:44 +02:00
Alex 0c7aeab6f8 Merge pull request 'garage_db: fix error handling logic (fix #1138)' (#1163) from fix-1138 into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1163
2025-09-14 17:26:08 +00:00
Alex Auvolat 5687fc0375 update rusqlite and snapshot using VACUUM INTO 2025-09-14 19:22:36 +02:00
Alex 97f1e9ab52 Merge pull request 'Add Plakar documentation (backup tools)' (#1119) from Lapineige/garage:Plakar_support into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1119
2025-09-14 16:08:36 +00:00
Lapineige 60b1d78b56 Add Plakar documentation 2025-09-14 18:07:49 +02:00
Alex Auvolat 4c895a7186 garage_db: fix error handling logic (fix #1138) 2025-09-14 18:03:31 +02:00
Alex c3b5cbf212 Merge pull request 'fix panic when cluster_layout cannot be saved (fix #1150)' (#1158) from fix-1150 into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1158
2025-09-13 15:58:52 +00:00
Alex 57a467b5c0 Merge pull request 'Block manager: limit simultaneous block reads from disk' (#1157) from block-max-simultaneous-reads into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1157
2025-09-13 15:53:24 +00:00
Alex Auvolat 6cf6db5c61 fix panic when cluster_layout cannot be saved (fix #1150) 2025-09-13 17:49:25 +02:00
Alex Auvolat d5a57e3e13 block: read_block: don't add not found blocks to resync queue 2025-09-13 17:38:23 +02:00
Alex Auvolat 5cf354acb4 block: maximum number of simultaneous reads 2025-09-13 17:38:06 +02:00
Alex 2b007ddea3 Merge pull request 'woodpecker: require the nix=enabled label' (#1152) from woodpecker-nix-flag into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1152
2025-09-04 09:10:10 +00:00
Alex Auvolat c8599a8636 woodpecker: require the nix=enabled label 2025-09-04 11:06:46 +02:00
Alex 0b901bf291 Merge pull request 'garage_db: reduce frequency of sqlite snapshot progress log (fix #1129)' (#1146) from fix-1129 into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1146
2025-08-27 22:26:32 +00:00
Alex Auvolat c8c20d6f47 garage_db: reduce frequency of sqlite snapshot progress log (fix #1129) 2025-08-28 00:07:35 +02:00
Alex e5db610e4c Merge pull request 'K2V client: allow custom HTTP client' (#731) from k2v/shared_http_client into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/731
Reviewed-by: maximilien <me@mricher.fr>
2025-08-27 21:21:09 +00:00
Alex 65c6f8adea Merge pull request 'garage_db: refactor open function' (#1142) from factor-db-open into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1142
2025-08-27 21:10:59 +00:00
Alex Auvolat 54b9bf02a3 garage_db: refactor open function 2025-08-27 23:03:09 +02:00
Alex 469153233f Merge pull request 'garage_db: rename len to approximate_len as it is used for stats only' (#1141) from db-approximate-len into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1141
2025-08-27 20:44:50 +00:00
Alex Auvolat 90bba5889a garage_db: rename len to approximate_len as it is used for stats only 2025-08-27 21:23:45 +02:00
Alex a64b567d43 Merge pull request 'Add experimental support for Fjall DB engine' (#906) from withings/garage:feat/fjall-db-engine into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/906
2025-08-27 19:09:40 +00:00
Alex Auvolat 6ea86db8cd document fjall db engine, remove flakey metadata_fsync implementation 2025-08-27 20:22:41 +02:00
Alex Auvolat aa69c06f2b fix potential race condition and naming bug in fjall adapter 2025-08-27 20:22:38 +02:00
Alex Auvolat a6c6c44310 nix: build and test fjall feature 2025-08-27 18:54:42 +02:00
Julien Kritter 96d7713915 Add support for an LSM-tree-based backend with Fjall 2025-08-27 18:54:34 +02:00
Alex d64498c3d3 Merge pull request 'log access keys' (#1122) from 1686a/log-access-key into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1122
2025-08-27 16:18:16 +00:00
trinity-1686a b340599e68 log access keys 2025-08-03 15:30:56 +02:00
Alex 5448012b27 Merge pull request 'Pixelfed_support' (#1118) from Lapineige/garage:Pixelfed_support into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1118
2025-08-02 15:03:57 +00:00
Alex ce34d11a65 Merge pull request 'don't die on SIGHUP' (#1121) from 1686a/handle-sighup into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1121
2025-08-02 14:53:58 +00:00
Alex 8cb7623ebd Merge pull request 'handle ECONNABORTED' (#1120) from 1686a/handle-econnaborted into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1120
2025-08-02 14:53:45 +00:00
trinity-1686a 5469c95877 handle ECONNABORTED 2025-08-02 13:14:01 +02:00
trinity-1686a f930c6f643 don't die on SIGHUP 2025-08-02 13:09:33 +02:00
Alex afcb22bf16 Merge pull request 'Fix typo in peertube buckets names' (#1117) from Lapineige/garage:main into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1117
2025-08-02 08:27:01 +00:00
Lapineige cc29a40d51 Actualiser doc/book/connect/apps/index.md 2025-08-01 21:35:15 +00:00
Lapineige 0f3f180c3e Merge branch 'main-v1' into main 2025-08-01 21:33:58 +00:00
Lapineige 70cf6004ae Fix typo in peertube buckets names 2025-08-01 21:32:59 +00:00
Alex c7571ff89b Merge pull request 'Fix some unsoundness in lmdb adapter unsafe' (#1099) from krtab/garage:fix_some_ub into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1099
2025-07-31 19:38:23 +00:00
Arthur Carcano 1b42919bf7 Fix some unsoundness in lmdb adapter unsafe 2025-07-25 23:33:51 +02:00
Quentin Dufour 8b35a946d9 Allow external HTTP client 2024-02-23 17:09:47 +01:00
55 changed files with 1197 additions and 253 deletions
+8
View File
@@ -1,3 +1,6 @@
labels:
nix: "enabled"
when: when:
event: event:
- push - push
@@ -28,6 +31,11 @@ steps:
commands: commands:
- nix-build -j4 --attr flakePackages.tests-sqlite - nix-build -j4 --attr flakePackages.tests-sqlite
- name: unit + func tests (fjall)
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --attr flakePackages.tests-fjall
- name: integration tests - name: integration tests
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
commands: commands:
+3
View File
@@ -1,3 +1,6 @@
labels:
nix: "enabled"
when: when:
event: event:
- deployment - deployment
+3
View File
@@ -1,3 +1,6 @@
labels:
nix: "enabled"
when: when:
event: event:
- deployment - deployment
Generated
+219 -23
View File
@@ -687,6 +687,12 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d2c12f985c78475a6b8d629afd0c360260ef34cfef52efccdcfd31972f81c2e" checksum = "2d2c12f985c78475a6b8d629afd0c360260ef34cfef52efccdcfd31972f81c2e"
[[package]]
name = "byteview"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.2.16" version = "1.2.16"
@@ -798,6 +804,12 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
[[package]]
name = "compare"
version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7"
[[package]] [[package]]
name = "core-foundation" name = "core-foundation"
version = "0.9.4" version = "0.9.4"
@@ -874,6 +886,15 @@ dependencies = [
"crossbeam-utils", "crossbeam-utils",
] ]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]] [[package]]
name = "crossbeam-queue" name = "crossbeam-queue"
version = "0.3.12" version = "0.3.12"
@@ -883,6 +904,16 @@ dependencies = [
"crossbeam-utils", "crossbeam-utils",
] ]
[[package]]
name = "crossbeam-skiplist"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]] [[package]]
name = "crossbeam-utils" name = "crossbeam-utils"
version = "0.8.21" version = "0.8.21"
@@ -954,6 +985,20 @@ dependencies = [
"num_cpus", "num_cpus",
] ]
[[package]]
name = "dashmap"
version = "6.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
dependencies = [
"cfg-if",
"crossbeam-utils",
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core 0.9.10",
]
[[package]] [[package]]
name = "deranged" name = "deranged"
version = "0.4.0" version = "0.4.0"
@@ -996,6 +1041,12 @@ dependencies = [
"syn 2.0.100", "syn 2.0.100",
] ]
[[package]]
name = "double-ended-peekable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
[[package]] [[package]]
name = "dyn-clone" name = "dyn-clone"
version = "1.0.19" version = "1.0.19"
@@ -1017,6 +1068,18 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "enum_dispatch"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd"
dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.100",
]
[[package]] [[package]]
name = "env_logger" name = "env_logger"
version = "0.10.2" version = "0.10.2"
@@ -1084,6 +1147,23 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "fjall"
version = "2.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc"
dependencies = [
"byteorder",
"byteview",
"dashmap 6.1.0",
"log",
"lsm-tree",
"path-absolutize",
"std-semaphore",
"tempfile",
"xxhash-rust",
]
[[package]] [[package]]
name = "fnv" name = "fnv"
version = "1.0.7" version = "1.0.7"
@@ -1200,7 +1280,7 @@ dependencies = [
[[package]] [[package]]
name = "garage" name = "garage"
version = "1.2.0" version = "1.3.0"
dependencies = [ dependencies = [
"assert-json-diff", "assert-json-diff",
"async-trait", "async-trait",
@@ -1256,7 +1336,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_api_admin" name = "garage_api_admin"
version = "1.2.0" version = "1.3.0"
dependencies = [ dependencies = [
"argon2", "argon2",
"async-trait", "async-trait",
@@ -1282,7 +1362,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_api_common" name = "garage_api_common"
version = "1.2.0" version = "1.3.0"
dependencies = [ dependencies = [
"base64 0.21.7", "base64 0.21.7",
"bytes", "bytes",
@@ -1316,7 +1396,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_api_k2v" name = "garage_api_k2v"
version = "1.2.0" version = "1.3.0"
dependencies = [ dependencies = [
"base64 0.21.7", "base64 0.21.7",
"err-derive", "err-derive",
@@ -1339,7 +1419,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_api_s3" name = "garage_api_s3"
version = "1.2.0" version = "1.3.0"
dependencies = [ dependencies = [
"aes-gcm", "aes-gcm",
"async-compression", "async-compression",
@@ -1384,7 +1464,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_block" name = "garage_block"
version = "1.2.0" version = "1.3.0"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-compression", "async-compression",
@@ -1409,11 +1489,13 @@ dependencies = [
[[package]] [[package]]
name = "garage_db" name = "garage_db"
version = "1.2.0" version = "1.3.0"
dependencies = [ dependencies = [
"err-derive", "err-derive",
"fjall",
"heed", "heed",
"mktemp", "mktemp",
"parking_lot 0.12.3",
"r2d2", "r2d2",
"r2d2_sqlite", "r2d2_sqlite",
"rusqlite", "rusqlite",
@@ -1422,7 +1504,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_model" name = "garage_model"
version = "1.2.0" version = "1.3.0"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"base64 0.21.7", "base64 0.21.7",
@@ -1449,7 +1531,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_net" name = "garage_net"
version = "1.2.0" version = "1.3.0"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"bytes", "bytes",
@@ -1474,7 +1556,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_rpc" name = "garage_rpc"
version = "1.2.0" version = "1.3.0"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-trait", "async-trait",
@@ -1506,7 +1588,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_table" name = "garage_table"
version = "1.2.0" version = "1.3.0"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-trait", "async-trait",
@@ -1527,7 +1609,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_util" name = "garage_util"
version = "1.2.0" version = "1.3.0"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-trait", "async-trait",
@@ -1559,7 +1641,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_web" name = "garage_web"
version = "1.2.0" version = "1.3.0"
dependencies = [ dependencies = [
"err-derive", "err-derive",
"garage_api_common", "garage_api_common",
@@ -1655,6 +1737,12 @@ dependencies = [
"syn 2.0.100", "syn 2.0.100",
] ]
[[package]]
name = "guardian"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f"
[[package]] [[package]]
name = "h2" name = "h2"
version = "0.3.26" version = "0.3.26"
@@ -1722,11 +1810,11 @@ dependencies = [
[[package]] [[package]]
name = "hashlink" name = "hashlink"
version = "0.9.1" version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
dependencies = [ dependencies = [
"hashbrown 0.14.5", "hashbrown 0.15.2",
] ]
[[package]] [[package]]
@@ -2229,6 +2317,15 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "interval-heap"
version = "0.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6"
dependencies = [
"compare",
]
[[package]] [[package]]
name = "ipnet" name = "ipnet"
version = "2.11.0" version = "2.11.0"
@@ -2522,9 +2619,9 @@ dependencies = [
[[package]] [[package]]
name = "libsqlite3-sys" name = "libsqlite3-sys"
version = "0.28.0" version = "0.35.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f"
dependencies = [ dependencies = [
"cc", "cc",
"pkg-config", "pkg-config",
@@ -2585,6 +2682,36 @@ dependencies = [
"hashbrown 0.15.2", "hashbrown 0.15.2",
] ]
[[package]]
name = "lsm-tree"
version = "2.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab73c02eadb3dc12c0024e5b61d6284e6d59064e67e74fbad77856caa56f62c7"
dependencies = [
"byteorder",
"crossbeam-skiplist",
"double-ended-peekable",
"enum_dispatch",
"guardian",
"interval-heap",
"log",
"lz4_flex",
"path-absolutize",
"quick_cache",
"rustc-hash",
"self_cell",
"tempfile",
"value-log",
"varint-rs",
"xxhash-rust",
]
[[package]]
name = "lz4_flex"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
[[package]] [[package]]
name = "matchers" name = "matchers"
version = "0.1.0" version = "0.1.0"
@@ -2839,7 +2966,7 @@ checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"crossbeam-channel", "crossbeam-channel",
"dashmap", "dashmap 4.0.2",
"fnv", "fnv",
"futures-channel", "futures-channel",
"futures-executor", "futures-executor",
@@ -3000,6 +3127,24 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "path-absolutize"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5"
dependencies = [
"path-dedot",
]
[[package]]
name = "path-dedot"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397"
dependencies = [
"once_cell",
]
[[package]] [[package]]
name = "pem" name = "pem"
version = "3.0.5" version = "3.0.5"
@@ -3295,6 +3440,16 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "quick_cache"
version = "0.6.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786"
dependencies = [
"equivalent",
"hashbrown 0.15.2",
]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.40" version = "1.0.40"
@@ -3323,9 +3478,9 @@ dependencies = [
[[package]] [[package]]
name = "r2d2_sqlite" name = "r2d2_sqlite"
version = "0.24.0" version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a982edf65c129796dba72f8775b292ef482b40d035e827a9825b3bc07ccc5f2" checksum = "63417e83dc891797eea3ad379f52a5986da4bca0d6ef28baf4d14034dd111b0c"
dependencies = [ dependencies = [
"r2d2", "r2d2",
"rusqlite", "rusqlite",
@@ -3514,9 +3669,9 @@ checksum = "3cd14fd5e3b777a7422cca79358c57a8f6e3a703d9ac187448d0daf220c2407f"
[[package]] [[package]]
name = "rusqlite" name = "rusqlite"
version = "0.31.0" version = "0.37.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f"
dependencies = [ dependencies = [
"bitflags 2.9.0", "bitflags 2.9.0",
"fallible-iterator", "fallible-iterator",
@@ -3532,6 +3687,12 @@ version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
[[package]]
name = "rustc-hash"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
[[package]] [[package]]
name = "rustc_version" name = "rustc_version"
version = "0.4.1" version = "0.4.1"
@@ -3774,6 +3935,12 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "self_cell"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749"
[[package]] [[package]]
name = "semver" name = "semver"
version = "1.0.26" version = "1.0.26"
@@ -3987,6 +4154,12 @@ dependencies = [
"syn 1.0.109", "syn 1.0.109",
] ]
[[package]]
name = "std-semaphore"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e"
[[package]] [[package]]
name = "strsim" name = "strsim"
version = "0.11.1" version = "0.11.1"
@@ -4664,6 +4837,29 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
[[package]]
name = "value-log"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c"
dependencies = [
"byteorder",
"byteview",
"interval-heap",
"log",
"path-absolutize",
"rustc-hash",
"tempfile",
"varint-rs",
"xxhash-rust",
]
[[package]]
name = "varint-rs"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]] [[package]]
name = "vcpkg" name = "vcpkg"
version = "0.2.15" version = "0.2.15"
+16 -14
View File
@@ -24,18 +24,18 @@ default-members = ["src/garage"]
# Internal Garage crates # Internal Garage crates
format_table = { version = "0.1.1", path = "src/format-table" } format_table = { version = "0.1.1", path = "src/format-table" }
garage_api_common = { version = "1.2.0", path = "src/api/common" } garage_api_common = { version = "1.3.0", path = "src/api/common" }
garage_api_admin = { version = "1.2.0", path = "src/api/admin" } garage_api_admin = { version = "1.3.0", path = "src/api/admin" }
garage_api_s3 = { version = "1.2.0", path = "src/api/s3" } garage_api_s3 = { version = "1.3.0", path = "src/api/s3" }
garage_api_k2v = { version = "1.2.0", path = "src/api/k2v" } garage_api_k2v = { version = "1.3.0", path = "src/api/k2v" }
garage_block = { version = "1.2.0", path = "src/block" } garage_block = { version = "1.3.0", path = "src/block" }
garage_db = { version = "1.2.0", path = "src/db", default-features = false } garage_db = { version = "1.3.0", path = "src/db", default-features = false }
garage_model = { version = "1.2.0", path = "src/model", default-features = false } garage_model = { version = "1.3.0", path = "src/model", default-features = false }
garage_net = { version = "1.2.0", path = "src/net" } garage_net = { version = "1.3.0", path = "src/net" }
garage_rpc = { version = "1.2.0", path = "src/rpc" } garage_rpc = { version = "1.3.0", path = "src/rpc" }
garage_table = { version = "1.2.0", path = "src/table" } garage_table = { version = "1.3.0", path = "src/table" }
garage_util = { version = "1.2.0", path = "src/util" } garage_util = { version = "1.3.0", path = "src/util" }
garage_web = { version = "1.2.0", path = "src/web" } garage_web = { version = "1.3.0", path = "src/web" }
k2v-client = { version = "0.0.4", path = "src/k2v-client" } k2v-client = { version = "0.0.4", path = "src/k2v-client" }
# External crates from crates.io # External crates from crates.io
@@ -65,6 +65,7 @@ md-5 = "0.10"
mktemp = "0.5" mktemp = "0.5"
nix = { version = "0.29", default-features = false, features = ["fs"] } nix = { version = "0.29", default-features = false, features = ["fs"] }
nom = "7.1" nom = "7.1"
parking_lot = "0.12"
parse_duration = "2.1" parse_duration = "2.1"
pin-project = "1.0.12" pin-project = "1.0.12"
pnet_datalink = "0.34" pnet_datalink = "0.34"
@@ -87,9 +88,10 @@ tracing-journald = "0.3.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] }
heed = { version = "0.11", default-features = false, features = ["lmdb"] } heed = { version = "0.11", default-features = false, features = ["lmdb"] }
rusqlite = "0.31.0" rusqlite = "0.37"
r2d2 = "0.8" r2d2 = "0.8"
r2d2_sqlite = "0.24" r2d2_sqlite = "0.31"
fjall = "2.4"
async-compression = { version = "0.4", features = ["tokio", "zstd"] } async-compression = { version = "0.4", features = ["tokio", "zstd"] }
zstd = { version = "0.13", default-features = false } zstd = { version = "0.13", default-features = false }
+4 -4
View File
@@ -12,7 +12,7 @@ In this section, we cover the following web applications:
| [Mastodon](#mastodon) | ✅ | Natively supported | | [Mastodon](#mastodon) | ✅ | Natively supported |
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` | | [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
| [ejabberd](#ejabberd) | ✅ | `mod_s3_upload` | | [ejabberd](#ejabberd) | ✅ | `mod_s3_upload` |
| [Pixelfed](#pixelfed) | | Not yet tested | | [Pixelfed](#pixelfed) | | Natively supported |
| [Pleroma](#pleroma) | ❓ | Not yet tested | | [Pleroma](#pleroma) | ❓ | Not yet tested |
| [Lemmy](#lemmy) | ✅ | Supported with pict-rs | | [Lemmy](#lemmy) | ✅ | Supported with pict-rs |
| [Funkwhale](#funkwhale) | ❓ | Not yet tested | | [Funkwhale](#funkwhale) | ❓ | Not yet tested |
@@ -191,10 +191,10 @@ garage key create peertube-key
Keep the Key ID and the Secret key in a pad, they will be needed later. Keep the Key ID and the Secret key in a pad, they will be needed later.
We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist). We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists).
```bash ```bash
garage bucket create peertube-videos garage bucket create peertube-videos
garage bucket create peertube-playlist garage bucket create peertube-playlists
``` ```
Now we allow our key to read and write on these buckets: Now we allow our key to read and write on these buckets:
@@ -253,7 +253,7 @@ object_storage:
proxify_private_files: false proxify_private_files: false
streaming_playlists: streaming_playlists:
bucket_name: 'peertube-playlist' bucket_name: 'peertube-playlists'
# Keep it empty for our example # Keep it empty for our example
prefix: '' prefix: ''
+46
View File
@@ -161,3 +161,49 @@ kopia repository validate-provider
You can then run all the standard kopia commands: `kopia snapshot create`, `kopia mount`... You can then run all the standard kopia commands: `kopia snapshot create`, `kopia mount`...
Everything should work out-of-the-box. Everything should work out-of-the-box.
## Plakar
Create your key and bucket on Garage server:
```bash
garage key create my-plakar-key
garage bucket create plakar-backups
garage bucket allow plakar-backups --read --write --key my-plakar-key
```
On Plakar server, add your Garage as a storage location:
```bash
plakar store add garageS3 s3://my-garage.tld/plakar-backups \
region=garage # Or as you've specified in garage.toml \
access_key=<Key ID from "garage key info my-plakar-key"> \
secret_access_key=<Secret key from "garage key info my-plakar-key">
```
Then create the repository.
```bash
plakar at @garageS3 create -plaintext # Unencrypted
# or
plakar at @garageS3 create #encrypted
```
If you encrypt your backups (Plakar default), you will need to define a strong passphrase. Do not forget to save your password safely. It will be needed to decrypt your backups.
After the repository has been created, check that everything works as expected (that might give an empty result as no file has been added yet, but no error message):
```bash
plakar at @garageS3 check
```
Now that everything is configure, you can use Garage as your backups storage. For instance sync it with a local backup storage:
```bash
$ plakar at ~/backups sync to @garageS3
```
Or list the S3 storage content:
```bash
$ plakar at @garageS3 ls
```
More information in Plakar documentation: https://www.plakar.io/docs/main/quickstart/
+5 -5
View File
@@ -96,14 +96,14 @@ to store 2 TB of data in total.
## Get a Docker image ## Get a Docker image
Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
We encourage you to use a fixed tag (eg. `v1.2.0`) and not the `latest` tag. We encourage you to use a fixed tag (eg. `v1.3.0`) and not the `latest` tag.
For this example, we will use the latest published version at the time of the writing which is `v1.2.0` but it's up to you For this example, we will use the latest published version at the time of the writing which is `v1.3.0` but it's up to you
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
For example: For example:
``` ```
sudo docker pull dxflrs/garage:v1.2.0 sudo docker pull dxflrs/garage:v1.3.0
``` ```
## Deploying and configuring Garage ## Deploying and configuring Garage
@@ -171,7 +171,7 @@ docker run \
-v /etc/garage.toml:/etc/garage.toml \ -v /etc/garage.toml:/etc/garage.toml \
-v /var/lib/garage/meta:/var/lib/garage/meta \ -v /var/lib/garage/meta:/var/lib/garage/meta \
-v /var/lib/garage/data:/var/lib/garage/data \ -v /var/lib/garage/data:/var/lib/garage/data \
dxflrs/garage:v1.2.0 dxflrs/garage:v1.3.0
``` ```
With this command line, Garage should be started automatically at each boot. With this command line, Garage should be started automatically at each boot.
@@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
version: "3" version: "3"
services: services:
garage: garage:
image: dxflrs/garage:v1.2.0 image: dxflrs/garage:v1.3.0
network_mode: "host" network_mode: "host"
restart: unless-stopped restart: unless-stopped
volumes: volumes:
+1 -1
View File
@@ -132,7 +132,7 @@ docker run \
-v /path/to/garage.toml:/etc/garage.toml \ -v /path/to/garage.toml:/etc/garage.toml \
-v /path/to/garage/meta:/var/lib/garage/meta \ -v /path/to/garage/meta:/var/lib/garage/meta \
-v /path/to/garage/data:/var/lib/garage/data \ -v /path/to/garage/data:/var/lib/garage/data \
dxflrs/garage:v1.2.0 dxflrs/garage:v1.3.0
``` ```
Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903` Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903`
@@ -24,6 +24,7 @@ db_engine = "lmdb"
block_size = "1M" block_size = "1M"
block_ram_buffer_max = "256MiB" block_ram_buffer_max = "256MiB"
block_max_concurrent_reads = 16
lmdb_map_size = "1T" lmdb_map_size = "1T"
@@ -96,6 +97,7 @@ The following gives details about each available configuration option.
Top-level configuration options, in alphabetical order: Top-level configuration options, in alphabetical order:
[`allow_punycode`](#allow_punycode), [`allow_punycode`](#allow_punycode),
[`allow_world_readable_secrets`](#allow_world_readable_secrets), [`allow_world_readable_secrets`](#allow_world_readable_secrets),
[`block_max_concurrent_reads`](`block_max_concurrent_reads),
[`block_ram_buffer_max`](#block_ram_buffer_max), [`block_ram_buffer_max`](#block_ram_buffer_max),
[`block_size`](#block_size), [`block_size`](#block_size),
[`bootstrap_peers`](#bootstrap_peers), [`bootstrap_peers`](#bootstrap_peers),
@@ -333,6 +335,7 @@ Since `v0.8.0`, Garage can use alternative storage backends as follows:
| --------- | ----------------- | ------------- | | --------- | ----------------- | ------------- |
| [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` | | [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
| [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` | | [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
| [Fjall](https://github.com/fjall-rs/fjall) (**experimental support** since `v1.3.0`) | `"fjall"` | `<metadata_dir>/db.fjall/` |
| [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` | | [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` |
Sled was supported until Garage v0.9.x, and was removed in Garage v1.0. Sled was supported until Garage v0.9.x, and was removed in Garage v1.0.
@@ -369,6 +372,14 @@ LMDB works very well, but is known to have the following limitations:
so it is not the best choice for high-performance storage clusters, so it is not the best choice for high-performance storage clusters,
but it should work fine in many cases. but it should work fine in many cases.
- Fjall: a storage engine based on LSM trees, which theoretically allow for
higher write throughput than other storage engines that are based on B-trees.
Using Fjall could potentially improve Garage's performance significantly in
write-heavy workloads. **Support for Fjall is experimental at this point**,
we have added it to Garage for evaluation purposes only. **Do not use it for
production-critical workloads.**
It is possible to convert Garage's metadata directory from one format to another It is possible to convert Garage's metadata directory from one format to another
using the `garage convert-db` command, which should be used as follows: using the `garage convert-db` command, which should be used as follows:
@@ -406,6 +417,7 @@ Here is how this option impacts the different database engines:
|----------|------------------------------------|-------------------------------| |----------|------------------------------------|-------------------------------|
| Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` | | Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` |
| LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` | | LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` |
| Fjall | default options | not supported |
Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`). Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).
@@ -512,6 +524,29 @@ node.
The default value is 256MiB. The default value is 256MiB.
#### `block_max_concurrent_reads` (since `v1.3.0` / `v2.1.0`) {#block_max_concurrent_reads}
The maximum number of blocks (individual files in the data directory) open
simultaneously for reading.
Reducing this number does not limit the number of data blocks that can be
transferred through the network simultaneously. This mechanism was just added
as a backpressure mechanism for HDD read speed: it helps avoid a situation
where too many requests are coming in and Garage is reading too many block
files simultaneously, thus not making timely progress on any of the reads.
When a request to read a data block comes in through the network, the requests
awaits for one of the `block_max_concurrent_reads` slots to be available
(internally implemented using a Semaphore object). Once it acquired a read
slot, it reads the entire block file to RAM and frees the slot as soon as the
block file is finished reading. Only after the slot is released will the
block's data start being transferred over the network. If the request fails to
acquire a reading slot wihtin 15 seconds, it fails with a timeout error.
Timeout events can be monitored through the `block_read_semaphore_timeouts`
metric in Prometheus: a non-zero number of such events indicates an I/O
bottleneck on HDD read speed.
#### `lmdb_map_size` {#lmdb_map_size} #### `lmdb_map_size` {#lmdb_map_size}
This parameters can be used to set the map size used by LMDB, This parameters can be used to set the map size used by LMDB,
+1 -1
View File
@@ -70,7 +70,7 @@ Example response body:
```json ```json
{ {
"node": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df", "node": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df",
"garageVersion": "v1.2.0", "garageVersion": "v1.3.0",
"garageFeatures": [ "garageFeatures": [
"k2v", "k2v",
"lmdb", "lmdb",
+3
View File
@@ -53,6 +53,9 @@
tests-sqlite = testWith { tests-sqlite = testWith {
GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite"; GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite";
}; };
tests-fjall = testWith {
GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall";
};
}; };
# ---- developpment shell, for making native builds only ---- # ---- developpment shell, for making native builds only ----
+1 -1
View File
@@ -68,7 +68,7 @@ let
rootFeatures = if features != null then rootFeatures = if features != null then
features features
else else
([ "bundled-libs" "lmdb" "sqlite" "k2v" ] ++ (lib.optionals release [ ([ "bundled-libs" "lmdb" "sqlite" "fjall" "k2v" ] ++ (lib.optionals release [
"consul-discovery" "consul-discovery"
"kubernetes-discovery" "kubernetes-discovery"
"metrics" "metrics"
+2 -2
View File
@@ -2,8 +2,8 @@ apiVersion: v2
name: garage name: garage
description: S3-compatible object store for small self-hosted geo-distributed deployments description: S3-compatible object store for small self-hosted geo-distributed deployments
type: application type: application
version: 0.7.1 version: 0.7.2
appVersion: "v1.2.0" appVersion: "v1.3.0"
home: https://garagehq.deuxfleurs.fr/ home: https://garagehq.deuxfleurs.fr/
icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg
+1 -1
View File
@@ -1,6 +1,6 @@
# garage # garage
![Version: 0.7.1](https://img.shields.io/badge/Version-0.7.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.2.0](https://img.shields.io/badge/AppVersion-v1.2.0-informational?style=flat-square) ![Version: 0.7.2](https://img.shields.io/badge/Version-0.7.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.3.0](https://img.shields.io/badge/AppVersion-v1.3.0-informational?style=flat-square)
S3-compatible object store for small self-hosted geo-distributed deployments S3-compatible object store for small self-hosted geo-distributed deployments
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_api_admin" name = "garage_api_admin"
version = "1.2.0" version = "1.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_api_common" name = "garage_api_common"
version = "1.2.0" version = "1.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+22 -11
View File
@@ -58,6 +58,12 @@ pub trait ApiHandler: Send + Sync + 'static {
req: Request<IncomingBody>, req: Request<IncomingBody>,
endpoint: Self::Endpoint, endpoint: Self::Endpoint,
) -> impl Future<Output = Result<Response<BoxBody<Self::Error>>, Self::Error>> + Send; ) -> impl Future<Output = Result<Response<BoxBody<Self::Error>>, Self::Error>> + Send;
/// Returns the key id used to authenticate this request. The ID returned must be safe to
/// log.
fn key_id_from_request(&self, _req: &Request<IncomingBody>) -> Option<String> {
None
}
} }
pub struct ApiServer<A: ApiHandler> { pub struct ApiServer<A: ApiHandler> {
@@ -142,19 +148,20 @@ impl<A: ApiHandler> ApiServer<A> {
) -> Result<Response<BoxBody<A::Error>>, http::Error> { ) -> Result<Response<BoxBody<A::Error>>, http::Error> {
let uri = req.uri().clone(); let uri = req.uri().clone();
if let Ok(forwarded_for_ip_addr) = let source = if let Ok(forwarded_for_ip_addr) =
forwarded_headers::handle_forwarded_for_headers(req.headers()) forwarded_headers::handle_forwarded_for_headers(req.headers())
{ {
info!( format!("{forwarded_for_ip_addr} (via {addr})")
"{} (via {}) {} {}",
forwarded_for_ip_addr,
addr,
req.method(),
uri
);
} else { } else {
info!("{} {} {}", addr, req.method(), uri); format!("{addr}")
} };
// we only do this to log the access key, so we can discard any error
let key = self
.api_handler
.key_id_from_request(&req)
.map(|k| format!("(key {k}) "))
.unwrap_or_default();
info!("{source} {key}{} {uri}", req.method());
debug!("{:?}", req); debug!("{:?}", req);
let tracer = opentelemetry::global::tracer("garage"); let tracer = opentelemetry::global::tracer("garage");
@@ -343,7 +350,11 @@ where
while !*must_exit.borrow() { while !*must_exit.borrow() {
let (stream, client_addr) = tokio::select! { let (stream, client_addr) = tokio::select! {
acc = listener.accept() => acc?, acc = listener.accept() => match acc {
Ok(r) => r,
Err(e) if e.kind() == std::io::ErrorKind::ConnectionAborted => continue,
Err(e) => return Err(e.into()),
},
_ = must_exit.changed() => continue, _ = must_exit.changed() => continue,
}; };
+2 -2
View File
@@ -417,7 +417,7 @@ pub async fn verify_v4(
// ============ Authorization header, or X-Amz-* query params ========= // ============ Authorization header, or X-Amz-* query params =========
pub struct Authorization { pub struct Authorization {
key_id: String, pub key_id: String,
scope: String, scope: String,
signed_headers: String, signed_headers: String,
signature: String, signature: String,
@@ -426,7 +426,7 @@ pub struct Authorization {
} }
impl Authorization { impl Authorization {
fn parse_header(headers: &HeaderMap) -> Result<Self, Error> { pub fn parse_header(headers: &HeaderMap) -> Result<Self, Error> {
let authorization = headers let authorization = headers
.get(AUTHORIZATION) .get(AUTHORIZATION)
.ok_or_bad_request("Missing authorization header")? .ok_or_bad_request("Missing authorization header")?
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_api_k2v" name = "garage_api_k2v"
version = "1.2.0" version = "1.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+6
View File
@@ -176,6 +176,12 @@ impl ApiHandler for K2VApiServer {
Ok(resp_ok) Ok(resp_ok)
} }
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
garage_api_common::signature::payload::Authorization::parse_header(req.headers())
.map(|auth| auth.key_id)
.ok()
}
} }
impl ApiEndpoint for K2VApiEndpoint { impl ApiEndpoint for K2VApiEndpoint {
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_api_s3" name = "garage_api_s3"
version = "1.2.0" version = "1.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+6
View File
@@ -343,6 +343,12 @@ impl ApiHandler for S3ApiServer {
Ok(resp_ok) Ok(resp_ok)
} }
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
garage_api_common::signature::payload::Authorization::parse_header(req.headers())
.map(|auth| auth.key_id)
.ok()
}
} }
impl ApiEndpoint for S3ApiEndpoint { impl ApiEndpoint for S3ApiEndpoint {
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_block" name = "garage_block"
version = "1.2.0" version = "1.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+19 -6
View File
@@ -50,6 +50,8 @@ pub const INLINE_THRESHOLD: usize = 3072;
// to delete the block locally. // to delete the block locally.
pub(crate) const BLOCK_GC_DELAY: Duration = Duration::from_secs(600); pub(crate) const BLOCK_GC_DELAY: Duration = Duration::from_secs(600);
const BLOCK_READ_SEMAPHORE_TIMEOUT: Duration = Duration::from_secs(15);
/// RPC messages used to share blocks of data between nodes /// RPC messages used to share blocks of data between nodes
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]
pub enum BlockRpc { pub enum BlockRpc {
@@ -87,6 +89,7 @@ pub struct BlockManager {
disable_scrub: bool, disable_scrub: bool,
mutation_lock: Vec<Mutex<BlockManagerLocked>>, mutation_lock: Vec<Mutex<BlockManagerLocked>>,
read_semaphore: Semaphore,
pub rc: BlockRc, pub rc: BlockRc,
pub resync: BlockResyncManager, pub resync: BlockResyncManager,
@@ -176,6 +179,8 @@ impl BlockManager {
.iter() .iter()
.map(|_| Mutex::new(BlockManagerLocked())) .map(|_| Mutex::new(BlockManagerLocked()))
.collect::<Vec<_>>(), .collect::<Vec<_>>(),
read_semaphore: Semaphore::new(config.block_max_concurrent_reads),
rc, rc,
resync, resync,
system, system,
@@ -408,8 +413,8 @@ impl BlockManager {
} }
/// Get number of items in the refcount table /// Get number of items in the refcount table
pub fn rc_len(&self) -> Result<usize, Error> { pub fn rc_approximate_len(&self) -> Result<usize, Error> {
Ok(self.rc.rc_table.len()?) Ok(self.rc.rc_table.approximate_len()?)
} }
/// Send command to start/stop/manager scrub worker /// Send command to start/stop/manager scrub worker
@@ -427,7 +432,7 @@ impl BlockManager {
/// List all resync errors /// List all resync errors
pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> { pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> {
let mut blocks = Vec::with_capacity(self.resync.errors.len()?); let mut blocks = Vec::with_capacity(self.resync.errors.approximate_len()?);
for ent in self.resync.errors.iter()? { for ent in self.resync.errors.iter()? {
let (hash, cnt) = ent?; let (hash, cnt) = ent?;
let cnt = ErrorCounter::decode(&cnt); let cnt = ErrorCounter::decode(&cnt);
@@ -557,9 +562,6 @@ impl BlockManager {
match self.find_block(hash).await { match self.find_block(hash).await {
Some(p) => self.read_block_from(hash, &p).await, Some(p) => self.read_block_from(hash, &p).await,
None => { None => {
// Not found but maybe we should have had it ??
self.resync
.put_to_resync(hash, 2 * self.system.rpc_helper().rpc_timeout())?;
return Err(Error::Message(format!( return Err(Error::Message(format!(
"block {:?} not found on node", "block {:?} not found on node",
hash hash
@@ -581,6 +583,15 @@ impl BlockManager {
) -> Result<DataBlock, Error> { ) -> Result<DataBlock, Error> {
let (header, path) = block_path.as_parts_ref(); let (header, path) = block_path.as_parts_ref();
let permit = tokio::select! {
sem = self.read_semaphore.acquire() => sem.ok_or_message("acquire read semaphore")?,
_ = tokio::time::sleep(BLOCK_READ_SEMAPHORE_TIMEOUT) => {
self.metrics.block_read_semaphore_timeouts.add(1);
debug!("read block {:?}: read_semaphore acquire timeout", hash);
return Err(Error::Message("read block: read_semaphore acquire timeout".into()));
}
};
let mut f = fs::File::open(&path).await?; let mut f = fs::File::open(&path).await?;
let mut data = vec![]; let mut data = vec![];
f.read_to_end(&mut data).await?; f.read_to_end(&mut data).await?;
@@ -605,6 +616,8 @@ impl BlockManager {
return Err(Error::CorruptData(*hash)); return Err(Error::CorruptData(*hash));
} }
drop(permit);
Ok(data) Ok(data)
} }
+9 -3
View File
@@ -22,6 +22,7 @@ pub struct BlockManagerMetrics {
pub(crate) bytes_read: BoundCounter<u64>, pub(crate) bytes_read: BoundCounter<u64>,
pub(crate) block_read_duration: BoundValueRecorder<f64>, pub(crate) block_read_duration: BoundValueRecorder<f64>,
pub(crate) block_read_semaphore_timeouts: BoundCounter<u64>,
pub(crate) bytes_written: BoundCounter<u64>, pub(crate) bytes_written: BoundCounter<u64>,
pub(crate) block_write_duration: BoundValueRecorder<f64>, pub(crate) block_write_duration: BoundValueRecorder<f64>,
pub(crate) delete_counter: BoundCounter<u64>, pub(crate) delete_counter: BoundCounter<u64>,
@@ -50,7 +51,7 @@ impl BlockManagerMetrics {
.init(), .init(),
_rc_size: meter _rc_size: meter
.u64_value_observer("block.rc_size", move |observer| { .u64_value_observer("block.rc_size", move |observer| {
if let Ok(value) = rc_tree.len() { if let Ok(value) = rc_tree.approximate_len() {
observer.observe(value as u64, &[]) observer.observe(value as u64, &[])
} }
}) })
@@ -58,7 +59,7 @@ impl BlockManagerMetrics {
.init(), .init(),
_resync_queue_len: meter _resync_queue_len: meter
.u64_value_observer("block.resync_queue_length", move |observer| { .u64_value_observer("block.resync_queue_length", move |observer| {
if let Ok(value) = resync_queue.len() { if let Ok(value) = resync_queue.approximate_len() {
observer.observe(value as u64, &[]); observer.observe(value as u64, &[]);
} }
}) })
@@ -68,7 +69,7 @@ impl BlockManagerMetrics {
.init(), .init(),
_resync_errored_blocks: meter _resync_errored_blocks: meter
.u64_value_observer("block.resync_errored_blocks", move |observer| { .u64_value_observer("block.resync_errored_blocks", move |observer| {
if let Ok(value) = resync_errors.len() { if let Ok(value) = resync_errors.approximate_len() {
observer.observe(value as u64, &[]); observer.observe(value as u64, &[]);
} }
}) })
@@ -119,6 +120,11 @@ impl BlockManagerMetrics {
.with_description("Duration of block read operations") .with_description("Duration of block read operations")
.init() .init()
.bind(&[]), .bind(&[]),
block_read_semaphore_timeouts: meter
.u64_counter("block.read_semaphore_timeouts")
.with_description("Number of block reads that failed due to semaphore acquire timeout")
.init()
.bind(&[]),
bytes_written: meter bytes_written: meter
.u64_counter("block.bytes_written") .u64_counter("block.bytes_written")
.with_description("Number of bytes written to disk") .with_description("Number of bytes written to disk")
+16 -6
View File
@@ -106,13 +106,13 @@ impl BlockResyncManager {
} }
/// Get length of resync queue /// Get length of resync queue
pub fn queue_len(&self) -> Result<usize, Error> { pub fn queue_approximate_len(&self) -> Result<usize, Error> {
Ok(self.queue.len()?) Ok(self.queue.approximate_len()?)
} }
/// Get number of blocks that have an error /// Get number of blocks that have an error
pub fn errors_len(&self) -> Result<usize, Error> { pub fn errors_approximate_len(&self) -> Result<usize, Error> {
Ok(self.errors.len()?) Ok(self.errors.approximate_len()?)
} }
/// Clear the error counter for a block and put it in queue immediately /// Clear the error counter for a block and put it in queue immediately
@@ -133,6 +133,14 @@ impl BlockResyncManager {
))) )))
} }
/// Clear the entire resync queue and list of errored blocks
/// Corresponds to `garage repair clear-resync-queue`
pub fn clear_resync_queue(&self) -> Result<(), Error> {
self.queue.clear()?;
self.errors.clear()?;
Ok(())
}
pub fn register_bg_vars(&self, vars: &mut vars::BgVars) { pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
let notify = self.notify.clone(); let notify = self.notify.clone();
vars.register_rw( vars.register_rw(
@@ -548,9 +556,11 @@ impl Worker for ResyncWorker {
} }
WorkerStatus { WorkerStatus {
queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64), queue_length: Some(self.manager.resync.queue_approximate_len().unwrap_or(0) as u64),
tranquility: Some(tranquility), tranquility: Some(tranquility),
persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64), persistent_errors: Some(
self.manager.resync.errors_approximate_len().unwrap_or(0) as u64
),
..Default::default() ..Default::default()
} }
} }
+6 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_db" name = "garage_db"
version = "1.2.0" version = "1.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@@ -16,10 +16,14 @@ err-derive.workspace = true
tracing.workspace = true tracing.workspace = true
heed = { workspace = true, optional = true } heed = { workspace = true, optional = true }
rusqlite = { workspace = true, optional = true, features = ["backup"] } rusqlite = { workspace = true, optional = true, features = ["backup"] }
r2d2 = { workspace = true, optional = true } r2d2 = { workspace = true, optional = true }
r2d2_sqlite = { workspace = true, optional = true } r2d2_sqlite = { workspace = true, optional = true }
fjall = { workspace = true, optional = true }
parking_lot = { workspace = true, optional = true }
[dev-dependencies] [dev-dependencies]
mktemp.workspace = true mktemp.workspace = true
@@ -27,4 +31,5 @@ mktemp.workspace = true
default = [ "lmdb", "sqlite" ] default = [ "lmdb", "sqlite" ]
bundled-libs = [ "rusqlite?/bundled" ] bundled-libs = [ "rusqlite?/bundled" ]
lmdb = [ "heed" ] lmdb = [ "heed" ]
fjall = [ "dep:fjall", "dep:parking_lot" ]
sqlite = [ "rusqlite", "r2d2", "r2d2_sqlite" ] sqlite = [ "rusqlite", "r2d2", "r2d2_sqlite" ]
+453
View File
@@ -0,0 +1,453 @@
use core::ops::Bound;
use std::path::PathBuf;
use std::sync::Arc;
use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard};
use fjall::{
PartitionCreateOptions, PersistMode, TransactionalKeyspace, TransactionalPartitionHandle,
WriteTransaction,
};
use crate::{
open::{Engine, OpenOpt},
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
TxResult, TxValueIter, Value, ValueIter,
};
pub use fjall;
// --
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
info!("Opening Fjall database at: {}", path.display());
if opt.fsync {
return Err(Error(
"metadata_fsync is not supported with the Fjall database engine".into(),
));
}
let mut config = fjall::Config::new(path);
if let Some(block_cache_size) = opt.fjall_block_cache_size {
config = config.cache_size(block_cache_size as u64);
}
let keyspace = config.open_transactional()?;
Ok(FjallDb::init(keyspace))
}
// -- err
impl From<fjall::Error> for Error {
fn from(e: fjall::Error) -> Error {
Error(format!("fjall: {}", e).into())
}
}
impl From<fjall::LsmError> for Error {
fn from(e: fjall::LsmError) -> Error {
Error(format!("fjall lsm_tree: {}", e).into())
}
}
impl From<fjall::Error> for TxOpError {
fn from(e: fjall::Error) -> TxOpError {
TxOpError(e.into())
}
}
// -- db
pub struct FjallDb {
keyspace: TransactionalKeyspace,
trees: RwLock<Vec<(String, TransactionalPartitionHandle)>>,
}
type ByteRefRangeBound<'r> = (Bound<&'r [u8]>, Bound<&'r [u8]>);
impl FjallDb {
pub fn init(keyspace: TransactionalKeyspace) -> Db {
let s = Self {
keyspace,
trees: RwLock::new(Vec::new()),
};
Db(Arc::new(s))
}
fn get_tree(
&self,
i: usize,
) -> Result<MappedRwLockReadGuard<'_, TransactionalPartitionHandle>> {
RwLockReadGuard::try_map(self.trees.read(), |trees: &Vec<_>| {
trees.get(i).map(|tup| &tup.1)
})
.map_err(|_| Error("invalid tree id".into()))
}
}
impl IDb for FjallDb {
fn engine(&self) -> String {
"Fjall (EXPERIMENTAL!)".into()
}
fn open_tree(&self, name: &str) -> Result<usize> {
let mut trees = self.trees.write();
let safe_name = encode_name(name)?;
if let Some(i) = trees.iter().position(|(name, _)| *name == safe_name) {
Ok(i)
} else {
let tree = self
.keyspace
.open_partition(&safe_name, PartitionCreateOptions::default())?;
let i = trees.len();
trees.push((safe_name, tree));
Ok(i)
}
}
fn list_trees(&self) -> Result<Vec<String>> {
Ok(self
.keyspace
.list_partitions()
.iter()
.map(|n| decode_name(&n))
.collect::<Result<Vec<_>>>()?)
}
fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
std::fs::create_dir_all(base_path)?;
let path = Engine::Fjall.db_path(base_path);
let source_state = self.keyspace.read_tx();
let copy_keyspace = fjall::Config::new(path).open()?;
for partition_name in self.keyspace.list_partitions() {
let source_partition = self
.keyspace
.open_partition(&partition_name, PartitionCreateOptions::default())?;
let copy_partition =
copy_keyspace.open_partition(&partition_name, PartitionCreateOptions::default())?;
for entry in source_state.iter(&source_partition) {
let (key, value) = entry?;
copy_partition.insert(key, value)?;
}
}
copy_keyspace.persist(PersistMode::SyncAll)?;
Ok(())
}
// ----
fn get(&self, tree_idx: usize, key: &[u8]) -> Result<Option<Value>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
let val = tx.get(&tree, key)?;
match val {
None => Ok(None),
Some(v) => Ok(Some(v.to_vec())),
}
}
fn approximate_len(&self, tree_idx: usize) -> Result<usize> {
let tree = self.get_tree(tree_idx)?;
Ok(tree.approximate_len())
}
fn is_empty(&self, tree_idx: usize) -> Result<bool> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(tx.is_empty(&tree)?)
}
fn insert(&self, tree_idx: usize, key: &[u8], value: &[u8]) -> Result<()> {
let tree = self.get_tree(tree_idx)?;
let mut tx = self.keyspace.write_tx();
tx.insert(&tree, key, value);
tx.commit()?;
Ok(())
}
fn remove(&self, tree_idx: usize, key: &[u8]) -> Result<()> {
let tree = self.get_tree(tree_idx)?;
let mut tx = self.keyspace.write_tx();
tx.remove(&tree, key);
tx.commit()?;
Ok(())
}
fn clear(&self, tree_idx: usize) -> Result<()> {
let mut trees = self.trees.write();
if tree_idx >= trees.len() {
return Err(Error("invalid tree id".into()));
}
let (name, tree) = trees.remove(tree_idx);
self.keyspace.delete_partition(tree)?;
let tree = self
.keyspace
.open_partition(&name, PartitionCreateOptions::default())?;
trees.insert(tree_idx, (name, tree));
Ok(())
}
fn iter(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(tx.iter(&tree).map(iterator_remap)))
}
fn iter_rev(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(tx.iter(&tree).rev().map(iterator_remap)))
}
fn range<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(
tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high))
.map(iterator_remap),
))
}
fn range_rev<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(
tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high))
.rev()
.map(iterator_remap),
))
}
// ----
fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()> {
let trees = self.trees.read();
let mut tx = FjallTx {
trees: &trees[..],
tx: self.keyspace.write_tx(),
};
let res = f.try_on(&mut tx);
match res {
TxFnResult::Ok(on_commit) => {
tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?;
Ok(on_commit)
}
TxFnResult::Abort => {
tx.tx.rollback();
Err(TxError::Abort(()))
}
TxFnResult::DbErr => {
tx.tx.rollback();
Err(TxError::Db(Error(
"(this message will be discarded)".into(),
)))
}
}
}
}
// ----
struct FjallTx<'a> {
trees: &'a [(String, TransactionalPartitionHandle)],
tx: WriteTransaction<'a>,
}
impl<'a> FjallTx<'a> {
fn get_tree(&self, i: usize) -> TxOpResult<&TransactionalPartitionHandle> {
self.trees.get(i).map(|tup| &tup.1).ok_or_else(|| {
TxOpError(Error(
"invalid tree id (it might have been openned after the transaction started)".into(),
))
})
}
}
impl<'a> ITx for FjallTx<'a> {
fn get(&self, tree_idx: usize, key: &[u8]) -> TxOpResult<Option<Value>> {
let tree = self.get_tree(tree_idx)?;
match self.tx.get(tree, key)? {
Some(v) => Ok(Some(v.to_vec())),
None => Ok(None),
}
}
fn len(&self, tree_idx: usize) -> TxOpResult<usize> {
let tree = self.get_tree(tree_idx)?;
Ok(self.tx.len(tree)? as usize)
}
fn insert(&mut self, tree_idx: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> {
let tree = self.get_tree(tree_idx)?.clone();
self.tx.insert(&tree, key, value);
Ok(())
}
fn remove(&mut self, tree_idx: usize, key: &[u8]) -> TxOpResult<()> {
let tree = self.get_tree(tree_idx)?.clone();
self.tx.remove(&tree, key);
Ok(())
}
fn clear(&mut self, _tree_idx: usize) -> TxOpResult<()> {
unimplemented!("LSM tree clearing in cross-partition transaction is not supported")
}
fn iter(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?.clone();
Ok(Box::new(self.tx.iter(&tree).map(iterator_remap_tx)))
}
fn iter_rev(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?.clone();
Ok(Box::new(self.tx.iter(&tree).rev().map(iterator_remap_tx)))
}
fn range<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let low = clone_bound(low);
let high = clone_bound(high);
Ok(Box::new(
self.tx
.range::<Vec<u8>, ByteVecRangeBounds>(&tree, (low, high))
.map(iterator_remap_tx),
))
}
fn range_rev<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let low = clone_bound(low);
let high = clone_bound(high);
Ok(Box::new(
self.tx
.range::<Vec<u8>, ByteVecRangeBounds>(&tree, (low, high))
.rev()
.map(iterator_remap_tx),
))
}
}
// -- maps fjall's (k, v) to ours
fn iterator_remap(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> Result<(Value, Value)> {
r.map(|(k, v)| (k.to_vec(), v.to_vec()))
.map_err(|e| e.into())
}
fn iterator_remap_tx(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> TxOpResult<(Value, Value)> {
r.map(|(k, v)| (k.to_vec(), v.to_vec()))
.map_err(|e| e.into())
}
// -- utils to deal with Garage's tightness on Bound lifetimes
type ByteVecBound = Bound<Vec<u8>>;
type ByteVecRangeBounds = (ByteVecBound, ByteVecBound);
fn clone_bound(bound: Bound<&[u8]>) -> ByteVecBound {
let value = match bound {
Bound::Excluded(v) | Bound::Included(v) => v.to_vec(),
Bound::Unbounded => vec![],
};
match bound {
Bound::Included(_) => Bound::Included(value),
Bound::Excluded(_) => Bound::Excluded(value),
Bound::Unbounded => Bound::Unbounded,
}
}
// -- utils to encode table names --
fn encode_name(s: &str) -> Result<String> {
let base = 'A' as u32;
let mut ret = String::with_capacity(s.len() + 10);
for c in s.chars() {
if c.is_alphanumeric() || c == '_' || c == '-' || c == '#' {
ret.push(c);
} else if c <= u8::MAX as char {
ret.push('$');
let c_hi = c as u32 / 16;
let c_lo = c as u32 % 16;
ret.push(char::from_u32(base + c_hi).unwrap());
ret.push(char::from_u32(base + c_lo).unwrap());
} else {
return Err(Error(
format!("table name {} could not be safely encoded", s).into(),
));
}
}
Ok(ret)
}
fn decode_name(s: &str) -> Result<String> {
use std::convert::TryFrom;
let errfn = || Error(format!("encoded table name {} is invalid", s).into());
let c_map = |c: char| {
let c = c as u32;
let base = 'A' as u32;
if (base..base + 16).contains(&c) {
Some(c - base)
} else {
None
}
};
let mut ret = String::with_capacity(s.len());
let mut it = s.chars();
while let Some(c) = it.next() {
if c == '$' {
let c_hi = it.next().and_then(c_map).ok_or_else(errfn)?;
let c_lo = it.next().and_then(c_map).ok_or_else(errfn)?;
let c_dec = char::try_from(c_hi * 16 + c_lo).map_err(|_| errfn())?;
ret.push(c_dec);
} else {
ret.push(c);
}
}
Ok(ret)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_encdec_name() {
for name in [
"testname",
"test_name",
"test name",
"test$name",
"test:name@help.me$get/this**right",
] {
let encname = encode_name(name).unwrap();
assert!(!encname.contains(' '));
assert!(!encname.contains('.'));
assert!(!encname.contains('*'));
assert_eq!(*name, decode_name(&encname).unwrap());
}
}
}
+48 -29
View File
@@ -1,6 +1,8 @@
#[macro_use] #[macro_use]
extern crate tracing; extern crate tracing;
#[cfg(feature = "fjall")]
pub mod fjall_adapter;
#[cfg(feature = "lmdb")] #[cfg(feature = "lmdb")]
pub mod lmdb_adapter; pub mod lmdb_adapter;
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]
@@ -104,32 +106,44 @@ impl Db {
result: Cell::new(None), result: Cell::new(None),
}; };
let tx_res = self.0.transaction(&f); let tx_res = self.0.transaction(&f);
let ret = f let fn_res = f.result.into_inner();
.result
.into_inner()
.expect("Transaction did not store result");
match tx_res { match (tx_res, fn_res) {
Ok(on_commit) => match ret { (Ok(on_commit), Some(Ok(value))) => {
Ok(value) => { // Transaction succeeded
on_commit.into_iter().for_each(|f| f()); // TxFn stored the value to return to the user in fn_res
Ok(value) // tx_res contains the on_commit list of callbacks, run them now
} on_commit.into_iter().for_each(|f| f());
_ => unreachable!(), Ok(value)
}, }
Err(TxError::Abort(())) => match ret { (Err(TxError::Abort(())), Some(Err(TxError::Abort(e)))) => {
Err(TxError::Abort(e)) => Err(TxError::Abort(e)), // Transaction was aborted by user code
_ => unreachable!(), // The abort error value is stored in fn_res
}, Err(TxError::Abort(e))
Err(TxError::Db(e2)) => match ret { }
// Ok was stored -> the error occurred when finalizing (Err(TxError::Db(_tx_e)), Some(Err(TxError::Db(fn_e)))) => {
// transaction // Transaction encountered a DB error in user code
Ok(_) => Err(TxError::Db(e2)), // The error value encountered is the one in fn_res,
// An error was already stored: that's the one we want to // tx_res contains only a dummy error message
// return Err(TxError::Db(fn_e))
Err(TxError::Db(e)) => Err(TxError::Db(e)), }
_ => unreachable!(), (Err(TxError::Db(tx_e)), None) => {
}, // Transaction encounterred a DB error when initializing the transaction,
// before user code was called
Err(TxError::Db(tx_e))
}
(Err(TxError::Db(tx_e)), Some(Ok(_))) => {
// Transaction encounterred a DB error when commiting the transaction,
// after user code was called
Err(TxError::Db(tx_e))
}
(tx_res, fn_res) => {
panic!(
"unexpected error case: tx_res={:?}, fn_res={:?}",
tx_res.map(|_| "..."),
fn_res.map(|x| x.map(|_| "...").map_err(|_| "..."))
);
}
} }
} }
@@ -152,7 +166,7 @@ impl Db {
let tree_names = other.list_trees()?; let tree_names = other.list_trees()?;
for name in tree_names { for name in tree_names {
let tree = self.open_tree(&name)?; let tree = self.open_tree(&name)?;
if tree.len()? > 0 { if !tree.is_empty()? {
return Err(Error(format!("tree {} already contains data", name).into())); return Err(Error(format!("tree {} already contains data", name).into()));
} }
@@ -194,8 +208,12 @@ impl Tree {
self.0.get(self.1, key.as_ref()) self.0.get(self.1, key.as_ref())
} }
#[inline] #[inline]
pub fn len(&self) -> Result<usize> { pub fn approximate_len(&self) -> Result<usize> {
self.0.len(self.1) self.0.approximate_len(self.1)
}
#[inline]
pub fn is_empty(&self) -> Result<bool> {
self.0.is_empty(self.1)
} }
#[inline] #[inline]
@@ -333,7 +351,8 @@ pub(crate) trait IDb: Send + Sync {
fn snapshot(&self, path: &PathBuf) -> Result<()>; fn snapshot(&self, path: &PathBuf) -> Result<()>;
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>; fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
fn len(&self, tree: usize) -> Result<usize>; fn approximate_len(&self, tree: usize) -> Result<usize>;
fn is_empty(&self, tree: usize) -> Result<bool>;
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>;
fn remove(&self, tree: usize, key: &[u8]) -> Result<()>; fn remove(&self, tree: usize, key: &[u8]) -> Result<()>;
+94 -29
View File
@@ -1,8 +1,8 @@
use core::ops::Bound; use core::ops::Bound;
use core::ptr::NonNull;
use std::collections::HashMap; use std::collections::HashMap;
use std::convert::TryInto; use std::convert::TryInto;
use std::marker::PhantomPinned;
use std::path::PathBuf; use std::path::PathBuf;
use std::pin::Pin; use std::pin::Pin;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
@@ -11,12 +11,55 @@ use heed::types::ByteSlice;
use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database}; use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database};
use crate::{ use crate::{
open::{Engine, OpenOpt},
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult, Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
TxResult, TxValueIter, Value, ValueIter, TxResult, TxValueIter, Value, ValueIter,
}; };
pub use heed; pub use heed;
// ---- top-level open function
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
info!("Opening LMDB database at: {}", path.display());
if let Err(e) = std::fs::create_dir_all(&path) {
return Err(Error(
format!("Unable to create LMDB data directory: {}", e).into(),
));
}
let map_size = match opt.lmdb_map_size {
None => recommended_map_size(),
Some(v) => v - (v % 4096),
};
let mut env_builder = heed::EnvOpenOptions::new();
env_builder.max_dbs(100);
env_builder.map_size(map_size);
env_builder.max_readers(2048);
unsafe {
env_builder.flag(heed::flags::Flags::MdbNoRdAhead);
env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
if !opt.fsync {
env_builder.flag(heed::flags::Flags::MdbNoSync);
}
}
match env_builder.open(&path) {
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
return Err(Error(
"OutOfMemory error while trying to open LMDB database. This can happen \
if your operating system is not allowing you to use sufficient virtual \
memory address space. Please check that no limit is set (ulimit -v). \
You may also try to set a smaller `lmdb_map_size` configuration parameter. \
On 32-bit machines, you should probably switch to another database engine."
.into(),
))
}
Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
Ok(db) => Ok(LmdbDb::init(db)),
}
}
// -- err // -- err
impl From<heed::Error> for Error { impl From<heed::Error> for Error {
@@ -104,10 +147,9 @@ impl IDb for LmdbDb {
Ok(ret2) Ok(ret2)
} }
fn snapshot(&self, to: &PathBuf) -> Result<()> { fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
std::fs::create_dir_all(to)?; std::fs::create_dir_all(base_path)?;
let mut path = to.clone(); let path = Engine::Lmdb.db_path(base_path);
path.push("data.mdb");
self.db self.db
.copy_to_path(path, heed::CompactionOption::Enabled)?; .copy_to_path(path, heed::CompactionOption::Enabled)?;
Ok(()) Ok(())
@@ -126,11 +168,16 @@ impl IDb for LmdbDb {
} }
} }
fn len(&self, tree: usize) -> Result<usize> { fn approximate_len(&self, tree: usize) -> Result<usize> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?; let tx = self.db.read_txn()?;
Ok(tree.len(&tx)?.try_into().unwrap()) Ok(tree.len(&tx)?.try_into().unwrap())
} }
fn is_empty(&self, tree: usize) -> Result<bool> {
let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?;
Ok(tree.is_empty(&tx)?)
}
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
@@ -159,13 +206,15 @@ impl IDb for LmdbDb {
fn iter(&self, tree: usize) -> Result<ValueIter<'_>> { fn iter(&self, tree: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?; let tx = self.db.read_txn()?;
TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) // Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) }
} }
fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> { fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?; let tx = self.db.read_txn()?;
TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?)) // Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?)) }
} }
fn range<'r>( fn range<'r>(
@@ -176,7 +225,8 @@ impl IDb for LmdbDb {
) -> Result<ValueIter<'_>> { ) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?; let tx = self.db.read_txn()?;
TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?)) // Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?)) }
} }
fn range_rev<'r>( fn range_rev<'r>(
&self, &self,
@@ -186,7 +236,8 @@ impl IDb for LmdbDb {
) -> Result<ValueIter<'_>> { ) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?; let tx = self.db.read_txn()?;
TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?)) // Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?)) }
} }
// ---- // ----
@@ -316,28 +367,41 @@ where
{ {
tx: RoTxn<'a>, tx: RoTxn<'a>,
iter: Option<I>, iter: Option<I>,
_pin: PhantomPinned,
} }
impl<'a, I> TxAndIterator<'a, I> impl<'a, I> TxAndIterator<'a, I>
where where
I: Iterator<Item = IteratorItem<'a>> + 'a, I: Iterator<Item = IteratorItem<'a>> + 'a,
{ {
fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>> fn iter(self: Pin<&mut Self>) -> &mut Option<I> {
// Safety: iter is not structural
unsafe { &mut self.get_unchecked_mut().iter }
}
/// Safety: iterfun must not store its argument anywhere but in its result.
unsafe fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>>
where where
F: FnOnce(&'a RoTxn<'a>) -> Result<I>, F: FnOnce(&'a RoTxn<'a>) -> Result<I>,
{ {
let res = TxAndIterator { tx, iter: None }; let res = TxAndIterator {
tx,
iter: None,
_pin: PhantomPinned,
};
let mut boxed = Box::pin(res); let mut boxed = Box::pin(res);
// This unsafe allows us to bypass lifetime checks let tx_lifetime_overextended: &'a RoTxn<'a> = {
let tx = unsafe { NonNull::from(&boxed.tx).as_ref() }; let tx = &boxed.tx;
let iter = iterfun(tx)?; // Safety: Artificially extending the lifetime because
// this reference will only be stored and accessed from the
// returned ValueIter which guarantees that it is destroyed
// before the tx it is pointing to.
unsafe { &*&raw const *tx }
};
let iter = iterfun(&tx_lifetime_overextended)?;
let mut_ref = Pin::as_mut(&mut boxed); *boxed.as_mut().iter() = Some(iter);
// This unsafe allows us to write in a field of the pinned struct
unsafe {
Pin::get_unchecked_mut(mut_ref).iter = Some(iter);
}
Ok(Box::new(TxAndIteratorPin(boxed))) Ok(Box::new(TxAndIteratorPin(boxed)))
} }
@@ -348,8 +412,10 @@ where
I: Iterator<Item = IteratorItem<'a>> + 'a, I: Iterator<Item = IteratorItem<'a>> + 'a,
{ {
fn drop(&mut self) { fn drop(&mut self) {
// ensure the iterator is dropped before the RoTxn it references // Safety: `new_unchecked` is okay because we know this value is never
drop(self.iter.take()); // used again after being dropped.
let this = unsafe { Pin::new_unchecked(self) };
drop(this.iter().take());
} }
} }
@@ -365,13 +431,12 @@ where
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
let mut_ref = Pin::as_mut(&mut self.0); let mut_ref = Pin::as_mut(&mut self.0);
// This unsafe allows us to mutably access the iterator field let next = mut_ref.iter().as_mut()?.next()?;
let next = unsafe { Pin::get_unchecked_mut(mut_ref).iter.as_mut()?.next() }; let res = match next {
match next { Err(e) => Err(e.into()),
None => None, Ok((k, v)) => Ok((k.to_vec(), v.to_vec())),
Some(Err(e)) => Some(Err(e.into())), };
Some(Ok((k, v))) => Some(Ok((k.to_vec(), v.to_vec()))), Some(res)
}
} }
} }
+28 -44
View File
@@ -11,6 +11,7 @@ use crate::{Db, Error, Result};
pub enum Engine { pub enum Engine {
Lmdb, Lmdb,
Sqlite, Sqlite,
Fjall,
} }
impl Engine { impl Engine {
@@ -19,8 +20,26 @@ impl Engine {
match self { match self {
Self::Lmdb => "lmdb", Self::Lmdb => "lmdb",
Self::Sqlite => "sqlite", Self::Sqlite => "sqlite",
Self::Fjall => "fjall",
} }
} }
/// Return engine-specific DB path from base path
pub fn db_path(&self, base_path: &PathBuf) -> PathBuf {
let mut ret = base_path.clone();
match self {
Self::Lmdb => {
ret.push("db.lmdb");
}
Self::Sqlite => {
ret.push("db.sqlite");
}
Self::Fjall => {
ret.push("db.fjall");
}
}
ret
}
} }
impl std::fmt::Display for Engine { impl std::fmt::Display for Engine {
@@ -36,10 +55,11 @@ impl std::str::FromStr for Engine {
match text { match text {
"lmdb" | "heed" => Ok(Self::Lmdb), "lmdb" | "heed" => Ok(Self::Lmdb),
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite), "sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
"fjall" => Ok(Self::Fjall),
"sled" => Err(Error("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into())), "sled" => Err(Error("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into())),
kind => Err(Error( kind => Err(Error(
format!( format!(
"Invalid DB engine: {} (options are: lmdb, sqlite)", "Invalid DB engine: {} (options are: lmdb, sqlite, fjall)",
kind kind
) )
.into(), .into(),
@@ -51,6 +71,7 @@ impl std::str::FromStr for Engine {
pub struct OpenOpt { pub struct OpenOpt {
pub fsync: bool, pub fsync: bool,
pub lmdb_map_size: Option<usize>, pub lmdb_map_size: Option<usize>,
pub fjall_block_cache_size: Option<usize>,
} }
impl Default for OpenOpt { impl Default for OpenOpt {
@@ -58,6 +79,7 @@ impl Default for OpenOpt {
Self { Self {
fsync: false, fsync: false,
lmdb_map_size: None, lmdb_map_size: None,
fjall_block_cache_size: None,
} }
} }
} }
@@ -66,53 +88,15 @@ pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
match engine { match engine {
// ---- Sqlite DB ---- // ---- Sqlite DB ----
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]
Engine::Sqlite => { Engine::Sqlite => crate::sqlite_adapter::open_db(path, opt),
info!("Opening Sqlite database at: {}", path.display());
let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
Ok(crate::sqlite_adapter::SqliteDb::new(manager, opt.fsync)?)
}
// ---- LMDB DB ---- // ---- LMDB DB ----
#[cfg(feature = "lmdb")] #[cfg(feature = "lmdb")]
Engine::Lmdb => { Engine::Lmdb => crate::lmdb_adapter::open_db(path, opt),
info!("Opening LMDB database at: {}", path.display());
if let Err(e) = std::fs::create_dir_all(&path) {
return Err(Error(
format!("Unable to create LMDB data directory: {}", e).into(),
));
}
let map_size = match opt.lmdb_map_size { // ---- Fjall DB ----
None => crate::lmdb_adapter::recommended_map_size(), #[cfg(feature = "fjall")]
Some(v) => v - (v % 4096), Engine::Fjall => crate::fjall_adapter::open_db(path, opt),
};
let mut env_builder = heed::EnvOpenOptions::new();
env_builder.max_dbs(100);
env_builder.map_size(map_size);
env_builder.max_readers(2048);
unsafe {
env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoRdAhead);
env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
if !opt.fsync {
env_builder.flag(heed::flags::Flags::MdbNoSync);
}
}
match env_builder.open(&path) {
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
return Err(Error(
"OutOfMemory error while trying to open LMDB database. This can happen \
if your operating system is not allowing you to use sufficient virtual \
memory address space. Please check that no limit is set (ulimit -v). \
You may also try to set a smaller `lmdb_map_size` configuration parameter. \
On 32-bit machines, you should probably switch to another database engine."
.into(),
))
}
Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
Ok(db) => Ok(crate::lmdb_adapter::LmdbDb::init(db)),
}
}
// Pattern is unreachable when all supported DB engines are compiled into binary. The allow // Pattern is unreachable when all supported DB engines are compiled into binary. The allow
// attribute is added so that we won't have to change this match in case stop building // attribute is added so that we won't have to change this match in case stop building
+28 -12
View File
@@ -11,12 +11,23 @@ use r2d2_sqlite::SqliteConnectionManager;
use rusqlite::{params, Rows, Statement, Transaction}; use rusqlite::{params, Rows, Statement, Transaction};
use crate::{ use crate::{
open::{Engine, OpenOpt},
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult, Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
TxResult, TxValueIter, Value, ValueIter, TxResult, TxValueIter, Value, ValueIter,
}; };
pub use rusqlite; pub use rusqlite;
// ---- top-level open function
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
info!("Opening Sqlite database at: {}", path.display());
let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
Ok(SqliteDb::new(manager, opt.fsync)?)
}
// ----
type Connection = r2d2::PooledConnection<SqliteConnectionManager>; type Connection = r2d2::PooledConnection<SqliteConnectionManager>;
// --- err // --- err
@@ -139,17 +150,18 @@ impl IDb for SqliteDb {
Ok(trees) Ok(trees)
} }
fn snapshot(&self, to: &PathBuf) -> Result<()> { fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
fn progress(p: rusqlite::backup::Progress) { std::fs::create_dir_all(base_path)?;
let percent = (p.pagecount - p.remaining) * 100 / p.pagecount; let path = Engine::Sqlite
info!("Sqlite snapshot progress: {}%", percent); .db_path(&base_path)
} .into_os_string()
std::fs::create_dir_all(to)?; .into_string()
let mut path = to.clone(); .map_err(|_| Error("invalid sqlite path string".into()))?;
path.push("db.sqlite");
self.db info!("Start sqlite VACUUM INTO `{}`", path);
.get()? self.db.get()?.execute("VACUUM INTO ?1", params![path])?;
.backup(rusqlite::DatabaseName::Main, path, Some(progress))?; info!("Finished sqlite VACUUM INTO `{}`", path);
Ok(()) Ok(())
} }
@@ -160,7 +172,7 @@ impl IDb for SqliteDb {
self.internal_get(&self.db.get()?, &tree, key) self.internal_get(&self.db.get()?, &tree, key)
} }
fn len(&self, tree: usize) -> Result<usize> { fn approximate_len(&self, tree: usize) -> Result<usize> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
let db = self.db.get()?; let db = self.db.get()?;
@@ -172,6 +184,10 @@ impl IDb for SqliteDb {
} }
} }
fn is_empty(&self, tree: usize) -> Result<bool> {
Ok(self.approximate_len(tree)? == 0)
}
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
let db = self.db.get()?; let db = self.db.get()?;
+14 -2
View File
@@ -1,7 +1,7 @@
use crate::*; use crate::*;
fn test_suite(db: Db) { fn test_suite(db: Db) {
let tree = db.open_tree("tree").unwrap(); let tree = db.open_tree("tree:this_is_a_tree").unwrap();
let ka: &[u8] = &b"test"[..]; let ka: &[u8] = &b"test"[..];
let kb: &[u8] = &b"zwello"[..]; let kb: &[u8] = &b"zwello"[..];
@@ -14,7 +14,7 @@ fn test_suite(db: Db) {
assert!(tree.insert(ka, va).is_ok()); assert!(tree.insert(ka, va).is_ok());
assert_eq!(tree.get(ka).unwrap().unwrap(), va); assert_eq!(tree.get(ka).unwrap().unwrap(), va);
assert_eq!(tree.len().unwrap(), 1); assert_eq!(tree.iter().unwrap().count(), 1);
// ---- test transaction logic ---- // ---- test transaction logic ----
@@ -148,3 +148,15 @@ fn test_sqlite_db() {
let db = SqliteDb::new(manager, false).unwrap(); let db = SqliteDb::new(manager, false).unwrap();
test_suite(db); test_suite(db);
} }
#[test]
#[cfg(feature = "fjall")]
fn test_fjall_db() {
use crate::fjall_adapter::{fjall, FjallDb};
let path = mktemp::Temp::new_dir().unwrap();
let config = fjall::Config::new(path).temporary(true);
let keyspace = config.open_transactional().unwrap();
let db = FjallDb::init(keyspace);
test_suite(db);
}
+2 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage" name = "garage"
version = "1.2.0" version = "1.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@@ -91,6 +91,7 @@ k2v = [ "garage_util/k2v", "garage_api_k2v" ]
# Database engines # Database engines
lmdb = [ "garage_model/lmdb" ] lmdb = [ "garage_model/lmdb" ]
sqlite = [ "garage_model/sqlite" ] sqlite = [ "garage_model/sqlite" ]
fjall = [ "garage_model/fjall" ]
# Automatic registration and discovery via Consul API # Automatic registration and discovery via Consul API
consul-discovery = [ "garage_rpc/consul-discovery" ] consul-discovery = [ "garage_rpc/consul-discovery" ]
+12 -7
View File
@@ -219,7 +219,7 @@ impl AdminRpcHandler {
// Gather block manager statistics // Gather block manager statistics
writeln!(&mut ret, "\nBlock manager stats:").unwrap(); writeln!(&mut ret, "\nBlock manager stats:").unwrap();
let rc_len = self.garage.block_manager.rc_len()?.to_string(); let rc_len = self.garage.block_manager.rc_approximate_len()?.to_string();
writeln!( writeln!(
&mut ret, &mut ret,
@@ -230,13 +230,13 @@ impl AdminRpcHandler {
writeln!( writeln!(
&mut ret, &mut ret,
" resync queue length: {}", " resync queue length: {}",
self.garage.block_manager.resync.queue_len()? self.garage.block_manager.resync.queue_approximate_len()?
) )
.unwrap(); .unwrap();
writeln!( writeln!(
&mut ret, &mut ret,
" blocks with resync errors: {}", " blocks with resync errors: {}",
self.garage.block_manager.resync.errors_len()? self.garage.block_manager.resync.errors_approximate_len()?
) )
.unwrap(); .unwrap();
@@ -346,16 +346,21 @@ impl AdminRpcHandler {
F: TableSchema + 'static, F: TableSchema + 'static,
R: TableReplication + 'static, R: TableReplication + 'static,
{ {
let data_len = t.data.store.len().map_err(GarageError::from)?.to_string(); let data_len = t
let mkl_len = t.merkle_updater.merkle_tree_len()?.to_string(); .data
.store
.approximate_len()
.map_err(GarageError::from)?
.to_string();
let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?.to_string();
Ok(format!( Ok(format!(
" {}\t{}\t{}\t{}\t{}", " {}\t{}\t{}\t{}\t{}",
F::TABLE_NAME, F::TABLE_NAME,
data_len, data_len,
mkl_len, mkl_len,
t.merkle_updater.todo_len()?, t.merkle_updater.todo_approximate_len()?,
t.data.gc_todo_len()? t.data.gc_todo_approximate_len()?
)) ))
} }
+4
View File
@@ -466,6 +466,10 @@ pub enum RepairWhat {
/// Repair (resync/rebalance) the set of stored blocks in the cluster /// Repair (resync/rebalance) the set of stored blocks in the cluster
#[structopt(name = "blocks", version = garage_version())] #[structopt(name = "blocks", version = garage_version())]
Blocks, Blocks,
/// Clear the block resync queue. The list of blocks in errored state
/// is cleared as well. You MUST run `garage repair blocks` after invoking this.
#[structopt(name = "clear-resync-queue", version = garage_version())]
ClearResyncQueue,
/// Repropagate object deletions to the version table /// Repropagate object deletions to the version table
#[structopt(name = "versions", version = garage_version())] #[structopt(name = "versions", version = garage_version())]
Versions, Versions,
+5
View File
@@ -92,6 +92,11 @@ pub async fn launch_online_repair(
info!("Repairing bucket aliases (foreground)"); info!("Repairing bucket aliases (foreground)");
garage.locked_helper().await.repair_aliases().await?; garage.locked_helper().await.repair_aliases().await?;
} }
RepairWhat::ClearResyncQueue => {
let garage = garage.clone();
tokio::task::spawn_blocking(move || garage.block_manager.resync.clear_resync_queue())
.await??
}
} }
Ok(()) Ok(())
} }
+15 -4
View File
@@ -183,10 +183,21 @@ fn watch_shutdown_signal() -> watch::Receiver<bool> {
let mut sigterm = let mut sigterm =
signal(SignalKind::terminate()).expect("Failed to install SIGTERM handler"); signal(SignalKind::terminate()).expect("Failed to install SIGTERM handler");
let mut sighup = signal(SignalKind::hangup()).expect("Failed to install SIGHUP handler"); let mut sighup = signal(SignalKind::hangup()).expect("Failed to install SIGHUP handler");
tokio::select! { loop {
_ = sigint.recv() => info!("Received SIGINT, shutting down."), tokio::select! {
_ = sigterm.recv() => info!("Received SIGTERM, shutting down."), _ = sigint.recv() => {
_ = sighup.recv() => info!("Received SIGHUP, shutting down."), info!("Received SIGINT, shutting down.");
break
}
_ = sigterm.recv() => {
info!("Received SIGTERM, shutting down.");
break
}
_ = sighup.recv() => {
info!("Received SIGHUP, reload not supported.");
continue
}
}
} }
send_cancel.send(true).unwrap(); send_cancel.send(true).unwrap();
}); });
+10
View File
@@ -72,6 +72,16 @@ impl K2vClient {
.enable_http2() .enable_http2()
.build(); .build();
let client = HttpClient::builder(TokioExecutor::new()).build(connector); let client = HttpClient::builder(TokioExecutor::new()).build(connector);
Self::new_with_client(config, client)
}
/// Create a new K2V client with an external client.
/// Useful for example if you plan on creating many clients but you want to mutualize the
/// underlying thread pools & co.
pub fn new_with_client(
config: K2vClientConfig,
client: HttpClient<HttpsConnector<HttpConnector>, Body>,
) -> Result<Self, Error> {
let user_agent: std::borrow::Cow<str> = match &config.user_agent { let user_agent: std::borrow::Cow<str> = match &config.user_agent {
Some(ua) => ua.into(), Some(ua) => ua.into(),
None => format!("k2v/{}", env!("CARGO_PKG_VERSION")).into(), None => format!("k2v/{}", env!("CARGO_PKG_VERSION")).into(),
+2 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_model" name = "garage_model"
version = "1.2.0" version = "1.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@@ -44,3 +44,4 @@ default = [ "lmdb", "sqlite" ]
k2v = [ "garage_util/k2v" ] k2v = [ "garage_util/k2v" ]
lmdb = [ "garage_db/lmdb" ] lmdb = [ "garage_db/lmdb" ]
sqlite = [ "garage_db/sqlite" ] sqlite = [ "garage_db/sqlite" ]
fjall = [ "garage_db/fjall" ]
+5 -9
View File
@@ -116,21 +116,17 @@ impl Garage {
info!("Opening database..."); info!("Opening database...");
let db_engine = db::Engine::from_str(&config.db_engine) let db_engine = db::Engine::from_str(&config.db_engine)
.ok_or_message("Invalid `db_engine` value in configuration file")?; .ok_or_message("Invalid `db_engine` value in configuration file")?;
let mut db_path = config.metadata_dir.clone(); let db_path = db_engine.db_path(&config.metadata_dir);
match db_engine {
db::Engine::Sqlite => {
db_path.push("db.sqlite");
}
db::Engine::Lmdb => {
db_path.push("db.lmdb");
}
}
let db_opt = db::OpenOpt { let db_opt = db::OpenOpt {
fsync: config.metadata_fsync, fsync: config.metadata_fsync,
lmdb_map_size: match config.lmdb_map_size { lmdb_map_size: match config.lmdb_map_size {
v if v == usize::default() => None, v if v == usize::default() => None,
v => Some(v), v => Some(v),
}, },
fjall_block_cache_size: match config.fjall_block_cache_size {
v if v == usize::default() => None,
v => Some(v),
},
}; };
let db = db::open_db(&db_path, db_engine, &db_opt) let db = db::open_db(&db_path, db_engine, &db_opt)
.ok_or_message("Unable to open metadata db")?; .ok_or_message("Unable to open metadata db")?;
+3 -3
View File
@@ -121,13 +121,13 @@ impl Worker for LifecycleWorker {
mpu_aborted, mpu_aborted,
.. ..
} => { } => {
let n_objects = self.garage.object_table.data.store.len().ok(); let n_objects = self.garage.object_table.data.store.approximate_len().ok();
let progress = match n_objects { let progress = match n_objects {
None => "...".to_string(), Some(total) if total > 0 => format!(
Some(total) => format!(
"~{:.2}%", "~{:.2}%",
100. * std::cmp::min(*counter, total) as f32 / total as f32 100. * std::cmp::min(*counter, total) as f32 / total as f32
), ),
_ => "...".to_string(),
}; };
WorkerStatus { WorkerStatus {
progress: Some(progress), progress: Some(progress),
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_net" name = "garage_net"
version = "1.2.0" version = "1.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_rpc" name = "garage_rpc"
version = "1.2.0" version = "1.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+6 -8
View File
@@ -229,13 +229,11 @@ impl LayoutManager {
} }
/// Save cluster layout data to disk /// Save cluster layout data to disk
async fn save_cluster_layout(&self) -> Result<(), Error> { async fn save_cluster_layout(&self) {
let layout = self.layout.read().unwrap().inner().clone(); let layout = self.layout.read().unwrap().inner().clone();
self.persist_cluster_layout if let Err(e) = self.persist_cluster_layout.save_async(&layout).await {
.save_async(&layout) error!("Failed to save cluster_layout: {}", e);
.await }
.expect("Cannot save current cluster layout");
Ok(())
} }
fn broadcast_update(self: &Arc<Self>, rpc: SystemRpc) { fn broadcast_update(self: &Arc<Self>, rpc: SystemRpc) {
@@ -313,7 +311,7 @@ impl LayoutManager {
self.change_notify.notify_waiters(); self.change_notify.notify_waiters();
self.broadcast_update(SystemRpc::AdvertiseClusterLayout(new_layout)); self.broadcast_update(SystemRpc::AdvertiseClusterLayout(new_layout));
self.save_cluster_layout().await?; self.save_cluster_layout().await;
} }
Ok(SystemRpc::Ok) Ok(SystemRpc::Ok)
@@ -328,7 +326,7 @@ impl LayoutManager {
if let Some(new_trackers) = self.merge_layout_trackers(trackers) { if let Some(new_trackers) = self.merge_layout_trackers(trackers) {
self.change_notify.notify_waiters(); self.change_notify.notify_waiters();
self.broadcast_update(SystemRpc::AdvertiseClusterLayoutTrackers(new_trackers)); self.broadcast_update(SystemRpc::AdvertiseClusterLayoutTrackers(new_trackers));
self.save_cluster_layout().await?; self.save_cluster_layout().await;
} }
Ok(SystemRpc::Ok) Ok(SystemRpc::Ok)
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_table" name = "garage_table"
version = "1.2.0" version = "1.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+2 -2
View File
@@ -367,7 +367,7 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
} }
} }
pub fn gc_todo_len(&self) -> Result<usize, Error> { pub fn gc_todo_approximate_len(&self) -> Result<usize, Error> {
Ok(self.gc_todo.len()?) Ok(self.gc_todo.approximate_len()?)
} }
} }
+1 -1
View File
@@ -313,7 +313,7 @@ impl<F: TableSchema, R: TableReplication> Worker for GcWorker<F, R> {
fn status(&self) -> WorkerStatus { fn status(&self) -> WorkerStatus {
WorkerStatus { WorkerStatus {
queue_length: Some(self.gc.data.gc_todo_len().unwrap_or(0) as u64), queue_length: Some(self.gc.data.gc_todo_approximate_len().unwrap_or(0) as u64),
..Default::default() ..Default::default()
} }
} }
+5 -5
View File
@@ -287,12 +287,12 @@ impl<F: TableSchema, R: TableReplication> MerkleUpdater<F, R> {
MerkleNode::decode_opt(&ent) MerkleNode::decode_opt(&ent)
} }
pub fn merkle_tree_len(&self) -> Result<usize, Error> { pub fn merkle_tree_approximate_len(&self) -> Result<usize, Error> {
Ok(self.data.merkle_tree.len()?) Ok(self.data.merkle_tree.approximate_len()?)
} }
pub fn todo_len(&self) -> Result<usize, Error> { pub fn todo_approximate_len(&self) -> Result<usize, Error> {
Ok(self.data.merkle_todo.len()?) Ok(self.data.merkle_todo.approximate_len()?)
} }
} }
@@ -306,7 +306,7 @@ impl<F: TableSchema, R: TableReplication> Worker for MerkleWorker<F, R> {
fn status(&self) -> WorkerStatus { fn status(&self) -> WorkerStatus {
WorkerStatus { WorkerStatus {
queue_length: Some(self.0.todo_len().unwrap_or(0) as u64), queue_length: Some(self.0.todo_approximate_len().unwrap_or(0) as u64),
..Default::default() ..Default::default()
} }
} }
+4 -4
View File
@@ -34,7 +34,7 @@ impl TableMetrics {
.u64_value_observer( .u64_value_observer(
"table.size", "table.size",
move |observer| { move |observer| {
if let Ok(value) = store.len() { if let Ok(value) = store.approximate_len() {
observer.observe( observer.observe(
value as u64, value as u64,
&[KeyValue::new("table_name", table_name)], &[KeyValue::new("table_name", table_name)],
@@ -48,7 +48,7 @@ impl TableMetrics {
.u64_value_observer( .u64_value_observer(
"table.merkle_tree_size", "table.merkle_tree_size",
move |observer| { move |observer| {
if let Ok(value) = merkle_tree.len() { if let Ok(value) = merkle_tree.approximate_len() {
observer.observe( observer.observe(
value as u64, value as u64,
&[KeyValue::new("table_name", table_name)], &[KeyValue::new("table_name", table_name)],
@@ -62,7 +62,7 @@ impl TableMetrics {
.u64_value_observer( .u64_value_observer(
"table.merkle_updater_todo_queue_length", "table.merkle_updater_todo_queue_length",
move |observer| { move |observer| {
if let Ok(v) = merkle_todo.len() { if let Ok(v) = merkle_todo.approximate_len() {
observer.observe( observer.observe(
v as u64, v as u64,
&[KeyValue::new("table_name", table_name)], &[KeyValue::new("table_name", table_name)],
@@ -76,7 +76,7 @@ impl TableMetrics {
.u64_value_observer( .u64_value_observer(
"table.gc_todo_queue_length", "table.gc_todo_queue_length",
move |observer| { move |observer| {
if let Ok(value) = gc_todo.len() { if let Ok(value) = gc_todo.approximate_len() {
observer.observe( observer.observe(
value as u64, value as u64,
&[KeyValue::new("table_name", table_name)], &[KeyValue::new("table_name", table_name)],
+1 -1
View File
@@ -27,7 +27,7 @@ impl<F: TableSchema, R: TableReplication> Worker for InsertQueueWorker<F, R> {
fn status(&self) -> WorkerStatus { fn status(&self) -> WorkerStatus {
WorkerStatus { WorkerStatus {
queue_length: Some(self.0.data.insert_queue.len().unwrap_or(0) as u64), queue_length: Some(self.0.data.insert_queue.approximate_len().unwrap_or(0) as u64),
..Default::default() ..Default::default()
} }
} }
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_util" name = "garage_util"
version = "1.2.0" version = "1.3.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+11
View File
@@ -75,6 +75,10 @@ pub struct Config {
)] )]
pub block_ram_buffer_max: usize, pub block_ram_buffer_max: usize,
/// Maximum number of concurrent reads of block files on disk
#[serde(default = "default_block_max_concurrent_reads")]
pub block_max_concurrent_reads: usize,
/// Skip the permission check of secret files. Useful when /// Skip the permission check of secret files. Useful when
/// POSIX ACLs (or more complex chmods) are used. /// POSIX ACLs (or more complex chmods) are used.
#[serde(default)] #[serde(default)]
@@ -122,6 +126,10 @@ pub struct Config {
#[serde(deserialize_with = "deserialize_capacity", default)] #[serde(deserialize_with = "deserialize_capacity", default)]
pub lmdb_map_size: usize, pub lmdb_map_size: usize,
/// Fjall block cache size
#[serde(deserialize_with = "deserialize_capacity", default)]
pub fjall_block_cache_size: usize,
// -- APIs // -- APIs
/// Configuration for S3 api /// Configuration for S3 api
pub s3_api: S3ApiConfig, pub s3_api: S3ApiConfig,
@@ -276,6 +284,9 @@ fn default_block_size() -> usize {
fn default_block_ram_buffer_max() -> usize { fn default_block_ram_buffer_max() -> usize {
256 * 1024 * 1024 256 * 1024 * 1024
} }
fn default_block_max_concurrent_reads() -> usize {
16
}
fn default_consistency_mode() -> String { fn default_consistency_mode() -> String {
"consistent".into() "consistent".into()
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_web" name = "garage_web"
version = "1.2.0" version = "1.3.0"
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"] authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"