Compare commits

...

109 Commits

Author SHA1 Message Date
Alex Auvolat 006fb18aea fix compilation warning 2025-09-15 15:52:53 +02:00
Alex Auvolat b43f309ec7 bump version to v2.1.0 2025-09-15 15:50:02 +02:00
Alex df4721387c Merge pull request 'Merge changes from v1.x into v2.x' (#1161) from sync-v2-to-v1 into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1161
2025-09-15 13:24:20 +00:00
Alex Auvolat 9c067c0cbd Merge branch 'main-v1' into sync-v2-to-v1 2025-09-14 21:04:04 +02:00
Alex 0bbf63ee0e Merge pull request 'update rusqlite and snapshot using VACUUM INTO' (#1164) from update-rusqlite into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1164
2025-09-14 18:28:01 +00:00
Alex 879d941d7b Merge pull request 'add garage repair clear-resync-queue (fix #1151)' (#1165) from clear-resync-queue into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1165
2025-09-14 17:50:41 +00:00
Alex Auvolat d726cf0299 add garage repair clear-resync-queue (fix #1151) 2025-09-14 19:34:44 +02:00
Alex 0c7aeab6f8 Merge pull request 'garage_db: fix error handling logic (fix #1138)' (#1163) from fix-1138 into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1163
2025-09-14 17:26:08 +00:00
Alex Auvolat 5687fc0375 update rusqlite and snapshot using VACUUM INTO 2025-09-14 19:22:36 +02:00
Alex 742129f4a3 Merge pull request 'openapi: fix query parameters (fix #1160)' (#1162) from fix-1160 into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1162
2025-09-14 16:46:43 +00:00
Alex 97f1e9ab52 Merge pull request 'Add Plakar documentation (backup tools)' (#1119) from Lapineige/garage:Plakar_support into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1119
2025-09-14 16:08:36 +00:00
Lapineige 60b1d78b56 Add Plakar documentation 2025-09-14 18:07:49 +02:00
Alex Auvolat 4c895a7186 garage_db: fix error handling logic (fix #1138) 2025-09-14 18:03:31 +02:00
Alex Auvolat 7a256b2ebb openapi: fix query parameters (fix #1160) 2025-09-14 17:27:37 +02:00
Alex Auvolat 909359ca4c Merge branch 'main-v1' into sync-v2-to-v1 2025-09-14 17:02:34 +02:00
Alex 3148fa3afe Merge pull request 'add error case for layout not ready, and fail earlier in many places' (#1159) from fix-1107 into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1159
2025-09-14 14:45:00 +00:00
Alex Auvolat 6b06459b99 fullcopy tables: fix read quorum 2025-09-13 21:34:09 +02:00
Alex Auvolat 4c139bcbca add error case for layout not ready, and fail earlier in many places 2025-09-13 20:52:51 +02:00
Alex c3b5cbf212 Merge pull request 'fix panic when cluster_layout cannot be saved (fix #1150)' (#1158) from fix-1150 into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1158
2025-09-13 15:58:52 +00:00
Alex 57a467b5c0 Merge pull request 'Block manager: limit simultaneous block reads from disk' (#1157) from block-max-simultaneous-reads into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1157
2025-09-13 15:53:24 +00:00
Alex Auvolat 6cf6db5c61 fix panic when cluster_layout cannot be saved (fix #1150) 2025-09-13 17:49:25 +02:00
Alex Auvolat d5a57e3e13 block: read_block: don't add not found blocks to resync queue 2025-09-13 17:38:23 +02:00
Alex Auvolat 5cf354acb4 block: maximum number of simultaneous reads 2025-09-13 17:38:06 +02:00
Alex 4758d8881f Merge pull request 'woodpecker: require the nix=enabled label' (#1153) from woodpecker-nix-flag-v2 into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1153
2025-09-04 09:15:17 +00:00
Alex Auvolat 61e19310c8 woodpecker: require the nix=enabled label 2025-09-04 11:11:03 +02:00
Alex 2b007ddea3 Merge pull request 'woodpecker: require the nix=enabled label' (#1152) from woodpecker-nix-flag into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1152
2025-09-04 09:10:10 +00:00
Alex Auvolat c8599a8636 woodpecker: require the nix=enabled label 2025-09-04 11:06:46 +02:00
Alex 17fe11fa81 Merge pull request 'fix(openapi): use query params, not path params for Update & Delete Bucket' (#1139) from milas/garage:milas/fix-openapi-v2-params into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1139
2025-09-04 07:23:45 +00:00
Milas Bowman 16128fca63 chore: re-generate OpenAPI spec
```
cargo run -- admin-api-schema > doc/api/garage-admin-v2.json
```
2025-09-03 21:08:54 -04:00
Alex 29570f3192 Merge pull request 'helm: add commonLabels to all resources and pod templates' (#1135) from Schwitzd/garage:feat/helm-commonLabels into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1135
Reviewed-by: maximilien <me@mricher.fr>
2025-09-02 08:28:35 +00:00
Schwitzd c35c1b5b9b helm: add commonLabels to all resources and pod templates 2025-09-01 15:51:05 +00:00
Alex 0b901bf291 Merge pull request 'garage_db: reduce frequency of sqlite snapshot progress log (fix #1129)' (#1146) from fix-1129 into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1146
2025-08-27 22:26:32 +00:00
Alex 7e203f634e Merge pull request 'fix server panic caused by garage json-api Metrics (fix #1132)' (#1144) from fix-1132 into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1144
2025-08-27 22:24:29 +00:00
Alex 99f7c0fc4b Merge pull request 'fix panic in garage status (fix #1131)' (#1145) from fix-1131 into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1145
2025-08-27 22:15:15 +00:00
Alex Auvolat c8c20d6f47 garage_db: reduce frequency of sqlite snapshot progress log (fix #1129) 2025-08-28 00:07:35 +02:00
Alex Auvolat fb95a8819f fix server panic caused by garage json-api Metrics (fix #1132) 2025-08-27 23:59:41 +02:00
Alex Auvolat 665addc03b fix panic in garage status (fix #1131) 2025-08-27 23:58:48 +02:00
Alex 7949927291 Merge pull request 'take consistency_mode (degraded/dangerous) into account in fullcopy tables' (#1143) from fix-fullcopy-quorums into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1143
2025-08-27 21:54:50 +00:00
Alex Auvolat 2ddb29ca35 take consistency_mode (degraded/dangerous) into account in fullcopy tables 2025-08-27 23:34:06 +02:00
Alex e5db610e4c Merge pull request 'K2V client: allow custom HTTP client' (#731) from k2v/shared_http_client into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/731
Reviewed-by: maximilien <me@mricher.fr>
2025-08-27 21:21:09 +00:00
Alex 65c6f8adea Merge pull request 'garage_db: refactor open function' (#1142) from factor-db-open into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1142
2025-08-27 21:10:59 +00:00
Alex Auvolat 54b9bf02a3 garage_db: refactor open function 2025-08-27 23:03:09 +02:00
Alex 469153233f Merge pull request 'garage_db: rename len to approximate_len as it is used for stats only' (#1141) from db-approximate-len into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1141
2025-08-27 20:44:50 +00:00
Alex Auvolat 90bba5889a garage_db: rename len to approximate_len as it is used for stats only 2025-08-27 21:23:45 +02:00
Alex a64b567d43 Merge pull request 'Add experimental support for Fjall DB engine' (#906) from withings/garage:feat/fjall-db-engine into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/906
2025-08-27 19:09:40 +00:00
Alex Auvolat 6ea86db8cd document fjall db engine, remove flakey metadata_fsync implementation 2025-08-27 20:22:41 +02:00
Alex Auvolat aa69c06f2b fix potential race condition and naming bug in fjall adapter 2025-08-27 20:22:38 +02:00
Alex Auvolat a6c6c44310 nix: build and test fjall feature 2025-08-27 18:54:42 +02:00
Julien Kritter 96d7713915 Add support for an LSM-tree-based backend with Fjall 2025-08-27 18:54:34 +02:00
Alex 30d8ec5368 Merge pull request 'fix(script/telemetry): remove broken panel override in grafana prometheus dashboard' (#1075) from RockWolf/garage:rockwolf-patch-1 into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1075
2025-08-27 16:21:39 +00:00
Alex 47772eb525 Merge pull request 'fix: return consistent cors headers on api error' (#1115) from Xstoudi/garage:fix/consistent-cors into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1115
2025-08-27 16:19:36 +00:00
Alex d64498c3d3 Merge pull request 'log access keys' (#1122) from 1686a/log-access-key into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1122
2025-08-27 16:18:16 +00:00
Alex c1ed770e64 Merge pull request 'log access keys Garage v2' (#1124) from 1686a/log-access-key-v2 into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1124
2025-08-27 16:17:39 +00:00
Alex 7e80e86934 Merge pull request 'Add dev scripts dependencies to devShellFull' (#1134) from atnnn/garage:dev-script-deps into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1134
2025-08-27 16:15:30 +00:00
Milas Bowman 4deb57815a fix(api): use query params, not path params for Update & Delete Bucket 2025-08-23 13:02:03 -04:00
Etienne Laurin df343dd808 Add dev scripts dependencies to devShellFull 2025-08-12 19:59:30 -07:00
Alex 17c73bafa2 Merge pull request 'fix: method name duplicated after client sdk generation' (#1111) from MagicRR/garage:fix-openapi-duplicate-method into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1111
2025-08-07 14:32:12 +00:00
MagicRR d8058e7475 fix: method name duplicated after client sdk generation 2025-08-07 10:14:56 +00:00
trinity-1686a 385fbc606d log user-defined admin api tokens 2025-08-03 15:50:36 +02:00
trinity-1686a 6f9d6919a9 log access keys 2025-08-03 15:31:51 +02:00
trinity-1686a b340599e68 log access keys 2025-08-03 15:30:56 +02:00
Alex 91fde4105d Merge pull request 'hide keys without any permissions from admin api' (#1123) from 1686a/hide-empty-keys into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1123
2025-08-03 11:18:51 +00:00
trinity-1686a d975960be3 hide keys without any permissions from admin api 2025-08-02 17:28:14 +02:00
Alex 5448012b27 Merge pull request 'Pixelfed_support' (#1118) from Lapineige/garage:Pixelfed_support into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1118
2025-08-02 15:03:57 +00:00
Alex ce34d11a65 Merge pull request 'don't die on SIGHUP' (#1121) from 1686a/handle-sighup into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1121
2025-08-02 14:53:58 +00:00
Alex 8cb7623ebd Merge pull request 'handle ECONNABORTED' (#1120) from 1686a/handle-econnaborted into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1120
2025-08-02 14:53:45 +00:00
trinity-1686a 5469c95877 handle ECONNABORTED 2025-08-02 13:14:01 +02:00
trinity-1686a f930c6f643 don't die on SIGHUP 2025-08-02 13:09:33 +02:00
Alex 6508acbe71 Merge pull request 'ignore checksums with empty strings' (#1116) from CobaltCause/garage:push-qsqlmltnouyv into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1116
2025-08-02 08:29:10 +00:00
Alex afcb22bf16 Merge pull request 'Fix typo in peertube buckets names' (#1117) from Lapineige/garage:main into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1117
2025-08-02 08:27:01 +00:00
Lapineige cc29a40d51 Actualiser doc/book/connect/apps/index.md 2025-08-01 21:35:15 +00:00
Lapineige 0f3f180c3e Merge branch 'main-v1' into main 2025-08-01 21:33:58 +00:00
Lapineige 70cf6004ae Fix typo in peertube buckets names 2025-08-01 21:32:59 +00:00
Xavier Stouder 985ad68ade fix: run cargo format 2025-07-31 22:52:27 +02:00
Alex c7571ff89b Merge pull request 'Fix some unsoundness in lmdb adapter unsafe' (#1099) from krtab/garage:fix_some_ub into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1099
2025-07-31 19:38:23 +00:00
Charles Hall b7a853b01f ignore checksums with empty/whitespace-only bodies
aws-sdk-cpp was observed to send request bodies like this via Lix:

```xml
<?xml version="1.0"?>
<CompleteMultipartUpload xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
    <Part>
        <ETag>"8a5031bda169553d6a232e6c11068774"</ETag>
        <ChecksumCRC32></ChecksumCRC32>
        <PartNumber>1</PartNumber>
    </Part>
    <Part>
        <ETag>"da977cc58b75bd17749c1ff460ba301a"</ETag>
        <ChecksumCRC32></ChecksumCRC32>
        <PartNumber>2</PartNumber>
    </Part>
</CompleteMultipartUpload>
```
2025-07-31 11:15:33 -07:00
Charles Hall 66faef9fb6 factor out repetitive else-if chain into macro 2025-07-30 18:44:51 -07:00
Charles Hall 13f67b6cd8 log incorrect multipart completion body 2025-07-30 17:08:43 -07:00
Xavier Stouder 0dabf9b22f fix: return consistent cors headers on api error 2025-07-30 22:20:48 +02:00
Arthur Carcano 1b42919bf7 Fix some unsoundness in lmdb adapter unsafe 2025-07-25 23:33:51 +02:00
Alex e226fb413f Merge pull request 'fix: return 204 on successful AbortMultipartUpload' (#1095) from nikeee/garage:fix-abort-multipart-upload into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1095
2025-07-11 17:12:32 +00:00
Niklas Mollenhauer 708a84f1d6 fix: return 204 on successful AbortMultipartUpload
Docs state that 204 should be returned on success:
https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html
```http
HTTP/1.1 204
x-amz-request-charged: RequestCharged
```
2025-07-11 16:19:51 +02:00
Alex 0465475599 Merge pull request 'documentation fixes' (#1101) from doc-fixes into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1101
2025-07-11 11:32:01 +00:00
Alex Auvolat 0a45317b3b doc: fix link to k2v spec 2025-07-11 13:30:03 +02:00
Alex Auvolat bb3b832024 doc: fix scrub tranquility command 2025-07-11 13:27:17 +02:00
Alex f8be15c37d Merge pull request 'api: add instrospect endpoint' (#1092) from Xstoudi/garage:feature/introspect-endpoint into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1092
2025-07-11 11:20:12 +00:00
Alex 1e05fc1d53 Merge pull request 'fix: GetBucketCORS returns 404 if there are no policies' (#1096) from nikeee/garage:fix-get-bucket-cors into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1096
2025-07-10 09:56:54 +00:00
Alex e5eff872f5 Merge pull request 'doc: Update use case' (#1098) from jonah/garage:main-v2 into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1098
2025-07-10 09:54:47 +00:00
Jonah Aragon 605ee4cdb1 doc: Update use case 2025-07-08 17:46:02 -05:00
Niklas Mollenhauer 71aef8770e fix: GetBucketCORS returns 404 if there are no policies
Similar issue @ ceph:
https://github.com/ceph/ceph/pull/27122/files
https://tracker.ceph.com/issues/38886

Implementation @ minio:
https://github.com/minio/minio/blob/de234b888c26cc191f3062a625af82640c966795/cmd/dummy-handlers.go#L196
2025-07-08 17:33:51 +02:00
Xavier Stouder b4f6ab963c api: correct according to review 2025-07-04 21:36:34 +02:00
Xavier Stouder 9a31b9c077 api: change endpoint name and allow it to be called even if not in current token scope 2025-07-03 21:59:23 +02:00
Xavier Stouder 58a96dc687 api: correct openapi def 2025-07-03 21:59:22 +02:00
Xavier Stouder 7bbb3ff9cf api: add instrospect endpoint
Fixes #1091
2025-07-01 23:14:09 +02:00
Alex f04af18193 Merge pull request 'reverse-proxy.md: use $http_host instead of $host for alternative port' (#1085) from dongdigua/garage:main-v2 into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1085
2025-06-23 07:53:44 +00:00
dongdigua 67e0fcc6ea reverse-proxy.md: use $http_host instead of $host for alternative port
if the nginx port is not 443, the host header won't match, causing a Forbidden: Invalid signature
2025-06-23 01:26:03 +00:00
Alex 78f03aec78 Merge pull request 'fix(openapi): set parameters in query instead of path for get params' (#1082) from Xstoudi/garage:fix/openapi-get-parameter into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1082
2025-06-20 10:25:51 +00:00
Xavier Stouder 56a23d936e chore: regenerate openapi spec v2 2025-06-19 23:55:00 +02:00
Xavier Stouder 9b6e45ca1f fix(openapi): set parameters in query instead of path for get params
get request params are wrongfully outputted as "in path" instead of "in query", it fix this.

Fixes #1081
2025-06-19 22:31:44 +02:00
RockWolf 27666ed265 fix(script/telemetry): remove broken panel override in grafana prometheus dashboard
Signed-off-by: RockWolf <rockwolf@noreply.localhost>
2025-06-16 20:36:15 +00:00
Alex e8e722cc66 Merge pull request 'update documentation pages on admin API' (#1074) from doc-admin-api into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1074
2025-06-15 13:44:05 +00:00
Alex Auvolat 80f818eb6c update documentation pages on admin API 2025-06-15 15:40:38 +02:00
Alex f899e023a0 Merge pull request 'documentation fixes' (#1072) from doc-fixes into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1072
2025-06-15 09:28:13 +00:00
Alex Auvolat 7556c536ae documentation fixes 2025-06-15 11:27:21 +02:00
Alex 2a20319fa9 Merge pull request 'openapi spec: add missing datatypes for path parameters' (#1071) from fix-openapi-params into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1071
2025-06-14 15:37:26 +00:00
Alex Auvolat 42baa29e50 openapi spec: add missing datatypes for path parameters 2025-06-14 17:25:37 +02:00
Alex f461348790 Merge pull request 'migration guide for Garage v2' (#1070) from upgrade-guide-v2 into main-v2
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1070
2025-06-14 15:19:39 +00:00
Alex Auvolat 4a8f7e15ce migration guide for Garage v2 2025-06-14 17:19:04 +02:00
Quentin Dufour 8b35a946d9 Allow external HTTP client 2024-02-23 17:09:47 +01:00
106 changed files with 2223 additions and 834 deletions
+8
View File
@@ -1,3 +1,6 @@
labels:
nix: "enabled"
when:
event:
- push
@@ -28,6 +31,11 @@ steps:
commands:
- nix-build -j4 --attr flakePackages.tests-sqlite
- name: unit + func tests (fjall)
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --attr flakePackages.tests-fjall
- name: integration tests
image: nixpkgs/nix:nixos-22.05
commands:
+3
View File
@@ -1,3 +1,6 @@
labels:
nix: "enabled"
when:
event:
- deployment
+3
View File
@@ -1,3 +1,6 @@
labels:
nix: "enabled"
when:
event:
- deployment
Generated
+219 -23
View File
@@ -687,6 +687,12 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d2c12f985c78475a6b8d629afd0c360260ef34cfef52efccdcfd31972f81c2e"
[[package]]
name = "byteview"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
[[package]]
name = "cc"
version = "1.2.16"
@@ -798,6 +804,12 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
[[package]]
name = "compare"
version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7"
[[package]]
name = "core-foundation"
version = "0.9.4"
@@ -888,6 +900,15 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.12"
@@ -897,6 +918,16 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-skiplist"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
@@ -968,6 +999,20 @@ dependencies = [
"num_cpus",
]
[[package]]
name = "dashmap"
version = "6.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
dependencies = [
"cfg-if",
"crossbeam-utils",
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core 0.9.10",
]
[[package]]
name = "deranged"
version = "0.4.0"
@@ -1010,6 +1055,12 @@ dependencies = [
"syn 2.0.100",
]
[[package]]
name = "double-ended-peekable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
[[package]]
name = "dyn-clone"
version = "1.0.19"
@@ -1031,6 +1082,18 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "enum_dispatch"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd"
dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.100",
]
[[package]]
name = "env_logger"
version = "0.10.2"
@@ -1098,6 +1161,23 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "fjall"
version = "2.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc"
dependencies = [
"byteorder",
"byteview",
"dashmap 6.1.0",
"log",
"lsm-tree",
"path-absolutize",
"std-semaphore",
"tempfile",
"xxhash-rust",
]
[[package]]
name = "fnv"
version = "1.0.7"
@@ -1214,7 +1294,7 @@ dependencies = [
[[package]]
name = "garage"
version = "2.0.0"
version = "2.1.0"
dependencies = [
"assert-json-diff",
"async-trait",
@@ -1270,7 +1350,7 @@ dependencies = [
[[package]]
name = "garage_api_admin"
version = "2.0.0"
version = "2.1.0"
dependencies = [
"argon2",
"async-trait",
@@ -1302,7 +1382,7 @@ dependencies = [
[[package]]
name = "garage_api_common"
version = "2.0.0"
version = "2.1.0"
dependencies = [
"base64 0.21.7",
"bytes",
@@ -1335,7 +1415,7 @@ dependencies = [
[[package]]
name = "garage_api_k2v"
version = "2.0.0"
version = "2.1.0"
dependencies = [
"base64 0.21.7",
"err-derive",
@@ -1358,7 +1438,7 @@ dependencies = [
[[package]]
name = "garage_api_s3"
version = "2.0.0"
version = "2.1.0"
dependencies = [
"aes-gcm",
"async-compression",
@@ -1403,7 +1483,7 @@ dependencies = [
[[package]]
name = "garage_block"
version = "2.0.0"
version = "2.1.0"
dependencies = [
"arc-swap",
"async-compression",
@@ -1427,11 +1507,13 @@ dependencies = [
[[package]]
name = "garage_db"
version = "2.0.0"
version = "2.1.0"
dependencies = [
"err-derive",
"fjall",
"heed",
"mktemp",
"parking_lot 0.12.3",
"r2d2",
"r2d2_sqlite",
"rusqlite",
@@ -1440,7 +1522,7 @@ dependencies = [
[[package]]
name = "garage_model"
version = "2.0.0"
version = "2.1.0"
dependencies = [
"argon2",
"async-trait",
@@ -1468,7 +1550,7 @@ dependencies = [
[[package]]
name = "garage_net"
version = "2.0.0"
version = "2.1.0"
dependencies = [
"arc-swap",
"bytes",
@@ -1493,7 +1575,7 @@ dependencies = [
[[package]]
name = "garage_rpc"
version = "2.0.0"
version = "2.1.0"
dependencies = [
"arc-swap",
"async-trait",
@@ -1525,7 +1607,7 @@ dependencies = [
[[package]]
name = "garage_table"
version = "2.0.0"
version = "2.1.0"
dependencies = [
"arc-swap",
"async-trait",
@@ -1546,7 +1628,7 @@ dependencies = [
[[package]]
name = "garage_util"
version = "2.0.0"
version = "2.1.0"
dependencies = [
"arc-swap",
"async-trait",
@@ -1578,7 +1660,7 @@ dependencies = [
[[package]]
name = "garage_web"
version = "2.0.0"
version = "2.1.0"
dependencies = [
"err-derive",
"garage_api_common",
@@ -1674,6 +1756,12 @@ dependencies = [
"syn 2.0.100",
]
[[package]]
name = "guardian"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f"
[[package]]
name = "h2"
version = "0.3.26"
@@ -1741,11 +1829,11 @@ dependencies = [
[[package]]
name = "hashlink"
version = "0.9.1"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
dependencies = [
"hashbrown 0.14.5",
"hashbrown 0.15.2",
]
[[package]]
@@ -2249,6 +2337,15 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "interval-heap"
version = "0.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6"
dependencies = [
"compare",
]
[[package]]
name = "ipnet"
version = "2.11.0"
@@ -2542,9 +2639,9 @@ dependencies = [
[[package]]
name = "libsqlite3-sys"
version = "0.28.0"
version = "0.35.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"
checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f"
dependencies = [
"cc",
"pkg-config",
@@ -2605,6 +2702,36 @@ dependencies = [
"hashbrown 0.15.2",
]
[[package]]
name = "lsm-tree"
version = "2.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab73c02eadb3dc12c0024e5b61d6284e6d59064e67e74fbad77856caa56f62c7"
dependencies = [
"byteorder",
"crossbeam-skiplist",
"double-ended-peekable",
"enum_dispatch",
"guardian",
"interval-heap",
"log",
"lz4_flex",
"path-absolutize",
"quick_cache",
"rustc-hash",
"self_cell",
"tempfile",
"value-log",
"varint-rs",
"xxhash-rust",
]
[[package]]
name = "lz4_flex"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
[[package]]
name = "matchers"
version = "0.1.0"
@@ -2859,7 +2986,7 @@ checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8"
dependencies = [
"async-trait",
"crossbeam-channel",
"dashmap",
"dashmap 4.0.2",
"fnv",
"futures-channel",
"futures-executor",
@@ -3020,6 +3147,24 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "path-absolutize"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5"
dependencies = [
"path-dedot",
]
[[package]]
name = "path-dedot"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397"
dependencies = [
"once_cell",
]
[[package]]
name = "pem"
version = "3.0.5"
@@ -3315,6 +3460,16 @@ dependencies = [
"serde",
]
[[package]]
name = "quick_cache"
version = "0.6.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786"
dependencies = [
"equivalent",
"hashbrown 0.15.2",
]
[[package]]
name = "quote"
version = "1.0.40"
@@ -3343,9 +3498,9 @@ dependencies = [
[[package]]
name = "r2d2_sqlite"
version = "0.24.0"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a982edf65c129796dba72f8775b292ef482b40d035e827a9825b3bc07ccc5f2"
checksum = "63417e83dc891797eea3ad379f52a5986da4bca0d6ef28baf4d14034dd111b0c"
dependencies = [
"r2d2",
"rusqlite",
@@ -3563,9 +3718,9 @@ checksum = "3cd14fd5e3b777a7422cca79358c57a8f6e3a703d9ac187448d0daf220c2407f"
[[package]]
name = "rusqlite"
version = "0.31.0"
version = "0.37.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae"
checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f"
dependencies = [
"bitflags 2.9.0",
"fallible-iterator",
@@ -3581,6 +3736,12 @@ version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
[[package]]
name = "rustc-hash"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
[[package]]
name = "rustc_version"
version = "0.4.1"
@@ -3823,6 +3984,12 @@ dependencies = [
"libc",
]
[[package]]
name = "self_cell"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749"
[[package]]
name = "semver"
version = "1.0.26"
@@ -4036,6 +4203,12 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "std-semaphore"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e"
[[package]]
name = "strsim"
version = "0.11.1"
@@ -4736,6 +4909,29 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
[[package]]
name = "value-log"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c"
dependencies = [
"byteorder",
"byteview",
"interval-heap",
"log",
"path-absolutize",
"rustc-hash",
"tempfile",
"varint-rs",
"xxhash-rust",
]
[[package]]
name = "varint-rs"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]]
name = "vcpkg"
version = "0.2.15"
+16 -14
View File
@@ -24,18 +24,18 @@ default-members = ["src/garage"]
# Internal Garage crates
format_table = { version = "0.1.1", path = "src/format-table" }
garage_api_common = { version = "2.0.0", path = "src/api/common" }
garage_api_admin = { version = "2.0.0", path = "src/api/admin" }
garage_api_s3 = { version = "2.0.0", path = "src/api/s3" }
garage_api_k2v = { version = "2.0.0", path = "src/api/k2v" }
garage_block = { version = "2.0.0", path = "src/block" }
garage_db = { version = "2.0.0", path = "src/db", default-features = false }
garage_model = { version = "2.0.0", path = "src/model", default-features = false }
garage_net = { version = "2.0.0", path = "src/net" }
garage_rpc = { version = "2.0.0", path = "src/rpc" }
garage_table = { version = "2.0.0", path = "src/table" }
garage_util = { version = "2.0.0", path = "src/util" }
garage_web = { version = "2.0.0", path = "src/web" }
garage_api_common = { version = "2.1.0", path = "src/api/common" }
garage_api_admin = { version = "2.1.0", path = "src/api/admin" }
garage_api_s3 = { version = "2.1.0", path = "src/api/s3" }
garage_api_k2v = { version = "2.1.0", path = "src/api/k2v" }
garage_block = { version = "2.1.0", path = "src/block" }
garage_db = { version = "2.1.0", path = "src/db", default-features = false }
garage_model = { version = "2.1.0", path = "src/model", default-features = false }
garage_net = { version = "2.1.0", path = "src/net" }
garage_rpc = { version = "2.1.0", path = "src/rpc" }
garage_table = { version = "2.1.0", path = "src/table" }
garage_util = { version = "2.1.0", path = "src/util" }
garage_web = { version = "2.1.0", path = "src/web" }
k2v-client = { version = "0.0.4", path = "src/k2v-client" }
# External crates from crates.io
@@ -64,6 +64,7 @@ md-5 = "0.10"
mktemp = "0.5"
nix = { version = "0.29", default-features = false, features = ["fs"] }
nom = "7.1"
parking_lot = "0.12"
parse_duration = "2.1"
paste = "1.0"
pin-project = "1.0.12"
@@ -87,9 +88,10 @@ tracing-journald = "0.3.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
heed = { version = "0.11", default-features = false, features = ["lmdb"] }
rusqlite = "0.31.0"
rusqlite = "0.37"
r2d2 = "0.8"
r2d2_sqlite = "0.24"
r2d2_sqlite = "0.31"
fjall = "2.4"
async-compression = { version = "0.4", features = ["tokio", "zstd"] }
zstd = { version = "0.13", default-features = false }
+156 -87
View File
@@ -12,7 +12,7 @@
"name": "AGPL-3.0",
"identifier": "AGPL-3.0"
},
"version": "v2.0.0"
"version": "v2.1.0"
},
"servers": [
{
@@ -31,9 +31,12 @@
"parameters": [
{
"name": "domain",
"in": "path",
"in": "query",
"description": "The domain name to check for",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -405,9 +408,12 @@
"parameters": [
{
"name": "node",
"in": "path",
"in": "query",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -437,9 +443,12 @@
"parameters": [
{
"name": "id",
"in": "path",
"in": "query",
"description": "Admin API token ID",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -462,9 +471,12 @@
"parameters": [
{
"name": "id",
"in": "path",
"in": "query",
"description": "ID of the bucket to delete",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -493,9 +505,12 @@
"parameters": [
{
"name": "id",
"in": "path",
"in": "query",
"description": "Access key ID",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -552,26 +567,20 @@
"parameters": [
{
"name": "id",
"in": "path",
"in": "query",
"description": "Admin API token ID",
"required": true,
"required": false,
"schema": {
"type": [
"string",
"null"
]
"type": "string"
}
},
{
"name": "search",
"in": "path",
"in": "query",
"description": "Partial token ID or name to search for",
"required": true,
"required": false,
"schema": {
"type": [
"string",
"null"
]
"type": "string"
}
}
],
@@ -602,9 +611,12 @@
"parameters": [
{
"name": "node",
"in": "path",
"in": "query",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -644,38 +656,29 @@
"parameters": [
{
"name": "id",
"in": "path",
"in": "query",
"description": "Exact bucket ID to look up",
"required": true,
"required": false,
"schema": {
"type": [
"string",
"null"
]
"type": "string"
}
},
{
"name": "globalAlias",
"in": "path",
"in": "query",
"description": "Global alias of bucket to look up",
"required": true,
"required": false,
"schema": {
"type": [
"string",
"null"
]
"type": "string"
}
},
{
"name": "search",
"in": "path",
"in": "query",
"description": "Partial ID or alias to search for",
"required": true,
"required": false,
"schema": {
"type": [
"string",
"null"
]
"type": "string"
}
}
],
@@ -813,6 +816,30 @@
}
}
},
"/v2/GetCurrentAdminTokenInfo": {
"get": {
"tags": [
"Admin API token"
],
"description": "\nReturn information about the calling admin API token.\n ",
"operationId": "GetCurrentAdminTokenInfo",
"responses": {
"200": {
"description": "Information about the admin token",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/GetCurrentAdminTokenInfoResponse"
}
}
}
},
"500": {
"description": "Internal server error"
}
}
}
},
"/v2/GetKeyInfo": {
"get": {
"tags": [
@@ -823,33 +850,27 @@
"parameters": [
{
"name": "id",
"in": "path",
"in": "query",
"description": "Access key ID",
"required": true,
"required": false,
"schema": {
"type": [
"string",
"null"
]
"type": "string"
}
},
{
"name": "search",
"in": "path",
"in": "query",
"description": "Partial key ID or name to search for",
"required": true,
"required": false,
"schema": {
"type": [
"string",
"null"
]
"type": "string"
}
},
{
"name": "showSecretKey",
"in": "path",
"in": "query",
"description": "Whether to return the secret access key",
"required": true,
"required": false,
"schema": {
"type": "boolean"
}
@@ -882,9 +903,12 @@
"parameters": [
{
"name": "node",
"in": "path",
"in": "query",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -914,9 +938,12 @@
"parameters": [
{
"name": "node",
"in": "path",
"in": "query",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -946,9 +973,12 @@
"parameters": [
{
"name": "node",
"in": "path",
"in": "query",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -988,9 +1018,12 @@
"parameters": [
{
"name": "node",
"in": "path",
"in": "query",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -1064,7 +1097,7 @@
"parameters": [
{
"name": "bucketId",
"in": "path",
"in": "query",
"required": true,
"schema": {
"type": "string"
@@ -1072,7 +1105,7 @@
},
{
"name": "key",
"in": "path",
"in": "query",
"required": true,
"schema": {
"type": "string"
@@ -1109,9 +1142,12 @@
"parameters": [
{
"name": "node",
"in": "path",
"in": "query",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -1175,9 +1211,12 @@
"parameters": [
{
"name": "node",
"in": "path",
"in": "query",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -1255,9 +1294,12 @@
"parameters": [
{
"name": "node",
"in": "path",
"in": "query",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -1321,9 +1363,12 @@
"parameters": [
{
"name": "node",
"in": "path",
"in": "query",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -1397,9 +1442,12 @@
"parameters": [
{
"name": "node",
"in": "path",
"in": "query",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -1463,9 +1511,12 @@
"parameters": [
{
"name": "node",
"in": "path",
"in": "query",
"description": "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -1505,9 +1556,12 @@
"parameters": [
{
"name": "id",
"in": "path",
"in": "query",
"description": "Admin API token ID",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -1547,9 +1601,12 @@
"parameters": [
{
"name": "id",
"in": "path",
"in": "query",
"description": "ID of the bucket to update",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -1627,9 +1684,12 @@
"parameters": [
{
"name": "id",
"in": "path",
"in": "query",
"description": "Access key ID",
"required": true
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
@@ -2383,7 +2443,7 @@
"knownNodes",
"connectedNodes",
"storageNodes",
"storageNodesOk",
"storageNodesUp",
"partitions",
"partitionsQuorum",
"partitionsAllOk"
@@ -2423,7 +2483,7 @@
"description": "the number of storage nodes currently registered in the cluster layout",
"minimum": 0
},
"storageNodesOk": {
"storageNodesUp": {
"type": "integer",
"description": "the number of storage nodes to which a connection is currently open",
"minimum": 0
@@ -2558,6 +2618,9 @@
}
}
},
"GetCurrentAdminTokenInfoResponse": {
"$ref": "#/components/schemas/GetAdminTokenInfoResponse"
},
"GetKeyInfoResponse": {
"type": "object",
"required": [
@@ -3991,6 +4054,12 @@
"enum": [
"aliases"
]
},
{
"type": "string",
"enum": [
"clearResyncQueue"
]
}
]
},
+4 -4
View File
@@ -12,7 +12,7 @@ In this section, we cover the following web applications:
| [Mastodon](#mastodon) | ✅ | Natively supported |
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
| [ejabberd](#ejabberd) | ✅ | `mod_s3_upload` |
| [Pixelfed](#pixelfed) | | Not yet tested |
| [Pixelfed](#pixelfed) | | Natively supported |
| [Pleroma](#pleroma) | ❓ | Not yet tested |
| [Lemmy](#lemmy) | ✅ | Supported with pict-rs |
| [Funkwhale](#funkwhale) | ❓ | Not yet tested |
@@ -191,10 +191,10 @@ garage key create peertube-key
Keep the Key ID and the Secret key in a pad, they will be needed later.
We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists).
```bash
garage bucket create peertube-videos
garage bucket create peertube-playlist
garage bucket create peertube-playlists
```
Now we allow our key to read and write on these buckets:
@@ -253,7 +253,7 @@ object_storage:
proxify_private_files: false
streaming_playlists:
bucket_name: 'peertube-playlist'
bucket_name: 'peertube-playlists'
# Keep it empty for our example
prefix: ''
+46
View File
@@ -161,3 +161,49 @@ kopia repository validate-provider
You can then run all the standard kopia commands: `kopia snapshot create`, `kopia mount`...
Everything should work out-of-the-box.
## Plakar
Create your key and bucket on Garage server:
```bash
garage key create my-plakar-key
garage bucket create plakar-backups
garage bucket allow plakar-backups --read --write --key my-plakar-key
```
On Plakar server, add your Garage as a storage location:
```bash
plakar store add garageS3 s3://my-garage.tld/plakar-backups \
region=garage # Or as you've specified in garage.toml \
access_key=<Key ID from "garage key info my-plakar-key"> \
secret_access_key=<Secret key from "garage key info my-plakar-key">
```
Then create the repository.
```bash
plakar at @garageS3 create -plaintext # Unencrypted
# or
plakar at @garageS3 create #encrypted
```
If you encrypt your backups (Plakar default), you will need to define a strong passphrase. Do not forget to save your password safely. It will be needed to decrypt your backups.
After the repository has been created, check that everything works as expected (that might give an empty result as no file has been added yet, but no error message):
```bash
plakar at @garageS3 check
```
Now that everything is configure, you can use Garage as your backups storage. For instance sync it with a local backup storage:
```bash
$ plakar at ~/backups sync to @garageS3
```
Or list the S3 storage content:
```bash
$ plakar at @garageS3 ls
```
More information in Plakar documentation: https://www.plakar.io/docs/main/quickstart/
+1 -1
View File
@@ -8,7 +8,7 @@ have published Ansible roles. We list them and compare them below.
## Comparison of Ansible roles
| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster ansible-role-garage](#eddster-ansible-role-garage) |
| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster2309 ansible-role-garage](#eddster2309-ansible-role-garage) |
|------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------|
| **Runtime** | Systemd | Docker | Systemd |
| **Target OS** | Any Linux | Any Linux | Any Linux |
+5 -5
View File
@@ -96,14 +96,14 @@ to store 2 TB of data in total.
## Get a Docker image
Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
We encourage you to use a fixed tag (eg. `v2.0.0`) and not the `latest` tag.
For this example, we will use the latest published version at the time of the writing which is `v2.0.0` but it's up to you
We encourage you to use a fixed tag (eg. `v2.1.0`) and not the `latest` tag.
For this example, we will use the latest published version at the time of the writing which is `v2.1.0` but it's up to you
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
For example:
```
sudo docker pull dxflrs/garage:v2.0.0
sudo docker pull dxflrs/garage:v2.1.0
```
## Deploying and configuring Garage
@@ -171,7 +171,7 @@ docker run \
-v /etc/garage.toml:/etc/garage.toml \
-v /var/lib/garage/meta:/var/lib/garage/meta \
-v /var/lib/garage/data:/var/lib/garage/data \
dxflrs/garage:v2.0.0
dxflrs/garage:v2.1.0
```
With this command line, Garage should be started automatically at each boot.
@@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
version: "3"
services:
garage:
image: dxflrs/garage:v2.0.0
image: dxflrs/garage:v2.1.0
network_mode: "host"
restart: unless-stopped
volumes:
+1 -1
View File
@@ -97,7 +97,7 @@ server {
location / {
proxy_pass http://s3_backend;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
proxy_set_header Host $http_host;
# Disable buffering to a temporary file.
proxy_max_temp_file_size 0;
}
+4 -2
View File
@@ -59,11 +59,13 @@ Garage themselves for the following tasks:
- Hosting of their homepage, [privacyguides.org](https://www.privacyguides.org/), and various other static sites
- As a Mastodon object storage backend for [mstdn.party](https://mstdn.party/) and [mstdn.plus](https://mstdn.plus/)
- As a PowerDNS authoritative zone backend through [Lightning Stream](https://doc.powerdns.com/lightningstream/latest/index.html) and [LMDB](https://doc.powerdns.com/authoritative/backends/lmdb.html)
- As a Mastodon media storage backend for [mstdn.party](https://mstdn.party/) and [mstdn.plus](https://mstdn.plus/)
- As a PeerTube storage backend for [neat.tube](https://neat.tube/)
- As a [Matrix media backend](https://github.com/matrix-org/synapse-s3-storage-provider)
Triplebit's Garage cluster is a multi-site cluster currently composed of
10 nodes in 3 physical locations.
15 storage nodes in 3 physical locations.
+1 -1
View File
@@ -42,7 +42,7 @@ You may pause an ongoing scrub using `garage repair scrub pause`, but note that
the scrub will resume automatically 24 hours later as Garage will not let your
cluster run without a regular scrub. If the scrub procedure is too intensive
for your servers and is slowing down your workload, the recommended solution
is to increase the "scrub tranquility" using `garage repair scrub set-tranquility`.
is to increase the "scrub tranquility" using `garage worker set scrub-tranquility`.
A higher tranquility value will make Garage take longer pauses between two block
verifications. Of course, scrubbing the entire data store will also take longer.
+1 -1
View File
@@ -132,7 +132,7 @@ docker run \
-v /path/to/garage.toml:/etc/garage.toml \
-v /path/to/garage/meta:/var/lib/garage/meta \
-v /path/to/garage/data:/var/lib/garage/data \
dxflrs/garage:v2.0.0
dxflrs/garage:v2.1.0
```
Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903`
+150 -40
View File
@@ -6,41 +6,167 @@ weight = 40
The Garage administration API is accessible through a dedicated server whose
listen address is specified in the `[admin]` section of the configuration
file (see [configuration file
reference](@/documentation/reference-manual/configuration.md))
reference](@/documentation/reference-manual/configuration.md)).
**WARNING.** At this point, there is no commitment to the stability of the APIs described in this document.
We will bump the version numbers prefixed to each API endpoint each time the syntax
or semantics change, meaning that code that relies on these endpoint will break
when changes are introduced.
Versions:
- Before Garage 0.7.2 - no admin API
- Garage 0.7.2 - admin APIv0
- Garage 0.9.0 - admin APIv1, deprecate admin APIv0
The current version of the admin API is v2. No breaking changes to the Garage
administration API will be published outside of a major release.
History of previous versions:
- Before Garage v0.7.2 - no admin API
- Garage v0.7.2 - admin API v0
- Garage v0.9.0 - admin API v1, deprecate admin API v0
- Garage v2.0.0 - admin API v2, deprecate admin API v1
## Access control
The admin API uses two different tokens for access control, that are specified in the config file's `[admin]` section:
### Using an API token
- `metrics_token`: the token for accessing the Metrics endpoint (if this token
is not set in the config file, the Metrics endpoint can be accessed without
access control);
- `admin_token`: the token for accessing all of the other administration
endpoints (if this token is not set in the config file, access to these
endpoints is disabled entirely).
These tokens are used as simple HTTP bearer tokens. In other words, to
authenticate access to an admin API endpoint, add the following HTTP header
to your request:
Administration API tokens tokens are used as simple HTTP bearer tokens. In
other words, to authenticate access to an admin API endpoint, add the following
HTTP header to your request:
```
Authorization: Bearer <token>
```
## Administration API endpoints
### User-defined API tokens
Cluster administrators may dynamically define administration tokens using the CLI commands under `garage admin-token`.
Such tokens may be limited in scope, meaning that they may enable access to only a subset of API calls.
They may also have an expiration date to limit their use in time.
Here is an example to create an administration token that is valid for 30 days
and gives access to only a subset of API calls, allowing it to create buckets
and access keys and give keys permissions on buckets:
```bash
$ garage admin-token create --expires-in 30d \
--scope ListBuckets,GetBucketInfo,ListKeys,GetKeyInfo,CreateBucket,CreateKey,AllowBucketKey,DenyBucketKey \
my-token
This is your secret bearer token, it will not be shown again by Garage:
8ed1830b10a276ff57061950.kOSIpxWK9zSGbTO9Xadpv3YndSFWma0_snXcYHaORXk
==== ADMINISTRATION TOKEN INFORMATION ====
Token ID: 8ed1830b10a276ff57061950
Token name: my-token
Created: 2025-06-15 15:12:44.160 +02:00
Validity: valid
Expiration: 2025-07-15 15:12:44.117 +02:00
Scope: ListBuckets
GetBucketInfo
ListKeys
GetKeyInfo
CreateBucket
CreateKey
AllowBucketKey
DenyBucketKey
```
When running this command, your token will be shown only once and **will never
be shown again by Garage**, so make sure to save it directly. The token is
hashed internally, and is identified by its prefix (32 hex digits followed by a
dot) which is saved in clear.
When running `garage admin-token list`, you might see something like this:
```
ID Created Name Expiration Scope
- - metrics_token (from daemon configuration) never Metrics
8ed1830b10a276ff57061950 2025-06-15 my-token 2025-07-15 15:12:44.117 +02:00 ListBuckets, ... (8)
```
### Master API tokens
The admin API can also use two different master tokens for access control,
specified in the config file's `[admin]` section:
- `metrics_token`: the token for accessing the Metrics endpoint. If this token
is not set in the config file, the Metrics endpoint can be accessed without
access control.
- `admin_token`: the token for accessing all of the other administration
endpoints. If this token is not set in the config file, access to these
endpoints is only possible with a user-defined admin token.
With the introduction of multiple user-defined admin tokens, the use of master
API tokens is now discouraged.
## Using the admin API
All of the admin API endpoints are described in the OpenAPI specification:
- APIv2 - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v2.html) - [OpenAPI JSON](https://garagehq.deuxfleurs.fr/api/garage-admin-v2.json)
- APIv1 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.yml)
- APIv0 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml)
Making a request to the API from the command line can be as simple as running:
```bash
curl -H 'Authorization: Bearer s3cr3t' http://localhost:3903/v2/GetClusterStatus | jq
```
For more advanced use cases, we recommend using an SDK.
[Go to the "Build your own app" section to know how to use our SDKs](@/documentation/build/_index.md)
### Making API calls from the `garage` CLI
Since v2.0.0, the `garage` binary provides a subcommand `garage json-api` that
allows you to invoke the API without making an HTTP request. This can be
useful for scripting Garage deployments.
`garage json-api` proxies API calls through Garage's internal RPC protocol,
therefore it does not require any form of authentication: RPC connection
parameters are discovered automatically to contact the locally-running Garage
instance (as when running any other `garage` CLI command).
For simple calls that take no parameters, usage is as follows:
```
$ garage json-api GetClusterHealth
{
"connectedNodes": 3,
"knownNodes": 3,
"partitions": 256,
"partitionsAllOk": 256,
"partitionsQuorum": 256,
"status": "healthy",
"storageNodes": 3,
"storageNodesOk": 3
}
```
If you need to specify a JSON body for your call, you can add it directly after
the name of the function you are calling:
```
$ garage json-api CreateAdminToken '{"name": "test"}'
```
Or you can feed it through stdin by adding a `-` as the last command parameter:
```
$ garage json-api CreateAdminToken -
{"name": "test"}
<EOF>
```
For admin API calls that would have taken query parameters in their HTTP version, these parameters can be passed in the JSON body object:
```
$ garage json-api GetAdminTokenInfo '{"id":"b0e6e0ace2c0b2aca4cdb2de"}'
```
For admin API calls that take both query parameters and a JSON body, combine them in the following fashion:
```
$ garage json-api UpdateAdminToken '{"id":"b0e6e0ace2c0b2aca4cdb2de", "body":{"name":"not a test"}}'
```
## Special administration API endpoints
### Metrics `GET /metrics`
@@ -83,7 +209,7 @@ content-length: 102
date: Tue, 08 Aug 2023 07:22:38 GMT
Garage is fully operational
Consult the full health check API endpoint at /v0/health for more details
Consult the full health check API endpoint at /v2/GetClusterHealth for more details
```
### On-demand TLS `GET /check`
@@ -126,23 +252,7 @@ $ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=exampl
200
```
**References:**
- [Using On-Demand TLS](https://caddyserver.com/docs/automatic-https#using-on-demand-tls)
- [Add option for a backend check to approve use of on-demand TLS](https://github.com/caddyserver/caddy/pull/1939)
- [Serving tens of thousands of domains over HTTPS with Caddy](https://caddy.community/t/serving-tens-of-thousands-of-domains-over-https-with-caddy/11179)
### Cluster operations
These endpoints have a dedicated OpenAPI spec.
- APIv1 - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.yml)
- APIv0 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml)
Requesting the API from the command line can be as simple as running:
```bash
curl -H 'Authorization: Bearer s3cr3t' http://localhost:3903/v0/status | jq
```
For more advanced use cases, we recommend using a SDK.
[Go to the "Build your own app" section to know how to use our SDKs](@/documentation/build/_index.md)
@@ -24,6 +24,7 @@ db_engine = "lmdb"
block_size = "1M"
block_ram_buffer_max = "256MiB"
block_max_concurrent_reads = 16
lmdb_map_size = "1T"
@@ -97,6 +98,7 @@ The following gives details about each available configuration option.
Top-level configuration options, in alphabetical order:
[`allow_punycode`](#allow_punycode),
[`allow_world_readable_secrets`](#allow_world_readable_secrets),
[`block_max_concurrent_reads`](`block_max_concurrent_reads),
[`block_ram_buffer_max`](#block_ram_buffer_max),
[`block_size`](#block_size),
[`bootstrap_peers`](#bootstrap_peers),
@@ -335,6 +337,7 @@ Since `v0.8.0`, Garage can use alternative storage backends as follows:
| --------- | ----------------- | ------------- |
| [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
| [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
| [Fjall](https://github.com/fjall-rs/fjall) (**experimental support** since `v1.3.0`) | `"fjall"` | `<metadata_dir>/db.fjall/` |
| [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` |
Sled was supported until Garage v0.9.x, and was removed in Garage v1.0.
@@ -371,6 +374,14 @@ LMDB works very well, but is known to have the following limitations:
so it is not the best choice for high-performance storage clusters,
but it should work fine in many cases.
- Fjall: a storage engine based on LSM trees, which theoretically allow for
higher write throughput than other storage engines that are based on B-trees.
Using Fjall could potentially improve Garage's performance significantly in
write-heavy workloads. **Support for Fjall is experimental at this point**,
we have added it to Garage for evaluation purposes only. **Do not use it for
production-critical workloads.**
It is possible to convert Garage's metadata directory from one format to another
using the `garage convert-db` command, which should be used as follows:
@@ -408,6 +419,7 @@ Here is how this option impacts the different database engines:
|----------|------------------------------------|-------------------------------|
| Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` |
| LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` |
| Fjall | default options | not supported |
Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).
@@ -514,6 +526,29 @@ node.
The default value is 256MiB.
#### `block_max_concurrent_reads` (since `v1.3.0` / `v2.1.0`) {#block_max_concurrent_reads}
The maximum number of blocks (individual files in the data directory) open
simultaneously for reading.
Reducing this number does not limit the number of data blocks that can be
transferred through the network simultaneously. This mechanism was just added
as a backpressure mechanism for HDD read speed: it helps avoid a situation
where too many requests are coming in and Garage is reading too many block
files simultaneously, thus not making timely progress on any of the reads.
When a request to read a data block comes in through the network, the requests
awaits for one of the `block_max_concurrent_reads` slots to be available
(internally implemented using a Semaphore object). Once it acquired a read
slot, it reads the entire block file to RAM and frees the slot as soon as the
block file is finished reading. Only after the slot is released will the
block's data start being transferred over the network. If the request fails to
acquire a reading slot wihtin 15 seconds, it fails with a timeout error.
Timeout events can be monitored through the `block_read_semaphore_timeouts`
metric in Prometheus: a non-zero number of such events indicates an I/O
bottleneck on HDD read speed.
#### `lmdb_map_size` {#lmdb_map_size}
This parameters can be used to set the map size used by LMDB,
+1 -1
View File
@@ -129,5 +129,5 @@ related to objects stored in an S3 bucket.
In the context of our research project, [Aérogramme](https://aerogramme.deuxfleurs.fr),
K2V is used to provide metadata and log storage for operations on encrypted e-mail storage.
Learn more on the specification of K2V [here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/k2v/doc/drafts/k2v-spec.md)
Learn more on the specification of K2V [here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/f8be15c37db857e177d543de7be863692628d567/doc/drafts/k2v-spec.md)
and on how to enable it in Garage [here](@/documentation/reference-manual/k2v.md).
+1 -1
View File
@@ -16,7 +16,7 @@ the `k2v` feature flag enabled can be obtained from our download page under
with `-k2v` (example: `v0.7.2-k2v`).
The specification of the K2V API can be found
[here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/main/doc/drafts/k2v-spec.md).
[here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/commit/f8be15c37db857e177d543de7be863692628d567/doc/drafts/k2v-spec.md).
This document also includes a high-level overview of K2V's design.
The K2V API uses AWSv4 signatures for authentification, same as the S3 API.
+1 -1
View File
@@ -1,6 +1,6 @@
+++
title = "Migrating from 0.3 to 0.4"
weight = 20
weight = 80
+++
**Migrating from 0.3 to 0.4 is unsupported. This document is only intended to
+1 -1
View File
@@ -1,6 +1,6 @@
+++
title = "Migrating from 0.5 to 0.6"
weight = 15
weight = 75
+++
**This guide explains how to migrate to 0.6 if you have an existing 0.5 cluster.
+1 -1
View File
@@ -1,6 +1,6 @@
+++
title = "Migrating from 0.6 to 0.7"
weight = 14
weight = 74
+++
**This guide explains how to migrate to 0.7 if you have an existing 0.6 cluster.
We don't recommend trying to migrate to 0.7 directly from 0.5 or older.**
+1 -1
View File
@@ -1,6 +1,6 @@
+++
title = "Migrating from 0.7 to 0.8"
weight = 13
weight = 73
+++
**This guide explains how to migrate to 0.8 if you have an existing 0.7 cluster.
+1 -1
View File
@@ -1,6 +1,6 @@
+++
title = "Migrating from 0.8 to 0.9"
weight = 12
weight = 72
+++
**This guide explains how to migrate to 0.9 if you have an existing 0.8 cluster.
+1 -1
View File
@@ -1,6 +1,6 @@
+++
title = "Migrating from 0.9 to 1.0"
weight = 11
weight = 71
+++
**This guide explains how to migrate to 1.0 if you have an existing 0.9 cluster.
+70
View File
@@ -0,0 +1,70 @@
+++
title = "Migrating from 1.0 to 2.0"
weight = 70
+++
**This guide explains how to migrate to v2.x if you have an existing v1.x.x cluster.
We don't recommend trying to migrate to v2.x directly from v0.9.x or older.**
This migration procedure has been tested on several clusters without issues.
However, it is still a *critical procedure* that might cause issues.
**Make sure to back up all your data before attempting it!**
You might also want to read our [general documentation on upgrading Garage](@/documentation/operations/upgrading.md).
## Changes introduced in v2.0
The following are **breaking changes** in Garage v2.0 that require your attention when migrating:
- The administration API has been completely reworked.
Some calls to the `/v1/` endpoints will still work but most will not.
New endpoints are prefixed by `/v2/`. **You will need to update all your code that makes use of the admin API.**
- `replication_mode` is no longer a supported configuration parameter,
please use `replication_factor` and `consistency_mode` instead.
## Migration procedure
The migration to Garage v2.0 can be done with almost no downtime,
by restarting all nodes at once in the new version.
The migration steps are as follows:
1. Do a `garage repair --all-nodes --yes tables`, check the logs and check that
all data seems to be synced correctly between nodes. If you have time, do
additional `garage repair` procedures (`blocks`, `versions`, `block_refs`,
etc.)
2. Ensure you have a snapshot of your Garage installation that you can restore
to in case the upgrade goes wrong, with one of the following options:
- You may use the `garage meta snapshot --all` command
to make a backup snapshot of the metadata directories of your nodes
for backup purposes. Once this command has completed, copy the following
files and directories from the `metadata_dir` of all your nodes
to somewhere safe: `snapshots`, `cluster_layout`, `data_layout`,
`node_key`, `node_key.pub`. (If you have set the `metadata_snapshots_dir`
to a different value in your config file, back up that directory instead.)
- If you are running a filesystem such as ZFS or BTRFS that support
snapshotting, you can create a filesystem-level snapshot of the `metadata_dir`
of all your nodes to be used as a restoration point if needed.
- You may also make a back-up manually: turn off each node
individually; back up its metadata folder (for instance, use the following
command if your metadata directory is `/var/lib/garage/meta`: `cd
/var/lib/garage ; tar -acf meta-v1.0.tar.zst meta/`); turn it back on
again. This will allow you to take a backup of all nodes without
impacting global cluster availability. You can do all nodes of a single
zone at once as this does not impact the availability of Garage.
3. Prepare your updated binaries and configuration files for Garage v2.0.
**Remember to update your configuration file to remove `replication_mode` and replace it by `replication_factor`.**
4. Shut down all v1.0 nodes simultaneously, and restart them all simultaneously
in v2.0. Use your favorite deployment tool (Ansible, Kubernetes, Nomad) to
achieve this as fast as possible. Garage v2.0 should be in a working state
as soon as enough nodes have started.
5. Monitor your cluster in the following hours to see if it works well under
your production load.
@@ -1,6 +1,6 @@
+++
title = "Testing strategy"
weight = 30
weight = 100
+++
+6
View File
@@ -53,6 +53,9 @@
tests-sqlite = testWith {
GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite";
};
tests-fjall = testWith {
GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall";
};
};
# ---- developpment shell, for making native builds only ----
@@ -78,6 +81,9 @@
cargo-outdated
cargo-machete
nixpkgs-fmt
openssl
socat
killall
];
};
};
+1 -1
View File
@@ -68,7 +68,7 @@ let
rootFeatures = if features != null then
features
else
([ "bundled-libs" "lmdb" "sqlite" "k2v" ] ++ (lib.optionals release [
([ "bundled-libs" "lmdb" "sqlite" "fjall" "k2v" ] ++ (lib.optionals release [
"consul-discovery"
"kubernetes-discovery"
"metrics"
+2 -2
View File
@@ -2,8 +2,8 @@ apiVersion: v2
name: garage
description: S3-compatible object store for small self-hosted geo-distributed deployments
type: application
version: 0.8.0
appVersion: "v2.0.0"
version: 0.9.1
appVersion: "v2.1.0"
home: https://garagehq.deuxfleurs.fr/
icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg
+2 -1
View File
@@ -1,6 +1,6 @@
# garage
![Version: 0.8.0](https://img.shields.io/badge/Version-0.8.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.0.0](https://img.shields.io/badge/AppVersion-v2.0.0-informational?style=flat-square)
![Version: 0.9.1](https://img.shields.io/badge/Version-0.9.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.1.0](https://img.shields.io/badge/AppVersion-v2.1.0-informational?style=flat-square)
S3-compatible object store for small self-hosted geo-distributed deployments
@@ -15,6 +15,7 @@ S3-compatible object store for small self-hosted geo-distributed deployments
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| affinity | object | `{}` | |
| commonLabels | object | `{}` | Extra labels for all resources |
| deployment.kind | string | `"StatefulSet"` | Switchable to DaemonSet |
| deployment.podManagementPolicy | string | `"OrderedReady"` | If using statefulset, allow Parallel or OrderedReady (default) |
| deployment.replicaCount | int | `3` | Number of StatefulSet replicas/garage nodes to start |
@@ -47,6 +47,9 @@ helm.sh/chart: {{ include "garage.chart" . }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{ with .Values.commonLabels }}
{{- toYaml . }}
{{- end }}
{{- end }}
{{/*
+1 -1
View File
@@ -21,7 +21,7 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "garage.selectorLabels" . | nindent 8 }}
{{- include "garage.labels" . | nindent 8 }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
+5
View File
@@ -2,6 +2,11 @@
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
# -- Additional labels to add to all resources created by this chart
commonLabels: {}
# app.kubernetes.io/part-of: storage
# team: platform
# Garage configuration. These values go to garage.toml
garage:
# -- Can be changed for better performance on certain systems
@@ -694,32 +694,7 @@
]
}
},
"overrides": [
{
"__systemRef": "hideSeriesFrom",
"matcher": {
"id": "byNames",
"options": {
"mode": "exclude",
"names": [
"10.83.2.3:3903"
],
"prefix": "All except:",
"readOnly": true
}
},
"properties": [
{
"id": "custom.hideFrom",
"value": {
"legend": false,
"tooltip": false,
"viz": true
}
}
]
}
]
"overrides": []
},
"gridPos": {
"h": 8,
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_admin"
version = "2.0.0"
version = "2.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+57
View File
@@ -175,6 +175,63 @@ impl RequestHandler for DeleteAdminTokenRequest {
}
}
impl RequestHandler for GetCurrentAdminTokenInfoRequest {
type Response = GetCurrentAdminTokenInfoResponse;
async fn handle(
self,
garage: &Arc<Garage>,
_admin: &Admin,
) -> Result<GetCurrentAdminTokenInfoResponse, Error> {
let now = now_msec();
if garage
.config
.admin
.metrics_token
.as_ref()
.is_some_and(|s| s == &self.admin_token)
{
return Ok(GetCurrentAdminTokenInfoResponse(
GetAdminTokenInfoResponse {
id: None,
created: None,
name: "metrics_token (from daemon configuration)".into(),
expiration: None,
expired: false,
scope: vec!["Metrics".into()],
},
));
}
if garage
.config
.admin
.admin_token
.as_ref()
.is_some_and(|s| s == &self.admin_token)
{
return Ok(GetCurrentAdminTokenInfoResponse(
GetAdminTokenInfoResponse {
id: None,
created: None,
name: "admin_token (from daemon configuration)".into(),
expiration: None,
expired: false,
scope: vec!["*".into()],
},
));
}
let (prefix, _) = self.admin_token.split_once('.').unwrap();
let token = get_existing_admin_token(&garage, &prefix.to_string()).await?;
Ok(GetCurrentAdminTokenInfoResponse(admin_token_info_results(
&token, now,
)))
}
}
// ---- helpers ----
fn admin_token_info_results(token: &AdminApiToken, now: u64) -> GetAdminTokenInfoResponse {
+48 -9
View File
@@ -12,7 +12,7 @@ use garage_rpc::*;
use garage_model::garage::Garage;
use garage_api_common::helpers::is_default;
use garage_api_common::{common_error::CommonError, helpers::is_default};
use crate::api_server::{find_matching_nodes, AdminRpc, AdminRpcResponse};
use crate::error::Error;
@@ -56,6 +56,7 @@ admin_endpoints![
CreateAdminToken,
UpdateAdminToken,
DeleteAdminToken,
GetCurrentAdminTokenInfo,
// Layout operations
GetClusterLayout,
@@ -144,6 +145,13 @@ pub struct MultiResponse<RB> {
pub error: HashMap<String, String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct MultiRequestQueryParams {
/// Node ID to query, or `*` for all nodes, or `self` for the node responding to the request
pub node: String,
}
// **********************************************
// Special endpoints
//
@@ -154,8 +162,10 @@ pub struct MultiResponse<RB> {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OptionsRequest;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct CheckDomainRequest {
/// The domain name to check for
pub domain: String,
}
@@ -257,7 +267,7 @@ pub struct GetClusterHealthResponse {
/// the number of storage nodes currently registered in the cluster layout
pub storage_nodes: usize,
/// the number of storage nodes to which a connection is currently open
pub storage_nodes_ok: usize,
pub storage_nodes_up: usize,
/// the total number of partitions of the data (currently always 256)
pub partitions: usize,
/// the number of partitions for which a quorum of write nodes is available
@@ -308,6 +318,7 @@ pub struct ListAdminTokensResponse(pub Vec<GetAdminTokenInfoResponse>);
// ---- GetAdminTokenInfo ----
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[serde(rename_all = "camelCase")]
pub struct GetAdminTokenInfoRequest {
/// Admin API token ID
@@ -353,9 +364,12 @@ pub struct CreateAdminTokenResponse {
// ---- UpdateAdminToken ----
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct UpdateAdminTokenRequest {
/// Admin API token ID
pub id: String,
#[param(ignore = true)]
pub body: UpdateAdminTokenRequestBody,
}
@@ -382,14 +396,25 @@ pub struct UpdateAdminTokenResponse(pub GetAdminTokenInfoResponse);
// ---- DeleteAdminToken ----
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct DeleteAdminTokenRequest {
/// Admin API token ID
pub id: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DeleteAdminTokenResponse;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GetCurrentAdminTokenInfoRequest {
pub admin_token: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "camelCase")]
pub struct GetCurrentAdminTokenInfoResponse(pub GetAdminTokenInfoResponse);
// **********************************************
// Layout operations
// **********************************************
@@ -648,6 +673,7 @@ pub struct ListKeysResponseItem {
// ---- GetKeyInfo ----
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[serde(rename_all = "camelCase")]
pub struct GetKeyInfoRequest {
/// Access key ID
@@ -724,9 +750,12 @@ pub struct ImportKeyResponse(pub GetKeyInfoResponse);
// ---- UpdateKey ----
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct UpdateKeyRequest {
/// Access key ID
pub id: String,
#[param(ignore = true)]
pub body: UpdateKeyRequestBody,
}
@@ -751,8 +780,10 @@ pub struct UpdateKeyRequestBody {
// ---- DeleteKey ----
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct DeleteKeyRequest {
/// Access key ID
pub id: String,
}
@@ -790,6 +821,7 @@ pub struct BucketLocalAlias {
// ---- GetBucketInfo ----
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[serde(rename_all = "camelCase")]
pub struct GetBucketInfoRequest {
/// Exact bucket ID to look up
@@ -878,9 +910,12 @@ pub struct CreateBucketLocalAlias {
// ---- UpdateBucket ----
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct UpdateBucketRequest {
/// ID of the bucket to update
pub id: String,
#[param(ignore = true)]
pub body: UpdateBucketRequestBody,
}
@@ -904,8 +939,10 @@ pub struct UpdateBucketWebsiteAccess {
// ---- DeleteBucket ----
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
pub struct DeleteBucketRequest {
/// ID of the bucket to delete
pub id: String,
}
@@ -928,6 +965,7 @@ pub struct CleanupIncompleteUploadsResponse {
}
#[derive(Debug, Clone, Serialize, Deserialize, IntoParams)]
#[into_params(parameter_in = Query)]
#[serde(rename_all = "camelCase")]
pub struct InspectObjectRequest {
pub bucket_id: String,
@@ -1115,6 +1153,7 @@ pub enum RepairType {
Rebalance,
Scrub(ScrubCommand),
Aliases,
ClearResyncQueue,
}
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
+43 -32
View File
@@ -217,6 +217,13 @@ impl ApiHandler for ArcAdminApiServer {
) -> Result<Response<ResBody>, Error> {
self.0.handle_http_api(req, endpoint).await
}
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
let auth_header = req.headers().get(AUTHORIZATION)?;
let token = parse_authorization(auth_header).ok()?;
let key_id = token.split_once('.')?.0;
Some(key_id.to_string())
}
}
impl ApiEndpoint for HttpEndpoint {
@@ -244,6 +251,15 @@ fn hash_bearer_token(token: &str) -> String {
.to_string()
}
fn parse_authorization(auth_header: &hyper::http::HeaderValue) -> Result<&str, Error> {
let token = auth_header
.to_str()?
.strip_prefix("Bearer ")
.ok_or_else(|| Error::forbidden("Invalid Authorization header"))?
.trim();
Ok(token)
}
fn verify_authorization(
garage: &Garage,
global_token_hash: Option<&str>,
@@ -260,11 +276,7 @@ fn verify_authorization(
"Bearer token must be provided in Authorization header",
))
}
Some(authorization) => authorization
.to_str()?
.strip_prefix("Bearer ")
.ok_or_else(|| Error::forbidden("Invalid Authorization header"))?
.trim(),
Some(authorization) => parse_authorization(authorization)?,
};
let token_hash_string = if let Some((prefix, _)) = token.split_once('.') {
@@ -273,7 +285,8 @@ fn verify_authorization(
.get_local(&EmptyKey, &prefix.to_string())?
.and_then(|k| k.state.into_option())
.filter(|p| !p.is_expired(now_msec()))
.filter(|p| p.has_scope(endpoint_name))
// GetCurrentAdminTokenInfo endpoint must be accessible even if it is not in the token scopes
.filter(|p| p.has_scope(endpoint_name) || endpoint_name == "GetCurrentAdminTokenInfo")
.ok_or_else(|| Error::forbidden(invalid_msg))?
.token_hash
} else {
@@ -293,38 +306,36 @@ fn verify_authorization(
}
pub(crate) fn find_matching_nodes(garage: &Garage, spec: &str) -> Result<Vec<Uuid>, Error> {
let mut res = vec![];
if spec == "*" {
res = garage.system.cluster_layout().all_nodes().to_vec();
if spec == "self" {
Ok(vec![garage.system.id])
} else {
// Collect all nodes currently up and/or in cluster layout
let mut res = vec![];
if let Ok(all_nodes) = garage.system.cluster_layout().all_nodes() {
res = all_nodes.to_vec();
}
for node in garage.system.get_known_nodes() {
if node.is_up && !res.contains(&node.id) {
res.push(node.id);
}
}
} else if spec == "self" {
res.push(garage.system.id);
} else {
let layout = garage.system.cluster_layout();
let known_nodes = garage.system.get_known_nodes();
let all_nodes = layout
.all_nodes()
.iter()
.copied()
.chain(known_nodes.iter().filter(|x| x.is_up).map(|x| x.id));
for node in all_nodes {
if !res.contains(&node) && hex::encode(node).starts_with(spec) {
res.push(node);
if spec == "*" {
// match all nodes
Ok(res)
} else {
// filter nodes that match spec
res.retain(|node| hex::encode(node).starts_with(spec));
if res.is_empty() {
Err(Error::bad_request(format!("No nodes matching {}", spec)))
} else if res.len() > 1 {
Err(Error::bad_request(format!(
"Multiple nodes matching {}: {:?}",
spec, res
)))
} else {
Ok(res)
}
}
if res.is_empty() {
return Err(Error::bad_request(format!("No nodes matching {}", spec)));
}
if res.len() > 1 {
return Err(Error::bad_request(format!(
"Multiple nodes matching {}: {:?}",
spec, res
)));
}
}
Ok(res)
}
+13 -12
View File
@@ -697,20 +697,21 @@ async fn bucket_info_results(
}),
keys: relevant_keys
.into_values()
.map(|key| {
.filter_map(|key| {
let p = key.state.as_option().unwrap();
GetBucketInfoKey {
let permissions = p
.authorized_buckets
.get(&bucket.id)
.filter(|p| p.is_any())
.map(|p| ApiBucketKeyPerm {
read: p.allow_read,
write: p.allow_write,
owner: p.allow_owner,
})?;
Some(GetBucketInfoKey {
access_key_id: key.key_id,
name: p.name.get().to_string(),
permissions: p
.authorized_buckets
.get(&bucket.id)
.map(|p| ApiBucketKeyPerm {
read: p.allow_read,
write: p.allow_write,
owner: p.allow_owner,
})
.unwrap_or_default(),
permissions,
bucket_local_aliases: p
.local_aliases
.items()
@@ -718,7 +719,7 @@ async fn bucket_info_results(
.filter(|(_, _, b)| *b == Some(bucket.id))
.map(|(n, _, _)| n.to_string())
.collect::<Vec<_>>(),
}
})
})
.collect::<Vec<_>>(),
objects: *counters.get(OBJECTS).unwrap_or(&0),
+54 -42
View File
@@ -56,48 +56,52 @@ impl RequestHandler for GetClusterStatusRequest {
})
.collect::<HashMap<_, _>>();
for (id, _, role) in layout.current().roles.items().iter() {
if let layout::NodeRoleV(Some(r)) = role {
let role = NodeAssignedRole {
zone: r.zone.to_string(),
capacity: r.capacity,
tags: r.tags.clone(),
};
match nodes.get_mut(id) {
None => {
nodes.insert(
*id,
NodeResp {
id: hex::encode(id),
role: Some(role),
..Default::default()
},
);
}
Some(n) => {
n.role = Some(role);
if let Ok(current_layout) = layout.current() {
for (id, _, role) in current_layout.roles.items().iter() {
if let layout::NodeRoleV(Some(r)) = role {
let role = NodeAssignedRole {
zone: r.zone.to_string(),
capacity: r.capacity,
tags: r.tags.clone(),
};
match nodes.get_mut(id) {
None => {
nodes.insert(
*id,
NodeResp {
id: hex::encode(id),
role: Some(role),
..Default::default()
},
);
}
Some(n) => {
n.role = Some(role);
}
}
}
}
}
for ver in layout.versions().iter().rev().skip(1) {
for (id, _, role) in ver.roles.items().iter() {
if let layout::NodeRoleV(Some(r)) = role {
if r.capacity.is_some() {
if let Some(n) = nodes.get_mut(id) {
if n.role.is_none() {
n.draining = true;
if let Ok(layout_versions) = layout.versions() {
for ver in layout_versions.iter().rev().skip(1) {
for (id, _, role) in ver.roles.items().iter() {
if let layout::NodeRoleV(Some(r)) = role {
if r.capacity.is_some() {
if let Some(n) = nodes.get_mut(id) {
if n.role.is_none() {
n.draining = true;
}
} else {
nodes.insert(
*id,
NodeResp {
id: hex::encode(id),
draining: true,
..Default::default()
},
);
}
} else {
nodes.insert(
*id,
NodeResp {
id: hex::encode(id),
draining: true,
..Default::default()
},
);
}
}
}
@@ -108,7 +112,7 @@ impl RequestHandler for GetClusterStatusRequest {
nodes.sort_by(|x, y| x.id.cmp(&y.id));
Ok(GetClusterStatusResponse {
layout_version: layout.current().version,
layout_version: layout.inner().current().version,
nodes,
})
}
@@ -134,7 +138,9 @@ impl RequestHandler for GetClusterHealthRequest {
known_nodes: health.known_nodes,
connected_nodes: health.connected_nodes,
storage_nodes: health.storage_nodes,
storage_nodes_ok: health.storage_nodes_ok,
// Translating storage_nodes_up (admin API context) to storage_nodes_ok (metrics context)
// TODO: when releasing major release, consider renaming all the fields in the metrics to storage_nodes_up
storage_nodes_up: health.storage_nodes_ok,
partitions: health.partitions,
partitions_quorum: health.partitions_quorum,
partitions_all_ok: health.partitions_all_ok,
@@ -157,9 +163,11 @@ impl RequestHandler for GetClusterStatisticsRequest {
// Gather storage node and free space statistics for current nodes
let layout = &garage.system.cluster_layout();
let mut node_partition_count = HashMap::<Uuid, u64>::new();
for short_id in layout.current().ring_assignment_data.iter() {
let id = layout.current().node_id_vec[*short_id as usize];
*node_partition_count.entry(id).or_default() += 1;
if let Ok(current_layout) = layout.current() {
for short_id in current_layout.ring_assignment_data.iter() {
let id = current_layout.node_id_vec[*short_id as usize];
*node_partition_count.entry(id).or_default() += 1;
}
}
let node_info = garage
.system
@@ -172,7 +180,11 @@ impl RequestHandler for GetClusterStatisticsRequest {
for (id, parts) in node_partition_count.iter() {
let info = node_info.get(id);
let status = info.map(|x| &x.status);
let role = layout.current().roles.get(id).and_then(|x| x.0.as_ref());
let role = layout
.current()
.ok()
.and_then(|l| l.roles.get(id))
.and_then(|x| x.0.as_ref());
let hostname = status.and_then(|x| x.hostname.as_deref()).unwrap_or("?");
let zone = role.map(|x| x.zone.as_str()).unwrap_or("?");
let capacity = role
+1
View File
@@ -95,6 +95,7 @@ impl ApiError for Error {
fn add_http_headers(&self, header_map: &mut HeaderMap<HeaderValue>) {
use hyper::header;
header_map.append(header::CONTENT_TYPE, "application/json".parse().unwrap());
header_map.append(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
}
fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody {
+13 -12
View File
@@ -230,9 +230,18 @@ async fn key_info_results(
},
buckets: relevant_buckets
.into_values()
.map(|bucket| {
.filter_map(|bucket| {
let state = bucket.state.as_option().unwrap();
KeyInfoBucketResponse {
let permissions = key_state
.authorized_buckets
.get(&bucket.id)
.filter(|p| p.is_any())
.map(|p| ApiBucketKeyPerm {
read: p.allow_read,
write: p.allow_write,
owner: p.allow_owner,
})?;
Some(KeyInfoBucketResponse {
id: hex::encode(bucket.id),
global_aliases: state
.aliases
@@ -248,16 +257,8 @@ async fn key_info_results(
.filter(|((k, _), _, a)| *a && *k == key.key_id)
.map(|((_, n), _, _)| n.to_string())
.collect::<Vec<_>>(),
permissions: key_state
.authorized_buckets
.get(&bucket.id)
.map(|p| ApiBucketKeyPerm {
read: p.allow_read,
write: p.allow_write,
owner: p.allow_owner,
})
.unwrap_or_default(),
}
permissions,
})
})
.collect::<Vec<_>>(),
};
+7 -5
View File
@@ -74,16 +74,18 @@ macro_rules! admin_endpoints {
type Response = AdminApiResponse;
async fn handle(self, garage: &Arc<Garage>, admin: &Admin) -> Result<AdminApiResponse, Error> {
Ok(match self {
match self {
$(
AdminApiRequest::$special_endpoint(_) => panic!(
concat!(stringify!($special_endpoint), " needs to go through a special handler")
AdminApiRequest::$special_endpoint(_) => Err(
Error::Common(CommonError::BadRequest(
concat!(stringify!($special_endpoint), " cannot be used outside of the HTTP Admin API").into()
))
),
)*
$(
AdminApiRequest::$endpoint(req) => AdminApiResponse::$endpoint(req.handle(garage, admin).await?),
AdminApiRequest::$endpoint(req) => Ok(AdminApiResponse::$endpoint(req.handle(garage, admin).await?)),
)*
})
}
}
}
}
+13 -8
View File
@@ -106,17 +106,17 @@ impl RequestHandler for LocalGetNodeStatisticsRequest {
// Gather block manager statistics
writeln!(&mut ret, "\nBlock manager stats:").unwrap();
let rc_len = garage.block_manager.rc_len()?.to_string();
let rc_len = garage.block_manager.rc_approximate_len()?.to_string();
ret += &format_table_to_string(vec![
format!(" number of RC entries:\t{} (~= number of blocks)", rc_len),
format!(
" resync queue length:\t{}",
garage.block_manager.resync.queue_len()?
garage.block_manager.resync.queue_approximate_len()?
),
format!(
" blocks with resync errors:\t{}",
garage.block_manager.resync.errors_len()?
garage.block_manager.resync.errors_approximate_len()?
),
]);
@@ -129,16 +129,21 @@ where
F: TableSchema + 'static,
R: TableReplication + 'static,
{
let data_len = t.data.store.len().map_err(GarageError::from)?.to_string();
let mkl_len = t.merkle_updater.merkle_tree_len()?.to_string();
let data_len = t
.data
.store
.approximate_len()
.map_err(GarageError::from)?
.to_string();
let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?.to_string();
Ok(format!(
" {}\t{}\t{}\t{}\t{}\t{}",
F::TABLE_NAME,
data_len,
mkl_len,
t.merkle_updater.todo_len()?,
t.data.insert_queue_len()?,
t.data.gc_todo_len()?
t.merkle_updater.todo_approximate_len()?,
t.data.insert_queue_approximate_len()?,
t.data.gc_todo_approximate_len()?
))
}
+34 -58
View File
@@ -46,9 +46,7 @@ a static website for the requested domain. This is used by reverse proxies such
as Caddy or Tricot, to avoid requesting TLS certificates for domain names that
do not correspond to an actual website.
",
params(
("domain", description = "The domain name to check for"),
),
params(CheckDomainRequest),
security(()),
responses(
(status = 200, description = "The domain name redirects to a static website bucket"),
@@ -167,9 +165,7 @@ fn CreateAdminToken() -> () {}
Updates information about the specified admin API token.
",
request_body = UpdateAdminTokenRequestBody,
params(
("id", description = "Admin API token ID"),
),
params(UpdateAdminTokenRequest),
responses(
(status = 200, description = "Admin token has been updated", body = UpdateAdminTokenResponse),
(status = 500, description = "Internal server error")
@@ -181,9 +177,7 @@ fn UpdateAdminToken() -> () {}
path = "/v2/DeleteAdminToken",
tag = "Admin API token",
description = "Delete an admin API token from the cluster, revoking all its permissions.",
params(
("id", description = "Admin API token ID"),
),
params(DeleteAdminTokenRequest),
responses(
(status = 200, description = "Admin token has been deleted"),
(status = 500, description = "Internal server error")
@@ -191,6 +185,19 @@ fn UpdateAdminToken() -> () {}
)]
fn DeleteAdminToken() -> () {}
#[utoipa::path(get,
path = "/v2/GetCurrentAdminTokenInfo",
tag = "Admin API token",
description = "
Return information about the calling admin API token.
",
responses(
(status = 200, description = "Information about the admin token", body = GetCurrentAdminTokenInfoResponse),
(status = 500, description = "Internal server error")
),
)]
fn GetCurrentAdminTokenInfo() -> () {}
// **********************************************
// Layout operations
// **********************************************
@@ -378,9 +385,7 @@ Updates information about the specified API access key.
*Note: the secret key is not returned in the response, `null` is sent instead.*
",
request_body = UpdateKeyRequestBody,
params(
("id", description = "Access key ID"),
),
params(UpdateKeyRequest),
responses(
(status = 200, description = "Access key has been updated", body = UpdateKeyResponse),
(status = 500, description = "Internal server error")
@@ -392,9 +397,7 @@ fn UpdateKey() -> () {}
path = "/v2/DeleteKey",
tag = "Access key",
description = "Delete a key from the cluster. Its access will be removed from all the buckets. Buckets are not automatically deleted and can be dangling. You should manually delete them before. ",
params(
("id", description = "Access key ID"),
),
params(DeleteKeyRequest),
responses(
(status = 200, description = "Access key has been deleted"),
(status = 500, description = "Internal server error")
@@ -465,9 +468,7 @@ In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or
to remove the quotas. An absent value will be considered the same as a `null`. It is not possible
to change only one of the two quotas.
",
params(
("id", description = "ID of the bucket to update"),
),
params(UpdateBucketRequest),
request_body = UpdateBucketRequestBody,
responses(
(status = 200, description = "Bucket has been updated", body = UpdateBucketResponse),
@@ -485,9 +486,7 @@ Deletes a storage bucket. A bucket cannot be deleted if it is not empty.
**Warning:** this will delete all aliases associated with the bucket!
",
params(
("id", description = "ID of the bucket to delete"),
),
params(DeleteBucketRequest),
responses(
(status = 200, description = "Bucket has been deleted"),
(status = 400, description = "Bucket is not empty"),
@@ -617,9 +616,7 @@ fn RemoveBucketAlias() -> () {}
description = "
Return information about the Garage daemon running on one or several nodes.
",
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
params(MultiRequestQueryParams),
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetNodeInfoResponse>),
(status = 500, description = "Internal server error")
@@ -635,9 +632,7 @@ Fetch statistics for one or several Garage nodes.
*Note: do not try to parse the `freeform` field of the response, it is given as a string specifically because its format is not stable.*
",
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
params(MultiRequestQueryParams),
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetNodeStatisticsResponse>),
(status = 500, description = "Internal server error")
@@ -651,9 +646,7 @@ fn GetNodeStatistics() -> () {}
description = "
Instruct one or several nodes to take a snapshot of their metadata databases.
",
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
params(MultiRequestQueryParams),
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalCreateMetadataSnapshotResponse>),
(status = 500, description = "Internal server error")
@@ -667,9 +660,7 @@ fn CreateMetadataSnapshot() -> () {}
description = "
Launch a repair operation on one or several cluster nodes.
",
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
params(MultiRequestQueryParams),
request_body = LocalLaunchRepairOperationRequest,
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalLaunchRepairOperationResponse>),
@@ -688,9 +679,7 @@ fn LaunchRepairOperation() -> () {}
description = "
List background workers currently running on one or several cluster nodes.
",
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
params(MultiRequestQueryParams),
request_body = LocalListWorkersRequest,
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalListWorkersResponse>),
@@ -705,9 +694,7 @@ fn ListWorkers() -> () {}
description = "
Get information about the specified background worker on one or several cluster nodes.
",
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
params(MultiRequestQueryParams),
request_body = LocalGetWorkerInfoRequest,
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetWorkerInfoResponse>),
@@ -722,9 +709,7 @@ fn GetWorkerInfo() -> () {}
description = "
Fetch values of one or several worker variables, from one or several cluster nodes.
",
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
params(MultiRequestQueryParams),
request_body = LocalGetWorkerVariableRequest,
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalGetWorkerVariableResponse>),
@@ -739,9 +724,7 @@ fn GetWorkerVariable() -> () {}
description = "
Set the value for a worker variable, on one or several cluster nodes.
",
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
params(MultiRequestQueryParams),
request_body = LocalSetWorkerVariableRequest,
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalSetWorkerVariableResponse>),
@@ -760,9 +743,7 @@ fn SetWorkerVariable() -> () {}
description = "
List data blocks that are currently in an errored state on one or several Garage nodes.
",
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
params(MultiRequestQueryParams),
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalListBlockErrorsResponse>),
(status = 500, description = "Internal server error")
@@ -776,9 +757,7 @@ fn ListBlockErrors() -> () {}
description = "
Get detailed information about a data block stored on a Garage node, including all object versions and in-progress multipart uploads that contain a reference to this block.
",
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
params(MultiRequestQueryParams),
request_body = LocalGetBlockInfoRequest,
responses(
(status = 200, description = "Detailed block information", body = MultiResponse<LocalGetBlockInfoResponse>),
@@ -793,9 +772,7 @@ fn GetBlockInfo() -> () {}
description = "
Instruct Garage node(s) to retry the resynchronization of one or several missing data block(s).
",
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
params(MultiRequestQueryParams),
request_body = LocalRetryBlockResyncRequest,
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalRetryBlockResyncResponse>),
@@ -812,9 +789,7 @@ Purge references to one or several missing data blocks.
This will remove all objects and in-progress multipart uploads that contain the specified data block(s). The objects will be permanently deleted from the buckets in which they appear. Use with caution.
",
params(
("node", description = "Node ID to query, or `*` for all nodes, or `self` for the node responding to the request"),
),
params(MultiRequestQueryParams),
request_body = LocalPurgeBlocksRequest,
responses(
(status = 200, description = "Responses from individual cluster nodes", body = MultiResponse<LocalPurgeBlocksResponse>),
@@ -843,7 +818,7 @@ impl Modify for SecurityAddon {
#[derive(OpenApi)]
#[openapi(
info(
version = "v2.0.0",
version = "v2.1.0",
title = "Garage administration API",
description = "Administrate your Garage cluster programatically, including status, layout, keys, buckets, and maintainance tasks.
@@ -872,6 +847,7 @@ impl Modify for SecurityAddon {
CreateAdminToken,
UpdateAdminToken,
DeleteAdminToken,
GetCurrentAdminTokenInfo,
// Layout operations
GetClusterLayout,
GetClusterLayoutHistory,
+9
View File
@@ -93,6 +93,15 @@ impl RequestHandler for LocalLaunchRepairOperationRequest {
info!("Repairing bucket aliases (foreground)");
garage.locked_helper().await.repair_aliases().await?;
}
RepairType::ClearResyncQueue => {
info!("Clearing resync queue (foreground)");
let garage = garage.clone();
tokio::task::spawn_blocking(move || {
garage.block_manager.resync.clear_resync_queue()
})
.await
.map_err(garage_util::error::Error::from)??;
}
}
Ok(LocalLaunchRepairOperationResponse)
}
+1
View File
@@ -40,6 +40,7 @@ impl AdminApiRequest {
POST CreateAdminToken (body),
POST UpdateAdminToken (body_field, query::id),
POST DeleteAdminToken (query::id),
GET GetCurrentAdminTokenInfo (admin_token),
// Layout endpoints
GET GetClusterLayout (),
GET GetClusterLayoutHistory (),
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_common"
version = "2.0.0"
version = "2.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+22 -11
View File
@@ -59,6 +59,12 @@ pub trait ApiHandler: Send + Sync + 'static {
req: Request<IncomingBody>,
endpoint: Self::Endpoint,
) -> impl Future<Output = Result<Response<BoxBody<Self::Error>>, Self::Error>> + Send;
/// Returns the key id used to authenticate this request. The ID returned must be safe to
/// log.
fn key_id_from_request(&self, _req: &Request<IncomingBody>) -> Option<String> {
None
}
}
pub struct ApiServer<A: ApiHandler> {
@@ -143,19 +149,20 @@ impl<A: ApiHandler> ApiServer<A> {
) -> Result<Response<BoxBody<A::Error>>, http::Error> {
let uri = req.uri().clone();
if let Ok(forwarded_for_ip_addr) =
let source = if let Ok(forwarded_for_ip_addr) =
forwarded_headers::handle_forwarded_for_headers(req.headers())
{
info!(
"{} (via {}) {} {}",
forwarded_for_ip_addr,
addr,
req.method(),
uri
);
format!("{forwarded_for_ip_addr} (via {addr})")
} else {
info!("{} {} {}", addr, req.method(), uri);
}
format!("{addr}")
};
// we only do this to log the access key, so we can discard any error
let key = self
.api_handler
.key_id_from_request(&req)
.map(|k| format!("(key {k}) "))
.unwrap_or_default();
info!("{source} {key}{} {uri}", req.method());
debug!("{:?}", req);
let tracer = opentelemetry::global::tracer("garage");
@@ -344,7 +351,11 @@ where
while !*must_exit.borrow() {
let (stream, client_addr) = tokio::select! {
acc = listener.accept() => acc?,
acc = listener.accept() => match acc {
Ok(r) => r,
Err(e) if e.kind() == std::io::ErrorKind::ConnectionAborted => continue,
Err(e) => return Err(e.into()),
},
_ = must_exit.changed() => continue,
};
+15
View File
@@ -83,6 +83,21 @@ macro_rules! router_match {
parse_json_body::< [<$api Request>], _, Error>($req).await?
})
}};
(@@gen_parse_request $api:ident, (admin_token), $query: expr, $req:expr) => {{
paste!({
let auth_header = $req.headers()
.get(hyper::header::AUTHORIZATION)
.ok_or_else(|| Error::bad_request("Missing Authorization header"))?
.to_str()
.map_err(|_| Error::bad_request("Invalid Authorization header"))?;
let admin_token = auth_header.strip_prefix("Bearer ")
.ok_or_else(|| Error::bad_request("Authorization header must be Bearer token"))?
.to_string();
[< $api Request >] { admin_token }
})
}};
(@@gen_parse_request $api:ident, (body_field, $($conv:ident $(($conv_arg:expr))? :: $param:ident),*), $query: expr, $req:expr)
=>
{{
+2 -2
View File
@@ -424,7 +424,7 @@ pub fn verify_v4(
// ============ Authorization header, or X-Amz-* query params =========
pub struct Authorization {
key_id: String,
pub key_id: String,
scope: String,
signed_headers: String,
signature: String,
@@ -433,7 +433,7 @@ pub struct Authorization {
}
impl Authorization {
fn parse_header(headers: &HeaderMap) -> Result<Self, Error> {
pub fn parse_header(headers: &HeaderMap) -> Result<Self, Error> {
let authorization = headers
.get(AUTHORIZATION)
.ok_or_bad_request("Missing authorization header")?
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_k2v"
version = "2.0.0"
version = "2.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+6
View File
@@ -171,6 +171,12 @@ impl ApiHandler for K2VApiServer {
Ok(resp_ok)
}
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
garage_api_common::signature::payload::Authorization::parse_header(req.headers())
.map(|auth| auth.key_id)
.ok()
}
}
impl ApiEndpoint for K2VApiEndpoint {
+1
View File
@@ -99,6 +99,7 @@ impl ApiError for Error {
fn add_http_headers(&self, header_map: &mut HeaderMap<HeaderValue>) {
use hyper::header;
header_map.append(header::CONTENT_TYPE, "application/json".parse().unwrap());
header_map.append(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
}
fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody {
+2 -2
View File
@@ -28,7 +28,7 @@ pub async fn handle_read_index(
let node_id_vec = garage
.system
.cluster_layout()
.all_nongateway_nodes()
.all_nongateway_nodes()?
.to_vec();
let (partition_keys, more, next_start) = read_range(
@@ -66,7 +66,7 @@ pub async fn handle_read_index(
bytes: *vals.get(&s_bytes).unwrap_or(&0),
}
})
.collect::<Vec<_>>(),
.collect(),
more,
next_start,
};
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api_s3"
version = "2.0.0"
version = "2.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+6
View File
@@ -340,6 +340,12 @@ impl ApiHandler for S3ApiServer {
Ok(resp_ok)
}
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
garage_api_common::signature::payload::Authorization::parse_header(req.headers())
.map(|auth| auth.key_id)
.ok()
}
}
impl ApiEndpoint for S3ApiEndpoint {
+1 -1
View File
@@ -29,7 +29,7 @@ pub async fn handle_get_cors(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
.body(string_body(xml))?)
} else {
Ok(Response::builder()
.status(StatusCode::NO_CONTENT)
.status(StatusCode::NOT_FOUND)
.body(empty_body())?)
}
}
+1
View File
@@ -176,6 +176,7 @@ impl ApiError for Error {
use hyper::header;
header_map.append(header::CONTENT_TYPE, "application/xml".parse().unwrap());
header_map.append(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*".parse().unwrap());
#[allow(clippy::single_match)]
match self {
+44 -43
View File
@@ -5,6 +5,7 @@ use std::sync::Arc;
use base64::prelude::*;
use crc_fast::{CrcAlgorithm, Digest as CrcDigest};
use futures::prelude::*;
use http::StatusCode;
use hyper::{header::HeaderValue, HeaderMap, Request, Response};
use md5::{Digest, Md5};
use sha1::Sha1;
@@ -302,8 +303,11 @@ pub async fn handle_complete_multipart_upload(
let body = req_body.collect().await?;
let body_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?;
let body_list_of_parts = parse_complete_multipart_upload_body(&body_xml)
.ok_or_bad_request("Invalid CompleteMultipartUpload XML")?;
let body_list_of_parts =
parse_complete_multipart_upload_body(&body_xml).ok_or_bad_request(format!(
"Invalid CompleteMultipartUpload XML:\n{}",
String::from_utf8_lossy(&body)
))?;
debug!(
"CompleteMultipartUpload list of parts: {:?}",
body_list_of_parts
@@ -548,7 +552,9 @@ pub async fn handle_abort_multipart_upload(
let final_object = Object::new(*bucket_id, key.to_string(), vec![object_version]);
garage.object_table.insert(&final_object).await?;
Ok(Response::new(empty_body()))
Ok(Response::builder()
.status(StatusCode::NO_CONTENT)
.body(empty_body())?)
}
// ======== helpers ============
@@ -600,6 +606,32 @@ struct CompleteMultipartUploadPart {
checksum: Option<ChecksumValue>,
}
macro_rules! extract_checksum_from {
($node:ident { $($name:expr => $variant:ident),* $(,)? }) => {
if false { None }
$(
else if let Some(node) = $node.children().find(|e| e.has_tag_name($name)) {
match node.last_child().map(|x| x.text()) {
// Child is text but empty post-trim, ignore it.
Some(Some(text)) if text.trim().is_empty() => None,
// Child is non-empty text, parse it.
Some(Some(text)) => Some(ChecksumValue::$variant(
BASE64_STANDARD.decode(text).ok()?[..].try_into().ok()?
)),
// Child is not text, reject it.
Some(None) => return None,
// No child, ignore it.
None => None,
}
}
)*
else { None }
}
}
fn parse_complete_multipart_upload_body(
xml: &roxmltree::Document,
) -> Option<Vec<CompleteMultipartUploadPart>> {
@@ -623,46 +655,15 @@ fn parse_complete_multipart_upload_body(
.children()
.find(|e| e.has_tag_name("PartNumber"))?
.text()?;
let checksum = if let Some(crc32) =
item.children().find(|e| e.has_tag_name("ChecksumCRC32"))
{
Some(ChecksumValue::Crc32(
BASE64_STANDARD.decode(crc32.text()?).ok()?[..]
.try_into()
.ok()?,
))
} else if let Some(crc32c) = item.children().find(|e| e.has_tag_name("ChecksumCRC32C"))
{
Some(ChecksumValue::Crc32c(
BASE64_STANDARD.decode(crc32c.text()?).ok()?[..]
.try_into()
.ok()?,
))
} else if let Some(crc64nvme) = item
.children()
.find(|e| e.has_tag_name("ChecksumCRC64NVME"))
{
Some(ChecksumValue::Crc64Nvme(
BASE64_STANDARD.decode(crc64nvme.text()?).ok()?[..]
.try_into()
.ok()?,
))
} else if let Some(sha1) = item.children().find(|e| e.has_tag_name("ChecksumSHA1")) {
Some(ChecksumValue::Sha1(
BASE64_STANDARD.decode(sha1.text()?).ok()?[..]
.try_into()
.ok()?,
))
} else if let Some(sha256) = item.children().find(|e| e.has_tag_name("ChecksumSHA256"))
{
Some(ChecksumValue::Sha256(
BASE64_STANDARD.decode(sha256.text()?).ok()?[..]
.try_into()
.ok()?,
))
} else {
None
};
let checksum = extract_checksum_from!(item {
"ChecksumCRC32" => Crc32,
"ChecksumCRC32C" => Crc32c,
"ChecksumCRC64NVME" => Crc64Nvme,
"ChecksumSHA1" => Sha1,
"ChecksumSHA256" => Sha256,
});
parts.push(CompleteMultipartUploadPart {
etag: etag.trim_matches('"').to_string(),
part_number: part_number.parse().ok()?,
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_block"
version = "2.0.0"
version = "2.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+24 -14
View File
@@ -48,6 +48,8 @@ pub const INLINE_THRESHOLD: usize = 3072;
// to delete the block locally.
pub(crate) const BLOCK_GC_DELAY: Duration = Duration::from_secs(600);
const BLOCK_READ_SEMAPHORE_TIMEOUT: Duration = Duration::from_secs(15);
/// RPC messages used to share blocks of data between nodes
#[derive(Debug, Serialize, Deserialize)]
pub enum BlockRpc {
@@ -85,6 +87,7 @@ pub struct BlockManager {
disable_scrub: bool,
mutation_lock: Vec<Mutex<BlockManagerLocked>>,
read_semaphore: Semaphore,
pub rc: BlockRc,
pub resync: BlockResyncManager,
@@ -174,6 +177,8 @@ impl BlockManager {
.iter()
.map(|_| Mutex::new(BlockManagerLocked()))
.collect::<Vec<_>>(),
read_semaphore: Semaphore::new(config.block_max_concurrent_reads),
rc,
resync,
system,
@@ -285,7 +290,7 @@ impl BlockManager {
let who = self
.system
.rpc_helper()
.block_read_nodes_of(hash, self.system.rpc_helper());
.block_read_nodes_of(hash, self.system.rpc_helper())?;
for node in who.iter() {
let node_id = NodeID::from(*node);
@@ -341,12 +346,9 @@ impl BlockManager {
/// layout version only: since blocks are immutable, we don't need to
/// do complex logic when several layour versions are active at once,
/// just move them directly to the new nodes.
pub(crate) fn storage_nodes_of(&self, hash: &Hash) -> Vec<Uuid> {
self.system
.cluster_layout()
.current()
.nodes_of(hash)
.collect()
pub(crate) fn storage_nodes_of(&self, hash: &Hash) -> Result<Vec<Uuid>, Error> {
let cluster_layout = self.system.cluster_layout();
Ok(cluster_layout.current()?.nodes_of(hash).collect())
}
// ---- Public interface ----
@@ -381,7 +383,7 @@ impl BlockManager {
prevent_compression: bool,
order_tag: Option<OrderTag>,
) -> Result<(), Error> {
let who = self.storage_nodes_of(&hash);
let who = self.storage_nodes_of(&hash)?;
let compression_level = self.compression_level.filter(|_| !prevent_compression);
let (header, bytes) = DataBlock::from_buffer(data, compression_level)
@@ -419,8 +421,8 @@ impl BlockManager {
}
/// Get number of items in the refcount table
pub fn rc_len(&self) -> Result<usize, Error> {
Ok(self.rc.rc_table.len()?)
pub fn rc_approximate_len(&self) -> Result<usize, Error> {
Ok(self.rc.rc_table.approximate_len()?)
}
/// Send command to start/stop/manager scrub worker
@@ -438,7 +440,7 @@ impl BlockManager {
/// List all resync errors
pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> {
let mut blocks = Vec::with_capacity(self.resync.errors.len()?);
let mut blocks = Vec::with_capacity(self.resync.errors.approximate_len()?);
for ent in self.resync.errors.iter()? {
let (hash, cnt) = ent?;
let cnt = ErrorCounter::decode(&cnt);
@@ -568,9 +570,6 @@ impl BlockManager {
match self.find_block(hash).await {
Some(p) => self.read_block_from(hash, &p).await,
None => {
// Not found but maybe we should have had it ??
self.resync
.put_to_resync(hash, 2 * self.system.rpc_helper().rpc_timeout())?;
return Err(Error::Message(format!(
"block {:?} not found on node",
hash
@@ -592,6 +591,15 @@ impl BlockManager {
) -> Result<DataBlock, Error> {
let (header, path) = block_path.as_parts_ref();
let permit = tokio::select! {
sem = self.read_semaphore.acquire() => sem.ok_or_message("acquire read semaphore")?,
_ = tokio::time::sleep(BLOCK_READ_SEMAPHORE_TIMEOUT) => {
self.metrics.block_read_semaphore_timeouts.add(1);
debug!("read block {:?}: read_semaphore acquire timeout", hash);
return Err(Error::Message("read block: read_semaphore acquire timeout".into()));
}
};
let mut f = fs::File::open(&path).await?;
let mut data = vec![];
f.read_to_end(&mut data).await?;
@@ -616,6 +624,8 @@ impl BlockManager {
return Err(Error::CorruptData(*hash));
}
drop(permit);
Ok(data)
}
+9 -3
View File
@@ -22,6 +22,7 @@ pub struct BlockManagerMetrics {
pub(crate) bytes_read: BoundCounter<u64>,
pub(crate) block_read_duration: BoundValueRecorder<f64>,
pub(crate) block_read_semaphore_timeouts: BoundCounter<u64>,
pub(crate) bytes_written: BoundCounter<u64>,
pub(crate) block_write_duration: BoundValueRecorder<f64>,
pub(crate) delete_counter: BoundCounter<u64>,
@@ -50,7 +51,7 @@ impl BlockManagerMetrics {
.init(),
_rc_size: meter
.u64_value_observer("block.rc_size", move |observer| {
if let Ok(value) = rc_tree.len() {
if let Ok(value) = rc_tree.approximate_len() {
observer.observe(value as u64, &[])
}
})
@@ -58,7 +59,7 @@ impl BlockManagerMetrics {
.init(),
_resync_queue_len: meter
.u64_value_observer("block.resync_queue_length", move |observer| {
if let Ok(value) = resync_queue.len() {
if let Ok(value) = resync_queue.approximate_len() {
observer.observe(value as u64, &[]);
}
})
@@ -68,7 +69,7 @@ impl BlockManagerMetrics {
.init(),
_resync_errored_blocks: meter
.u64_value_observer("block.resync_errored_blocks", move |observer| {
if let Ok(value) = resync_errors.len() {
if let Ok(value) = resync_errors.approximate_len() {
observer.observe(value as u64, &[]);
}
})
@@ -119,6 +120,11 @@ impl BlockManagerMetrics {
.with_description("Duration of block read operations")
.init()
.bind(&[]),
block_read_semaphore_timeouts: meter
.u64_counter("block.read_semaphore_timeouts")
.with_description("Number of block reads that failed due to semaphore acquire timeout")
.init()
.bind(&[]),
bytes_written: meter
.u64_counter("block.bytes_written")
.with_description("Number of bytes written to disk")
+18 -8
View File
@@ -104,13 +104,13 @@ impl BlockResyncManager {
}
/// Get length of resync queue
pub fn queue_len(&self) -> Result<usize, Error> {
Ok(self.queue.len()?)
pub fn queue_approximate_len(&self) -> Result<usize, Error> {
Ok(self.queue.approximate_len()?)
}
/// Get number of blocks that have an error
pub fn errors_len(&self) -> Result<usize, Error> {
Ok(self.errors.len()?)
pub fn errors_approximate_len(&self) -> Result<usize, Error> {
Ok(self.errors.approximate_len()?)
}
/// Clear the error counter for a block and put it in queue immediately
@@ -131,6 +131,14 @@ impl BlockResyncManager {
)))
}
/// Clear the entire resync queue and list of errored blocks
/// Corresponds to `garage repair clear-resync-queue`
pub fn clear_resync_queue(&self) -> Result<(), Error> {
self.queue.clear()?;
self.errors.clear()?;
Ok(())
}
pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
let notify = self.notify.clone();
vars.register_rw(
@@ -375,7 +383,7 @@ impl BlockResyncManager {
info!("Resync block {:?}: offloading and deleting", hash);
let existing_path = existing_path.unwrap();
let mut who = manager.storage_nodes_of(hash);
let mut who = manager.storage_nodes_of(hash)?;
if who.len() < manager.write_quorum {
return Err(Error::Message("Not trying to offload block because we don't have a quorum of nodes to write to".to_string()));
}
@@ -458,7 +466,7 @@ impl BlockResyncManager {
// First, check whether we are still supposed to store that
// block in the latest cluster layout version.
let storage_nodes = manager.storage_nodes_of(&hash);
let storage_nodes = manager.storage_nodes_of(&hash)?;
if !storage_nodes.contains(&manager.system.id) {
info!(
@@ -540,9 +548,11 @@ impl Worker for ResyncWorker {
}
WorkerStatus {
queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64),
queue_length: Some(self.manager.resync.queue_approximate_len().unwrap_or(0) as u64),
tranquility: Some(tranquility),
persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64),
persistent_errors: Some(
self.manager.resync.errors_approximate_len().unwrap_or(0) as u64
),
..Default::default()
}
}
+6 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_db"
version = "2.0.0"
version = "2.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@@ -16,10 +16,14 @@ err-derive.workspace = true
tracing.workspace = true
heed = { workspace = true, optional = true }
rusqlite = { workspace = true, optional = true, features = ["backup"] }
r2d2 = { workspace = true, optional = true }
r2d2_sqlite = { workspace = true, optional = true }
fjall = { workspace = true, optional = true }
parking_lot = { workspace = true, optional = true }
[dev-dependencies]
mktemp.workspace = true
@@ -27,4 +31,5 @@ mktemp.workspace = true
default = [ "lmdb", "sqlite" ]
bundled-libs = [ "rusqlite?/bundled" ]
lmdb = [ "heed" ]
fjall = [ "dep:fjall", "dep:parking_lot" ]
sqlite = [ "rusqlite", "r2d2", "r2d2_sqlite" ]
+453
View File
@@ -0,0 +1,453 @@
use core::ops::Bound;
use std::path::PathBuf;
use std::sync::Arc;
use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard};
use fjall::{
PartitionCreateOptions, PersistMode, TransactionalKeyspace, TransactionalPartitionHandle,
WriteTransaction,
};
use crate::{
open::{Engine, OpenOpt},
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
TxResult, TxValueIter, Value, ValueIter,
};
pub use fjall;
// --
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
info!("Opening Fjall database at: {}", path.display());
if opt.fsync {
return Err(Error(
"metadata_fsync is not supported with the Fjall database engine".into(),
));
}
let mut config = fjall::Config::new(path);
if let Some(block_cache_size) = opt.fjall_block_cache_size {
config = config.cache_size(block_cache_size as u64);
}
let keyspace = config.open_transactional()?;
Ok(FjallDb::init(keyspace))
}
// -- err
impl From<fjall::Error> for Error {
fn from(e: fjall::Error) -> Error {
Error(format!("fjall: {}", e).into())
}
}
impl From<fjall::LsmError> for Error {
fn from(e: fjall::LsmError) -> Error {
Error(format!("fjall lsm_tree: {}", e).into())
}
}
impl From<fjall::Error> for TxOpError {
fn from(e: fjall::Error) -> TxOpError {
TxOpError(e.into())
}
}
// -- db
pub struct FjallDb {
keyspace: TransactionalKeyspace,
trees: RwLock<Vec<(String, TransactionalPartitionHandle)>>,
}
type ByteRefRangeBound<'r> = (Bound<&'r [u8]>, Bound<&'r [u8]>);
impl FjallDb {
pub fn init(keyspace: TransactionalKeyspace) -> Db {
let s = Self {
keyspace,
trees: RwLock::new(Vec::new()),
};
Db(Arc::new(s))
}
fn get_tree(
&self,
i: usize,
) -> Result<MappedRwLockReadGuard<'_, TransactionalPartitionHandle>> {
RwLockReadGuard::try_map(self.trees.read(), |trees: &Vec<_>| {
trees.get(i).map(|tup| &tup.1)
})
.map_err(|_| Error("invalid tree id".into()))
}
}
impl IDb for FjallDb {
fn engine(&self) -> String {
"Fjall (EXPERIMENTAL!)".into()
}
fn open_tree(&self, name: &str) -> Result<usize> {
let mut trees = self.trees.write();
let safe_name = encode_name(name)?;
if let Some(i) = trees.iter().position(|(name, _)| *name == safe_name) {
Ok(i)
} else {
let tree = self
.keyspace
.open_partition(&safe_name, PartitionCreateOptions::default())?;
let i = trees.len();
trees.push((safe_name, tree));
Ok(i)
}
}
fn list_trees(&self) -> Result<Vec<String>> {
Ok(self
.keyspace
.list_partitions()
.iter()
.map(|n| decode_name(&n))
.collect::<Result<Vec<_>>>()?)
}
fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
std::fs::create_dir_all(base_path)?;
let path = Engine::Fjall.db_path(base_path);
let source_state = self.keyspace.read_tx();
let copy_keyspace = fjall::Config::new(path).open()?;
for partition_name in self.keyspace.list_partitions() {
let source_partition = self
.keyspace
.open_partition(&partition_name, PartitionCreateOptions::default())?;
let copy_partition =
copy_keyspace.open_partition(&partition_name, PartitionCreateOptions::default())?;
for entry in source_state.iter(&source_partition) {
let (key, value) = entry?;
copy_partition.insert(key, value)?;
}
}
copy_keyspace.persist(PersistMode::SyncAll)?;
Ok(())
}
// ----
fn get(&self, tree_idx: usize, key: &[u8]) -> Result<Option<Value>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
let val = tx.get(&tree, key)?;
match val {
None => Ok(None),
Some(v) => Ok(Some(v.to_vec())),
}
}
fn approximate_len(&self, tree_idx: usize) -> Result<usize> {
let tree = self.get_tree(tree_idx)?;
Ok(tree.approximate_len())
}
fn is_empty(&self, tree_idx: usize) -> Result<bool> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(tx.is_empty(&tree)?)
}
fn insert(&self, tree_idx: usize, key: &[u8], value: &[u8]) -> Result<()> {
let tree = self.get_tree(tree_idx)?;
let mut tx = self.keyspace.write_tx();
tx.insert(&tree, key, value);
tx.commit()?;
Ok(())
}
fn remove(&self, tree_idx: usize, key: &[u8]) -> Result<()> {
let tree = self.get_tree(tree_idx)?;
let mut tx = self.keyspace.write_tx();
tx.remove(&tree, key);
tx.commit()?;
Ok(())
}
fn clear(&self, tree_idx: usize) -> Result<()> {
let mut trees = self.trees.write();
if tree_idx >= trees.len() {
return Err(Error("invalid tree id".into()));
}
let (name, tree) = trees.remove(tree_idx);
self.keyspace.delete_partition(tree)?;
let tree = self
.keyspace
.open_partition(&name, PartitionCreateOptions::default())?;
trees.insert(tree_idx, (name, tree));
Ok(())
}
fn iter(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(tx.iter(&tree).map(iterator_remap)))
}
fn iter_rev(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(tx.iter(&tree).rev().map(iterator_remap)))
}
fn range<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(
tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high))
.map(iterator_remap),
))
}
fn range_rev<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(
tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high))
.rev()
.map(iterator_remap),
))
}
// ----
fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()> {
let trees = self.trees.read();
let mut tx = FjallTx {
trees: &trees[..],
tx: self.keyspace.write_tx(),
};
let res = f.try_on(&mut tx);
match res {
TxFnResult::Ok(on_commit) => {
tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?;
Ok(on_commit)
}
TxFnResult::Abort => {
tx.tx.rollback();
Err(TxError::Abort(()))
}
TxFnResult::DbErr => {
tx.tx.rollback();
Err(TxError::Db(Error(
"(this message will be discarded)".into(),
)))
}
}
}
}
// ----
struct FjallTx<'a> {
trees: &'a [(String, TransactionalPartitionHandle)],
tx: WriteTransaction<'a>,
}
impl<'a> FjallTx<'a> {
fn get_tree(&self, i: usize) -> TxOpResult<&TransactionalPartitionHandle> {
self.trees.get(i).map(|tup| &tup.1).ok_or_else(|| {
TxOpError(Error(
"invalid tree id (it might have been openned after the transaction started)".into(),
))
})
}
}
impl<'a> ITx for FjallTx<'a> {
fn get(&self, tree_idx: usize, key: &[u8]) -> TxOpResult<Option<Value>> {
let tree = self.get_tree(tree_idx)?;
match self.tx.get(tree, key)? {
Some(v) => Ok(Some(v.to_vec())),
None => Ok(None),
}
}
fn len(&self, tree_idx: usize) -> TxOpResult<usize> {
let tree = self.get_tree(tree_idx)?;
Ok(self.tx.len(tree)? as usize)
}
fn insert(&mut self, tree_idx: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> {
let tree = self.get_tree(tree_idx)?.clone();
self.tx.insert(&tree, key, value);
Ok(())
}
fn remove(&mut self, tree_idx: usize, key: &[u8]) -> TxOpResult<()> {
let tree = self.get_tree(tree_idx)?.clone();
self.tx.remove(&tree, key);
Ok(())
}
fn clear(&mut self, _tree_idx: usize) -> TxOpResult<()> {
unimplemented!("LSM tree clearing in cross-partition transaction is not supported")
}
fn iter(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?.clone();
Ok(Box::new(self.tx.iter(&tree).map(iterator_remap_tx)))
}
fn iter_rev(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?.clone();
Ok(Box::new(self.tx.iter(&tree).rev().map(iterator_remap_tx)))
}
fn range<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let low = clone_bound(low);
let high = clone_bound(high);
Ok(Box::new(
self.tx
.range::<Vec<u8>, ByteVecRangeBounds>(&tree, (low, high))
.map(iterator_remap_tx),
))
}
fn range_rev<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let low = clone_bound(low);
let high = clone_bound(high);
Ok(Box::new(
self.tx
.range::<Vec<u8>, ByteVecRangeBounds>(&tree, (low, high))
.rev()
.map(iterator_remap_tx),
))
}
}
// -- maps fjall's (k, v) to ours
fn iterator_remap(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> Result<(Value, Value)> {
r.map(|(k, v)| (k.to_vec(), v.to_vec()))
.map_err(|e| e.into())
}
fn iterator_remap_tx(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> TxOpResult<(Value, Value)> {
r.map(|(k, v)| (k.to_vec(), v.to_vec()))
.map_err(|e| e.into())
}
// -- utils to deal with Garage's tightness on Bound lifetimes
type ByteVecBound = Bound<Vec<u8>>;
type ByteVecRangeBounds = (ByteVecBound, ByteVecBound);
fn clone_bound(bound: Bound<&[u8]>) -> ByteVecBound {
let value = match bound {
Bound::Excluded(v) | Bound::Included(v) => v.to_vec(),
Bound::Unbounded => vec![],
};
match bound {
Bound::Included(_) => Bound::Included(value),
Bound::Excluded(_) => Bound::Excluded(value),
Bound::Unbounded => Bound::Unbounded,
}
}
// -- utils to encode table names --
fn encode_name(s: &str) -> Result<String> {
let base = 'A' as u32;
let mut ret = String::with_capacity(s.len() + 10);
for c in s.chars() {
if c.is_alphanumeric() || c == '_' || c == '-' || c == '#' {
ret.push(c);
} else if c <= u8::MAX as char {
ret.push('$');
let c_hi = c as u32 / 16;
let c_lo = c as u32 % 16;
ret.push(char::from_u32(base + c_hi).unwrap());
ret.push(char::from_u32(base + c_lo).unwrap());
} else {
return Err(Error(
format!("table name {} could not be safely encoded", s).into(),
));
}
}
Ok(ret)
}
fn decode_name(s: &str) -> Result<String> {
use std::convert::TryFrom;
let errfn = || Error(format!("encoded table name {} is invalid", s).into());
let c_map = |c: char| {
let c = c as u32;
let base = 'A' as u32;
if (base..base + 16).contains(&c) {
Some(c - base)
} else {
None
}
};
let mut ret = String::with_capacity(s.len());
let mut it = s.chars();
while let Some(c) = it.next() {
if c == '$' {
let c_hi = it.next().and_then(c_map).ok_or_else(errfn)?;
let c_lo = it.next().and_then(c_map).ok_or_else(errfn)?;
let c_dec = char::try_from(c_hi * 16 + c_lo).map_err(|_| errfn())?;
ret.push(c_dec);
} else {
ret.push(c);
}
}
Ok(ret)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_encdec_name() {
for name in [
"testname",
"test_name",
"test name",
"test$name",
"test:name@help.me$get/this**right",
] {
let encname = encode_name(name).unwrap();
assert!(!encname.contains(' '));
assert!(!encname.contains('.'));
assert!(!encname.contains('*'));
assert_eq!(*name, decode_name(&encname).unwrap());
}
}
}
+48 -29
View File
@@ -1,6 +1,8 @@
#[macro_use]
extern crate tracing;
#[cfg(feature = "fjall")]
pub mod fjall_adapter;
#[cfg(feature = "lmdb")]
pub mod lmdb_adapter;
#[cfg(feature = "sqlite")]
@@ -104,32 +106,44 @@ impl Db {
result: Cell::new(None),
};
let tx_res = self.0.transaction(&f);
let ret = f
.result
.into_inner()
.expect("Transaction did not store result");
let fn_res = f.result.into_inner();
match tx_res {
Ok(on_commit) => match ret {
Ok(value) => {
on_commit.into_iter().for_each(|f| f());
Ok(value)
}
_ => unreachable!(),
},
Err(TxError::Abort(())) => match ret {
Err(TxError::Abort(e)) => Err(TxError::Abort(e)),
_ => unreachable!(),
},
Err(TxError::Db(e2)) => match ret {
// Ok was stored -> the error occurred when finalizing
// transaction
Ok(_) => Err(TxError::Db(e2)),
// An error was already stored: that's the one we want to
// return
Err(TxError::Db(e)) => Err(TxError::Db(e)),
_ => unreachable!(),
},
match (tx_res, fn_res) {
(Ok(on_commit), Some(Ok(value))) => {
// Transaction succeeded
// TxFn stored the value to return to the user in fn_res
// tx_res contains the on_commit list of callbacks, run them now
on_commit.into_iter().for_each(|f| f());
Ok(value)
}
(Err(TxError::Abort(())), Some(Err(TxError::Abort(e)))) => {
// Transaction was aborted by user code
// The abort error value is stored in fn_res
Err(TxError::Abort(e))
}
(Err(TxError::Db(_tx_e)), Some(Err(TxError::Db(fn_e)))) => {
// Transaction encountered a DB error in user code
// The error value encountered is the one in fn_res,
// tx_res contains only a dummy error message
Err(TxError::Db(fn_e))
}
(Err(TxError::Db(tx_e)), None) => {
// Transaction encounterred a DB error when initializing the transaction,
// before user code was called
Err(TxError::Db(tx_e))
}
(Err(TxError::Db(tx_e)), Some(Ok(_))) => {
// Transaction encounterred a DB error when commiting the transaction,
// after user code was called
Err(TxError::Db(tx_e))
}
(tx_res, fn_res) => {
panic!(
"unexpected error case: tx_res={:?}, fn_res={:?}",
tx_res.map(|_| "..."),
fn_res.map(|x| x.map(|_| "...").map_err(|_| "..."))
);
}
}
}
@@ -152,7 +166,7 @@ impl Db {
let tree_names = other.list_trees()?;
for name in tree_names {
let tree = self.open_tree(&name)?;
if tree.len()? > 0 {
if !tree.is_empty()? {
return Err(Error(format!("tree {} already contains data", name).into()));
}
@@ -194,8 +208,12 @@ impl Tree {
self.0.get(self.1, key.as_ref())
}
#[inline]
pub fn len(&self) -> Result<usize> {
self.0.len(self.1)
pub fn approximate_len(&self) -> Result<usize> {
self.0.approximate_len(self.1)
}
#[inline]
pub fn is_empty(&self) -> Result<bool> {
self.0.is_empty(self.1)
}
#[inline]
@@ -333,7 +351,8 @@ pub(crate) trait IDb: Send + Sync {
fn snapshot(&self, path: &PathBuf) -> Result<()>;
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
fn len(&self, tree: usize) -> Result<usize>;
fn approximate_len(&self, tree: usize) -> Result<usize>;
fn is_empty(&self, tree: usize) -> Result<bool>;
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>;
fn remove(&self, tree: usize, key: &[u8]) -> Result<()>;
+94 -29
View File
@@ -1,8 +1,8 @@
use core::ops::Bound;
use core::ptr::NonNull;
use std::collections::HashMap;
use std::convert::TryInto;
use std::marker::PhantomPinned;
use std::path::PathBuf;
use std::pin::Pin;
use std::sync::{Arc, RwLock};
@@ -11,12 +11,55 @@ use heed::types::ByteSlice;
use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database};
use crate::{
open::{Engine, OpenOpt},
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
TxResult, TxValueIter, Value, ValueIter,
};
pub use heed;
// ---- top-level open function
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
info!("Opening LMDB database at: {}", path.display());
if let Err(e) = std::fs::create_dir_all(&path) {
return Err(Error(
format!("Unable to create LMDB data directory: {}", e).into(),
));
}
let map_size = match opt.lmdb_map_size {
None => recommended_map_size(),
Some(v) => v - (v % 4096),
};
let mut env_builder = heed::EnvOpenOptions::new();
env_builder.max_dbs(100);
env_builder.map_size(map_size);
env_builder.max_readers(2048);
unsafe {
env_builder.flag(heed::flags::Flags::MdbNoRdAhead);
env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
if !opt.fsync {
env_builder.flag(heed::flags::Flags::MdbNoSync);
}
}
match env_builder.open(&path) {
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
return Err(Error(
"OutOfMemory error while trying to open LMDB database. This can happen \
if your operating system is not allowing you to use sufficient virtual \
memory address space. Please check that no limit is set (ulimit -v). \
You may also try to set a smaller `lmdb_map_size` configuration parameter. \
On 32-bit machines, you should probably switch to another database engine."
.into(),
))
}
Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
Ok(db) => Ok(LmdbDb::init(db)),
}
}
// -- err
impl From<heed::Error> for Error {
@@ -104,10 +147,9 @@ impl IDb for LmdbDb {
Ok(ret2)
}
fn snapshot(&self, to: &PathBuf) -> Result<()> {
std::fs::create_dir_all(to)?;
let mut path = to.clone();
path.push("data.mdb");
fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
std::fs::create_dir_all(base_path)?;
let path = Engine::Lmdb.db_path(base_path);
self.db
.copy_to_path(path, heed::CompactionOption::Enabled)?;
Ok(())
@@ -126,11 +168,16 @@ impl IDb for LmdbDb {
}
}
fn len(&self, tree: usize) -> Result<usize> {
fn approximate_len(&self, tree: usize) -> Result<usize> {
let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?;
Ok(tree.len(&tx)?.try_into().unwrap())
}
fn is_empty(&self, tree: usize) -> Result<bool> {
let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?;
Ok(tree.is_empty(&tx)?)
}
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
let tree = self.get_tree(tree)?;
@@ -159,13 +206,15 @@ impl IDb for LmdbDb {
fn iter(&self, tree: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?;
TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?))
// Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) }
}
fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?;
TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?))
// Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?)) }
}
fn range<'r>(
@@ -176,7 +225,8 @@ impl IDb for LmdbDb {
) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?;
TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?))
// Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?)) }
}
fn range_rev<'r>(
&self,
@@ -186,7 +236,8 @@ impl IDb for LmdbDb {
) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?;
TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?))
// Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?)) }
}
// ----
@@ -316,28 +367,41 @@ where
{
tx: RoTxn<'a>,
iter: Option<I>,
_pin: PhantomPinned,
}
impl<'a, I> TxAndIterator<'a, I>
where
I: Iterator<Item = IteratorItem<'a>> + 'a,
{
fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>>
fn iter(self: Pin<&mut Self>) -> &mut Option<I> {
// Safety: iter is not structural
unsafe { &mut self.get_unchecked_mut().iter }
}
/// Safety: iterfun must not store its argument anywhere but in its result.
unsafe fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>>
where
F: FnOnce(&'a RoTxn<'a>) -> Result<I>,
{
let res = TxAndIterator { tx, iter: None };
let res = TxAndIterator {
tx,
iter: None,
_pin: PhantomPinned,
};
let mut boxed = Box::pin(res);
// This unsafe allows us to bypass lifetime checks
let tx = unsafe { NonNull::from(&boxed.tx).as_ref() };
let iter = iterfun(tx)?;
let tx_lifetime_overextended: &'a RoTxn<'a> = {
let tx = &boxed.tx;
// Safety: Artificially extending the lifetime because
// this reference will only be stored and accessed from the
// returned ValueIter which guarantees that it is destroyed
// before the tx it is pointing to.
unsafe { &*&raw const *tx }
};
let iter = iterfun(&tx_lifetime_overextended)?;
let mut_ref = Pin::as_mut(&mut boxed);
// This unsafe allows us to write in a field of the pinned struct
unsafe {
Pin::get_unchecked_mut(mut_ref).iter = Some(iter);
}
*boxed.as_mut().iter() = Some(iter);
Ok(Box::new(TxAndIteratorPin(boxed)))
}
@@ -348,8 +412,10 @@ where
I: Iterator<Item = IteratorItem<'a>> + 'a,
{
fn drop(&mut self) {
// ensure the iterator is dropped before the RoTxn it references
drop(self.iter.take());
// Safety: `new_unchecked` is okay because we know this value is never
// used again after being dropped.
let this = unsafe { Pin::new_unchecked(self) };
drop(this.iter().take());
}
}
@@ -365,13 +431,12 @@ where
fn next(&mut self) -> Option<Self::Item> {
let mut_ref = Pin::as_mut(&mut self.0);
// This unsafe allows us to mutably access the iterator field
let next = unsafe { Pin::get_unchecked_mut(mut_ref).iter.as_mut()?.next() };
match next {
None => None,
Some(Err(e)) => Some(Err(e.into())),
Some(Ok((k, v))) => Some(Ok((k.to_vec(), v.to_vec()))),
}
let next = mut_ref.iter().as_mut()?.next()?;
let res = match next {
Err(e) => Err(e.into()),
Ok((k, v)) => Ok((k.to_vec(), v.to_vec())),
};
Some(res)
}
}
+28 -44
View File
@@ -11,6 +11,7 @@ use crate::{Db, Error, Result};
pub enum Engine {
Lmdb,
Sqlite,
Fjall,
}
impl Engine {
@@ -19,8 +20,26 @@ impl Engine {
match self {
Self::Lmdb => "lmdb",
Self::Sqlite => "sqlite",
Self::Fjall => "fjall",
}
}
/// Return engine-specific DB path from base path
pub fn db_path(&self, base_path: &PathBuf) -> PathBuf {
let mut ret = base_path.clone();
match self {
Self::Lmdb => {
ret.push("db.lmdb");
}
Self::Sqlite => {
ret.push("db.sqlite");
}
Self::Fjall => {
ret.push("db.fjall");
}
}
ret
}
}
impl std::fmt::Display for Engine {
@@ -36,10 +55,11 @@ impl std::str::FromStr for Engine {
match text {
"lmdb" | "heed" => Ok(Self::Lmdb),
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
"fjall" => Ok(Self::Fjall),
"sled" => Err(Error("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into())),
kind => Err(Error(
format!(
"Invalid DB engine: {} (options are: lmdb, sqlite)",
"Invalid DB engine: {} (options are: lmdb, sqlite, fjall)",
kind
)
.into(),
@@ -51,6 +71,7 @@ impl std::str::FromStr for Engine {
pub struct OpenOpt {
pub fsync: bool,
pub lmdb_map_size: Option<usize>,
pub fjall_block_cache_size: Option<usize>,
}
impl Default for OpenOpt {
@@ -58,6 +79,7 @@ impl Default for OpenOpt {
Self {
fsync: false,
lmdb_map_size: None,
fjall_block_cache_size: None,
}
}
}
@@ -66,53 +88,15 @@ pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
match engine {
// ---- Sqlite DB ----
#[cfg(feature = "sqlite")]
Engine::Sqlite => {
info!("Opening Sqlite database at: {}", path.display());
let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
Ok(crate::sqlite_adapter::SqliteDb::new(manager, opt.fsync)?)
}
Engine::Sqlite => crate::sqlite_adapter::open_db(path, opt),
// ---- LMDB DB ----
#[cfg(feature = "lmdb")]
Engine::Lmdb => {
info!("Opening LMDB database at: {}", path.display());
if let Err(e) = std::fs::create_dir_all(&path) {
return Err(Error(
format!("Unable to create LMDB data directory: {}", e).into(),
));
}
Engine::Lmdb => crate::lmdb_adapter::open_db(path, opt),
let map_size = match opt.lmdb_map_size {
None => crate::lmdb_adapter::recommended_map_size(),
Some(v) => v - (v % 4096),
};
let mut env_builder = heed::EnvOpenOptions::new();
env_builder.max_dbs(100);
env_builder.map_size(map_size);
env_builder.max_readers(2048);
unsafe {
env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoRdAhead);
env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
if !opt.fsync {
env_builder.flag(heed::flags::Flags::MdbNoSync);
}
}
match env_builder.open(&path) {
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
return Err(Error(
"OutOfMemory error while trying to open LMDB database. This can happen \
if your operating system is not allowing you to use sufficient virtual \
memory address space. Please check that no limit is set (ulimit -v). \
You may also try to set a smaller `lmdb_map_size` configuration parameter. \
On 32-bit machines, you should probably switch to another database engine."
.into(),
))
}
Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
Ok(db) => Ok(crate::lmdb_adapter::LmdbDb::init(db)),
}
}
// ---- Fjall DB ----
#[cfg(feature = "fjall")]
Engine::Fjall => crate::fjall_adapter::open_db(path, opt),
// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
// attribute is added so that we won't have to change this match in case stop building
+28 -12
View File
@@ -11,12 +11,23 @@ use r2d2_sqlite::SqliteConnectionManager;
use rusqlite::{params, Rows, Statement, Transaction};
use crate::{
open::{Engine, OpenOpt},
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
TxResult, TxValueIter, Value, ValueIter,
};
pub use rusqlite;
// ---- top-level open function
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
info!("Opening Sqlite database at: {}", path.display());
let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
Ok(SqliteDb::new(manager, opt.fsync)?)
}
// ----
type Connection = r2d2::PooledConnection<SqliteConnectionManager>;
// --- err
@@ -139,17 +150,18 @@ impl IDb for SqliteDb {
Ok(trees)
}
fn snapshot(&self, to: &PathBuf) -> Result<()> {
fn progress(p: rusqlite::backup::Progress) {
let percent = (p.pagecount - p.remaining) * 100 / p.pagecount;
info!("Sqlite snapshot progress: {}%", percent);
}
std::fs::create_dir_all(to)?;
let mut path = to.clone();
path.push("db.sqlite");
self.db
.get()?
.backup(rusqlite::DatabaseName::Main, path, Some(progress))?;
fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
std::fs::create_dir_all(base_path)?;
let path = Engine::Sqlite
.db_path(&base_path)
.into_os_string()
.into_string()
.map_err(|_| Error("invalid sqlite path string".into()))?;
info!("Start sqlite VACUUM INTO `{}`", path);
self.db.get()?.execute("VACUUM INTO ?1", params![path])?;
info!("Finished sqlite VACUUM INTO `{}`", path);
Ok(())
}
@@ -160,7 +172,7 @@ impl IDb for SqliteDb {
self.internal_get(&self.db.get()?, &tree, key)
}
fn len(&self, tree: usize) -> Result<usize> {
fn approximate_len(&self, tree: usize) -> Result<usize> {
let tree = self.get_tree(tree)?;
let db = self.db.get()?;
@@ -172,6 +184,10 @@ impl IDb for SqliteDb {
}
}
fn is_empty(&self, tree: usize) -> Result<bool> {
Ok(self.approximate_len(tree)? == 0)
}
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
let tree = self.get_tree(tree)?;
let db = self.db.get()?;
+14 -2
View File
@@ -1,7 +1,7 @@
use crate::*;
fn test_suite(db: Db) {
let tree = db.open_tree("tree").unwrap();
let tree = db.open_tree("tree:this_is_a_tree").unwrap();
let ka: &[u8] = &b"test"[..];
let kb: &[u8] = &b"zwello"[..];
@@ -14,7 +14,7 @@ fn test_suite(db: Db) {
assert!(tree.insert(ka, va).is_ok());
assert_eq!(tree.get(ka).unwrap().unwrap(), va);
assert_eq!(tree.len().unwrap(), 1);
assert_eq!(tree.iter().unwrap().count(), 1);
// ---- test transaction logic ----
@@ -148,3 +148,15 @@ fn test_sqlite_db() {
let db = SqliteDb::new(manager, false).unwrap();
test_suite(db);
}
#[test]
#[cfg(feature = "fjall")]
fn test_fjall_db() {
use crate::fjall_adapter::{fjall, FjallDb};
let path = mktemp::Temp::new_dir().unwrap();
let config = fjall::Config::new(path).temporary(true);
let keyspace = config.open_transactional().unwrap();
let db = FjallDb::init(keyspace);
test_suite(db);
}
+2 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage"
version = "2.0.0"
version = "2.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@@ -93,6 +93,7 @@ k2v = [ "garage_util/k2v", "garage_api_k2v", "garage_api_admin/k2v" ]
# Database engines
lmdb = [ "garage_model/lmdb" ]
sqlite = [ "garage_model/sqlite" ]
fjall = [ "garage_model/fjall" ]
# Automatic registration and discovery via Consul API
consul-discovery = [ "garage_rpc/consul-discovery" ]
+1 -1
View File
@@ -100,7 +100,7 @@ impl Cli {
drain_msg = true;
"draining metadata.."
}
_ => unreachable!(),
_ => continue,
};
failed_nodes.push(format!(
+1
View File
@@ -92,6 +92,7 @@ impl Cli {
ScrubCmd::Resume => ScrubCommand::Resume,
}),
RepairWhat::Aliases => RepairType::Aliases,
RepairWhat::ClearResyncQueue => RepairType::ClearResyncQueue,
};
let res = self
+4
View File
@@ -645,6 +645,10 @@ pub enum RepairWhat {
/// Repair (resync/rebalance) the set of stored blocks in the cluster
#[structopt(name = "blocks", version = garage_version())]
Blocks,
/// Clear the block resync queue. The list of blocks in errored state
/// is cleared as well. You MUST run `garage repair blocks` after invoking this.
#[structopt(name = "clear-resync-queue", version = garage_version())]
ClearResyncQueue,
/// Repropagate object deletions to the version table
#[structopt(name = "versions", version = garage_version())]
Versions,
+15 -4
View File
@@ -180,10 +180,21 @@ fn watch_shutdown_signal() -> watch::Receiver<bool> {
let mut sigterm =
signal(SignalKind::terminate()).expect("Failed to install SIGTERM handler");
let mut sighup = signal(SignalKind::hangup()).expect("Failed to install SIGHUP handler");
tokio::select! {
_ = sigint.recv() => info!("Received SIGINT, shutting down."),
_ = sigterm.recv() => info!("Received SIGTERM, shutting down."),
_ = sighup.recv() => info!("Received SIGHUP, shutting down."),
loop {
tokio::select! {
_ = sigint.recv() => {
info!("Received SIGINT, shutting down.");
break
}
_ = sigterm.recv() => {
info!("Received SIGTERM, shutting down.");
break
}
_ = sighup.recv() => {
info!("Received SIGHUP, reload not supported.");
continue
}
}
}
send_cancel.send(true).unwrap();
});
+10
View File
@@ -72,6 +72,16 @@ impl K2vClient {
.enable_http2()
.build();
let client = HttpClient::builder(TokioExecutor::new()).build(connector);
Self::new_with_client(config, client)
}
/// Create a new K2V client with an external client.
/// Useful for example if you plan on creating many clients but you want to mutualize the
/// underlying thread pools & co.
pub fn new_with_client(
config: K2vClientConfig,
client: HttpClient<HttpsConnector<HttpConnector>, Body>,
) -> Result<Self, Error> {
let user_agent: std::borrow::Cow<str> = match &config.user_agent {
Some(ua) => ua.into(),
None => format!("k2v/{}", env!("CARGO_PKG_VERSION")).into(),
+2 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_model"
version = "2.0.0"
version = "2.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@@ -45,3 +45,4 @@ default = [ "lmdb", "sqlite" ]
k2v = [ "garage_util/k2v" ]
lmdb = [ "garage_db/lmdb" ]
sqlite = [ "garage_db/sqlite" ]
fjall = [ "garage_db/fjall" ]
+6 -9
View File
@@ -119,21 +119,17 @@ impl Garage {
info!("Opening database...");
let db_engine = db::Engine::from_str(&config.db_engine)
.ok_or_message("Invalid `db_engine` value in configuration file")?;
let mut db_path = config.metadata_dir.clone();
match db_engine {
db::Engine::Sqlite => {
db_path.push("db.sqlite");
}
db::Engine::Lmdb => {
db_path.push("db.lmdb");
}
}
let db_path = db_engine.db_path(&config.metadata_dir);
let db_opt = db::OpenOpt {
fsync: config.metadata_fsync,
lmdb_map_size: match config.lmdb_map_size {
v if v == usize::default() => None,
v => Some(v),
},
fjall_block_cache_size: match config.fjall_block_cache_size {
v if v == usize::default() => None,
v => Some(v),
},
};
let db = db::open_db(&db_path, db_engine, &db_opt)
.ok_or_message("Unable to open metadata db")?;
@@ -161,6 +157,7 @@ impl Garage {
let control_rep_param = TableFullReplication {
system: system.clone(),
consistency_mode,
};
info!("Initialize block manager...");
+1 -1
View File
@@ -227,7 +227,7 @@ impl<'a> BucketHelper<'a> {
.0
.system
.cluster_layout()
.all_nongateway_nodes()
.all_nongateway_nodes()?
.to_vec();
let k2vindexes = self
.0
+4 -5
View File
@@ -84,17 +84,16 @@ impl<T: CountedItem> Entry<T::CP, T::CS> for CounterEntry<T> {
impl<T: CountedItem> CounterEntry<T> {
pub fn filtered_values(&self, layout: &LayoutHelper) -> HashMap<String, i64> {
let nodes = layout.all_nongateway_nodes();
self.filtered_values_with_nodes(&nodes)
self.filtered_values_internal(layout.all_nongateway_nodes().ok())
}
pub fn filtered_values_with_nodes(&self, nodes: &[Uuid]) -> HashMap<String, i64> {
fn filtered_values_internal(&self, nodes_opt: Option<&[Uuid]>) -> HashMap<String, i64> {
let mut ret = HashMap::new();
for (name, vals) in self.values.iter() {
let new_vals = vals
.node_values
.iter()
.filter(|(n, _)| nodes.contains(n))
.filter(|(n, _)| nodes_opt.map(|nodes| nodes.contains(n)).unwrap_or(true))
.map(|(_, (_, v))| *v)
.collect::<Vec<_>>();
if !new_vals.is_empty() {
@@ -153,7 +152,7 @@ impl<T: CountedItem> TableSchema for CounterTable<T> {
}
let is_tombstone = entry
.filtered_values_with_nodes(&filter.1[..])
.filtered_values_internal(Some(&filter.1[..]))
.iter()
.all(|(_, v)| *v == 0);
filter.0.apply(is_tombstone)
+6 -6
View File
@@ -126,7 +126,7 @@ impl K2VRpcHandler {
.item_table
.data
.replication
.storage_nodes(&partition.hash());
.storage_nodes(&partition.hash())?;
who.sort();
self.system
@@ -165,7 +165,7 @@ impl K2VRpcHandler {
.item_table
.data
.replication
.storage_nodes(&partition.hash());
.storage_nodes(&partition.hash())?;
who.sort();
call_list.entry(who).or_default().push(InsertedItem {
@@ -222,7 +222,7 @@ impl K2VRpcHandler {
.item_table
.data
.replication
.storage_nodes(&poll_key.partition.hash());
.storage_nodes(&poll_key.partition.hash())?;
let rpc = self.system.rpc_helper().try_call_many(
&self.endpoint,
@@ -233,7 +233,7 @@ impl K2VRpcHandler {
timeout_msec,
},
RequestStrategy::with_priority(PRIO_NORMAL)
.with_quorum(self.item_table.data.replication.read_quorum())
.with_quorum(self.item_table.data.replication.read_quorum()?)
.send_all_at_once(true)
.without_timeout(),
);
@@ -283,8 +283,8 @@ impl K2VRpcHandler {
.item_table
.data
.replication
.storage_nodes(&range.partition.hash());
let quorum = self.item_table.data.replication.read_quorum();
.storage_nodes(&range.partition.hash())?;
let quorum = self.item_table.data.replication.read_quorum()?;
let msg = K2VRpc::PollRange {
range,
seen_str,
+3 -3
View File
@@ -121,13 +121,13 @@ impl Worker for LifecycleWorker {
mpu_aborted,
..
} => {
let n_objects = self.garage.object_table.data.store.len().ok();
let n_objects = self.garage.object_table.data.store.approximate_len().ok();
let progress = match n_objects {
None => "...".to_string(),
Some(total) => format!(
Some(total) if total > 0 => format!(
"~{:.2}%",
100. * std::cmp::min(*counter, total) as f32 / total as f32
),
_ => "...".to_string(),
};
WorkerStatus {
progress: Some(progress),
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_net"
version = "2.0.0"
version = "2.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_rpc"
version = "2.0.0"
version = "2.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+32 -17
View File
@@ -4,6 +4,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use serde::{Deserialize, Serialize};
use garage_util::data::*;
use garage_util::error::Error;
use super::*;
use crate::replication_mode::*;
@@ -145,24 +146,31 @@ impl LayoutHelper {
}
/// Returns the current layout version
pub fn current(&self) -> &LayoutVersion {
self.inner().current()
pub fn current(&self) -> Result<&LayoutVersion, Error> {
if !self.is_check_ok {
return Err(Error::LayoutNotReady);
}
Ok(self.inner().current())
}
/// Returns all layout versions currently active in the cluster
pub fn versions(&self) -> &[LayoutVersion] {
&self.inner().versions
pub fn versions(&self) -> Result<&[LayoutVersion], Error> {
if !self.is_check_ok {
return Err(Error::LayoutNotReady);
}
Ok(&self.inner().versions)
}
/// Returns the latest layout version for which it is safe to read data from,
/// i.e. the version whose version number is sync_map_min
pub fn read_version(&self) -> &LayoutVersion {
pub fn read_version(&self) -> Result<&LayoutVersion, Error> {
let sync_min = self.sync_map_min;
self.versions()
let versions = self.versions()?;
Ok(versions
.iter()
.find(|x| x.version == sync_min)
.or(self.versions().last())
.unwrap()
.or(versions.last())
.unwrap())
}
pub fn is_check_ok(&self) -> bool {
@@ -171,14 +179,20 @@ impl LayoutHelper {
/// Return all nodes that have a role (gateway or storage)
/// in one of the currently active layout versions
pub fn all_nodes(&self) -> &[Uuid] {
&self.all_nodes
pub fn all_nodes(&self) -> Result<&[Uuid], Error> {
if !self.is_check_ok {
return Err(Error::LayoutNotReady);
}
Ok(&self.all_nodes)
}
/// Return all nodes that are configured to store data
/// in one of the currently active layout versions
pub fn all_nongateway_nodes(&self) -> &[Uuid] {
&self.all_nongateway_nodes
pub fn all_nongateway_nodes(&self) -> Result<&[Uuid], Error> {
if !self.is_check_ok {
return Err(Error::LayoutNotReady);
}
Ok(&self.all_nongateway_nodes)
}
pub fn ack_map_min(&self) -> u64 {
@@ -193,7 +207,7 @@ impl LayoutHelper {
pub fn sync_digest(&self) -> SyncLayoutDigest {
SyncLayoutDigest {
current: self.current().version,
current: self.inner().current().version,
ack_map_min: self.ack_map_min(),
min_stored: self.inner().min_stored(),
}
@@ -201,8 +215,8 @@ impl LayoutHelper {
pub(crate) fn digest(&self) -> RpcLayoutDigest {
RpcLayoutDigest {
current_version: self.current().version,
active_versions: self.versions().len(),
current_version: self.inner().current().version,
active_versions: self.inner().versions.len(),
trackers_hash: self.trackers_hash,
staging_hash: self.staging_hash,
}
@@ -246,7 +260,8 @@ impl LayoutHelper {
pub(crate) fn update_ack_to_max_free(&mut self, local_node_id: Uuid) -> bool {
let max_free = self
.versions()
.inner()
.versions
.iter()
.map(|x| x.version)
.skip_while(|v| {
@@ -256,7 +271,7 @@ impl LayoutHelper {
.unwrap_or(true)
})
.next()
.unwrap_or(self.current().version);
.unwrap_or(self.inner().current().version);
let changed = self.update(|layout| {
layout
.update_trackers
+15 -16
View File
@@ -105,7 +105,7 @@ impl LayoutManager {
}
pub fn add_table(&self, table_name: &'static str) {
let first_version = self.layout().versions().first().unwrap().version;
let first_version = self.layout().inner().versions.first().unwrap().version;
self.table_sync_version
.lock()
@@ -139,19 +139,20 @@ impl LayoutManager {
// ---- ACK LOCKING ----
pub fn write_lock_with<T, F>(self: &Arc<Self>, f: F) -> WriteLock<T>
pub fn write_lock_with<T, F>(self: &Arc<Self>, f: F) -> Result<WriteLock<T>, Error>
where
F: FnOnce(&LayoutHelper) -> T,
F: FnOnce(&[LayoutVersion]) -> T,
{
let layout = self.layout();
let version = layout.current().version;
let value = f(&layout);
let current_version = layout.current()?.version;
let versions = layout.versions()?;
let value = f(versions);
layout
.ack_lock
.get(&version)
.get(&current_version)
.unwrap()
.fetch_add(1, Ordering::Relaxed);
WriteLock::new(version, self, value)
Ok(WriteLock::new(current_version, self, value))
}
// ---- INTERNALS ---
@@ -228,13 +229,11 @@ impl LayoutManager {
}
/// Save cluster layout data to disk
async fn save_cluster_layout(&self) -> Result<(), Error> {
async fn save_cluster_layout(&self) {
let layout = self.layout.read().unwrap().inner().clone();
self.persist_cluster_layout
.save_async(&layout)
.await
.expect("Cannot save current cluster layout");
Ok(())
if let Err(e) = self.persist_cluster_layout.save_async(&layout).await {
error!("Failed to save cluster_layout: {}", e);
}
}
fn broadcast_update(self: &Arc<Self>, rpc: SystemRpc) {
@@ -312,7 +311,7 @@ impl LayoutManager {
self.change_notify.notify_waiters();
self.broadcast_update(SystemRpc::AdvertiseClusterLayout(new_layout));
self.save_cluster_layout().await?;
self.save_cluster_layout().await;
}
Ok(SystemRpc::Ok)
@@ -327,7 +326,7 @@ impl LayoutManager {
if let Some(new_trackers) = self.merge_layout_trackers(trackers) {
self.change_notify.notify_waiters();
self.broadcast_update(SystemRpc::AdvertiseClusterLayoutTrackers(new_trackers));
self.save_cluster_layout().await?;
self.save_cluster_layout().await;
}
Ok(SystemRpc::Ok)
@@ -369,7 +368,7 @@ impl<T> Drop for WriteLock<T> {
let layout = self.layout_manager.layout(); // acquire read lock
if let Some(counter) = layout.ack_lock.get(&self.layout_version) {
let prev_lock = counter.fetch_sub(1, Ordering::Relaxed);
if prev_lock == 1 && layout.current().version > self.layout_version {
if prev_lock == 1 && layout.current().unwrap().version > self.layout_version {
drop(layout); // release read lock, write lock will be acquired
self.layout_manager.ack_new_version();
}
+8 -9
View File
@@ -118,15 +118,14 @@ impl LayoutVersion {
pub fn nodes_of(&self, position: &Hash) -> impl Iterator<Item = Uuid> + '_ {
let data = &self.ring_assignment_data;
let partition_nodes = if data.len() == self.replication_factor * (1 << PARTITION_BITS) {
let partition_idx = self.partition_of(position) as usize;
let partition_start = partition_idx * self.replication_factor;
let partition_end = (partition_idx + 1) * self.replication_factor;
&data[partition_start..partition_end]
} else {
warn!("Ring not yet ready, read/writes will be lost!");
&[]
};
if data.len() != self.replication_factor * (1 << PARTITION_BITS) {
panic!(".nodes_of() called on invalid LayoutVersion (this is a bug)");
}
let partition_idx = self.partition_of(position) as usize;
let partition_start = partition_idx * self.replication_factor;
let partition_end = (partition_idx + 1) * self.replication_factor;
let partition_nodes = &data[partition_start..partition_end];
partition_nodes
.iter()
+16 -12
View File
@@ -345,7 +345,7 @@ impl RpcHelper {
// Reorder requests to priorize closeness / low latency
let request_order =
self.request_order(&self.0.layout.read().unwrap().current(), to.iter().copied());
self.request_order(self.0.layout.read().unwrap().current()?, to.iter().copied());
let send_all_at_once = strategy.rs_send_all_at_once.unwrap_or(false);
// Build future for each request
@@ -567,25 +567,29 @@ impl RpcHelper {
/// The preference order, for each layout version, is given by `request_order`,
/// based on factors such as nodes being in the same datacenter,
/// having low ping, etc.
pub fn block_read_nodes_of(&self, position: &Hash, rpc_helper: &RpcHelper) -> Vec<Uuid> {
pub fn block_read_nodes_of(
&self,
position: &Hash,
rpc_helper: &RpcHelper,
) -> Result<Vec<Uuid>, Error> {
let layout = self.0.layout.read().unwrap();
let current_layout = layout.current()?;
// Compute, for each layout version, the set of nodes that might store
// the block, and put them in their preferred order as of `request_order`.
let mut vernodes = layout.versions().iter().map(|ver| {
let mut vernodes = vec![];
for ver in layout.versions()?.iter() {
let nodes = ver.nodes_of(position);
rpc_helper.request_order(layout.current(), nodes)
});
vernodes.push(rpc_helper.request_order(current_layout, nodes))
}
let mut ret = if layout.versions().len() == 1 {
let mut ret = if vernodes.len() == 1 {
// If we have only one active layout version, then these are the
// only nodes we ask in step 1
vernodes.next().unwrap()
vernodes.into_iter().next().unwrap()
} else {
let vernodes = vernodes.collect::<Vec<_>>();
let mut nodes = Vec::<Uuid>::with_capacity(12);
for i in 0..layout.current().replication_factor {
for i in 0..current_layout.replication_factor {
for vn in vernodes.iter() {
if let Some(n) = vn.get(i) {
if !nodes.contains(&n) {
@@ -608,14 +612,14 @@ impl RpcHelper {
let old_ver_iter = layout.inner().old_versions.iter().rev();
for ver in old_ver_iter {
let nodes = ver.nodes_of(position);
for node in rpc_helper.request_order(layout.current(), nodes) {
for node in rpc_helper.request_order(current_layout, nodes) {
if !ret.contains(&node) {
ret.push(node);
}
}
}
ret
Ok(ret)
}
fn request_order(
+52 -18
View File
@@ -453,11 +453,28 @@ impl System {
// Acquire a rwlock read-lock to the current cluster layout
let layout = self.cluster_layout();
let layout_versions = match layout.versions() {
Ok(v) => v,
Err(_) => {
// Layout not yet configured, special case
return ClusterHealth {
status: ClusterHealthStatus::Unavailable,
known_nodes: nodes.len(),
connected_nodes,
storage_nodes: 0,
storage_nodes_ok: 0,
partitions: 0,
partitions_quorum: 0,
partitions_all_ok: 0,
};
}
};
let current_layout = layout_versions.last().unwrap();
// Obtain information about nodes that have a role as storage nodes
// in one of the active layout versions
let mut storage_nodes = HashSet::<Uuid>::with_capacity(16);
for ver in layout.versions().iter() {
for ver in layout_versions.iter() {
storage_nodes.extend(
ver.roles
.items()
@@ -471,11 +488,11 @@ impl System {
// Determine the number of partitions that have:
// - a quorum of up nodes for all write sets (i.e. are available)
// - for which all nodes in all write sets are up (i.e. are fully healthy)
let partitions = layout.current().partitions().collect::<Vec<_>>();
let partitions = current_layout.partitions().collect::<Vec<_>>();
let mut partitions_quorum = 0;
let mut partitions_all_ok = 0;
for (_, hash) in partitions.iter() {
let mut write_sets = layout.versions().iter().map(|x| x.nodes_of(hash));
let mut write_sets = layout_versions.iter().map(|x| x.nodes_of(hash));
let has_quorum = write_sets
.clone()
.all(|set| set.filter(|x| node_up(x)).count() >= quorum);
@@ -630,21 +647,37 @@ impl System {
async fn discovery_loop(self: &Arc<Self>, mut stop_signal: watch::Receiver<bool>) {
while !*stop_signal.borrow() {
let n_connected = self
let peers_up = self
.peering
.get_peer_list()
.iter()
.filter(|p| p.is_up())
.count();
.map(|p| Uuid::from(p.id))
.collect::<Vec<_>>();
let not_configured = !self.cluster_layout().is_check_ok();
let no_peers = n_connected < self.replication_factor.into();
let expected_n_nodes = self.cluster_layout().all_nodes().len();
let bad_peers = n_connected != expected_n_nodes;
if not_configured || no_peers || bad_peers {
info!("Doing a bootstrap/discovery step (not_configured: {}, no_peers: {}, bad_peers: {})", not_configured, no_peers, bad_peers);
let do_bootstrap = match self.cluster_layout().all_nodes() {
Err(_) => {
debug!("doing bootstrap/discovery step (layout not configured)");
true
}
Ok(all_nodes) => {
// Do bootstrap if we have fewer peers than the replication
// factor,
// or if some peers in the layout are not connected
let do_bootstrap = peers_up.len() < self.replication_factor.into()
|| all_nodes.iter().any(|x| !peers_up.contains(x));
if do_bootstrap {
debug!(
"doing bootstrap/discovery step (peers_up: {}, all_nodes: {})",
peers_up.len(),
all_nodes.len()
);
}
do_bootstrap
}
};
if do_bootstrap {
let mut ping_list = resolve_peers(&self.bootstrap_peers).await;
// Add peer list from list stored on disk
@@ -687,12 +720,13 @@ impl System {
}
}
if !not_configured && !no_peers {
// If the layout is configured, and we already have some connections
// to other nodes in the cluster, we can skip trying to connect to
// nodes that are not in the cluster layout.
let layout = self.cluster_layout();
ping_list.retain(|(id, _)| layout.all_nodes().contains(&(*id).into()));
if let Ok(all_nodes) = self.cluster_layout().all_nodes() {
if peers_up.len() >= self.replication_factor.into() {
// If the layout is configured, and we already have some connections
// to other nodes in the cluster, we can skip trying to connect to
// nodes that are not in the cluster layout.
ping_list.retain(|(id, _)| all_nodes.contains(&(*id).into()));
}
}
for (node_id, node_addr) in ping_list {
+12 -6
View File
@@ -216,10 +216,13 @@ impl SystemMetrics {
.u64_value_observer("cluster_layout_node_connected", move |observer| {
let layout = system.cluster_layout();
let nodes = system.get_known_nodes();
for id in layout.all_nodes().iter() {
for id in layout.all_nodes().unwrap_or_default().iter() {
let mut kv = vec![KeyValue::new("id", format!("{:?}", id))];
if let Some(role) =
layout.current().roles.get(id).and_then(|r| r.0.as_ref())
if let Some(role) = layout
.current()
.ok()
.and_then(|l| l.roles.get(id))
.and_then(|r| r.0.as_ref())
{
kv.push(KeyValue::new("role_zone", role.zone.clone()));
match role.capacity {
@@ -260,10 +263,13 @@ impl SystemMetrics {
.u64_value_observer("cluster_layout_node_disconnected_time", move |observer| {
let layout = system.cluster_layout();
let nodes = system.get_known_nodes();
for id in layout.all_nodes().iter() {
for id in layout.all_nodes().unwrap_or_default().iter() {
let mut kv = vec![KeyValue::new("id", format!("{:?}", id))];
if let Some(role) =
layout.current().roles.get(id).and_then(|r| r.0.as_ref())
if let Some(role) = layout
.current()
.ok()
.and_then(|l| l.roles.get(id))
.and_then(|r| r.0.as_ref())
{
kv.push(KeyValue::new("role_zone", role.zone.clone()));
match role.capacity {
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_table"
version = "2.0.0"
version = "2.1.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+5 -5
View File
@@ -254,7 +254,7 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
// any node of the partition is unavailable.
let pk_hash = Hash::try_from(&tree_key[..32]).unwrap();
// TODO: this probably breaks when the layout changes
let nodes = self.replication.storage_nodes(&pk_hash);
let nodes = self.replication.storage_nodes(&pk_hash)?;
if nodes.first() == Some(&self.system.id) {
GcTodoEntry::new(tree_key, new_bytes_hash).save(&self.gc_todo)?;
}
@@ -368,11 +368,11 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
}
}
pub fn insert_queue_len(&self) -> Result<usize, Error> {
Ok(self.insert_queue.len()?)
pub fn insert_queue_approximate_len(&self) -> Result<usize, Error> {
Ok(self.insert_queue.approximate_len()?)
}
pub fn gc_todo_len(&self) -> Result<usize, Error> {
Ok(self.gc_todo.len()?)
pub fn gc_todo_approximate_len(&self) -> Result<usize, Error> {
Ok(self.gc_todo.approximate_len()?)
}
}
+2 -2
View File
@@ -153,7 +153,7 @@ impl<F: TableSchema, R: TableReplication> TableGc<F, R> {
let mut partitions = HashMap::new();
for entry in entries {
let pkh = Hash::try_from(&entry.key[..32]).unwrap();
let mut nodes = self.data.replication.storage_nodes(&pkh);
let mut nodes = self.data.replication.storage_nodes(&pkh)?;
nodes.retain(|x| *x != self.system.id);
nodes.sort();
@@ -313,7 +313,7 @@ impl<F: TableSchema, R: TableReplication> Worker for GcWorker<F, R> {
fn status(&self) -> WorkerStatus {
WorkerStatus {
queue_length: Some(self.gc.data.gc_todo_len().unwrap_or(0) as u64),
queue_length: Some(self.gc.data.gc_todo_approximate_len().unwrap_or(0) as u64),
..Default::default()
}
}
+6 -6
View File
@@ -102,7 +102,7 @@ impl<F: TableSchema, R: TableReplication> MerkleUpdater<F, R> {
partition: self
.data
.replication
.partition_of(&Hash::try_from(&k[0..32]).unwrap()),
.partition_of(&Hash::try_from(&k[0..32]).unwrap())?,
prefix: vec![],
};
self.data
@@ -287,12 +287,12 @@ impl<F: TableSchema, R: TableReplication> MerkleUpdater<F, R> {
MerkleNode::decode_opt(&ent)
}
pub fn merkle_tree_len(&self) -> Result<usize, Error> {
Ok(self.data.merkle_tree.len()?)
pub fn merkle_tree_approximate_len(&self) -> Result<usize, Error> {
Ok(self.data.merkle_tree.approximate_len()?)
}
pub fn todo_len(&self) -> Result<usize, Error> {
Ok(self.data.merkle_todo.len()?)
pub fn todo_approximate_len(&self) -> Result<usize, Error> {
Ok(self.data.merkle_todo.approximate_len()?)
}
}
@@ -306,7 +306,7 @@ impl<F: TableSchema, R: TableReplication> Worker for MerkleWorker<F, R> {
fn status(&self) -> WorkerStatus {
WorkerStatus {
queue_length: Some(self.0.todo_len().unwrap_or(0) as u64),
queue_length: Some(self.0.todo_approximate_len().unwrap_or(0) as u64),
..Default::default()
}
}
+5 -5
View File
@@ -36,7 +36,7 @@ impl TableMetrics {
.u64_value_observer(
"table.size",
move |observer| {
if let Ok(value) = store.len() {
if let Ok(value) = store.approximate_len() {
observer.observe(
value as u64,
&[KeyValue::new("table_name", table_name)],
@@ -50,7 +50,7 @@ impl TableMetrics {
.u64_value_observer(
"table.merkle_tree_size",
move |observer| {
if let Ok(value) = merkle_tree.len() {
if let Ok(value) = merkle_tree.approximate_len() {
observer.observe(
value as u64,
&[KeyValue::new("table_name", table_name)],
@@ -64,7 +64,7 @@ impl TableMetrics {
.u64_value_observer(
"table.merkle_updater_todo_queue_length",
move |observer| {
if let Ok(v) = merkle_todo.len() {
if let Ok(v) = merkle_todo.approximate_len() {
observer.observe(
v as u64,
&[KeyValue::new("table_name", table_name)],
@@ -78,7 +78,7 @@ impl TableMetrics {
.u64_value_observer(
"table.insert_queue_length",
move |observer| {
if let Ok(v) = insert_queue.len() {
if let Ok(v) = insert_queue.approximate_len() {
observer.observe(
v as u64,
&[KeyValue::new("table_name", table_name)],
@@ -92,7 +92,7 @@ impl TableMetrics {
.u64_value_observer(
"table.gc_todo_queue_length",
move |observer| {
if let Ok(value) = gc_todo.len() {
if let Ok(value) = gc_todo.approximate_len() {
observer.observe(
value as u64,
&[KeyValue::new("table_name", table_name)],
+1 -1
View File
@@ -27,7 +27,7 @@ impl<F: TableSchema, R: TableReplication> Worker for InsertQueueWorker<F, R> {
fn status(&self) -> WorkerStatus {
WorkerStatus {
queue_length: Some(self.0.data.insert_queue.len().unwrap_or(0) as u64),
queue_length: Some(self.0.data.insert_queue.approximate_len().unwrap_or(0) as u64),
..Default::default()
}
}
+56 -44
View File
@@ -2,8 +2,9 @@ use std::sync::Arc;
use std::time::Duration;
use garage_rpc::layout::*;
use garage_rpc::system::System;
use garage_rpc::{replication_mode::ConsistencyMode, system::System};
use garage_util::data::*;
use garage_util::error::Error;
use crate::replication::*;
@@ -22,6 +23,7 @@ use crate::replication::*;
pub struct TableFullReplication {
/// The membership manager of this node
pub system: Arc<System>,
pub consistency_mode: ConsistencyMode,
}
impl TableReplication for TableFullReplication {
@@ -33,53 +35,64 @@ impl TableReplication for TableFullReplication {
// Also, it's generally a much bigger problem for fullcopy tables to be out of sync.
const ANTI_ENTROPY_INTERVAL: Duration = Duration::from_secs(10);
fn storage_nodes(&self, _hash: &Hash) -> Vec<Uuid> {
self.system.cluster_layout().all_nodes().to_vec()
fn storage_nodes(&self, _hash: &Hash) -> Result<Vec<Uuid>, Error> {
Ok(self.system.cluster_layout().all_nodes()?.to_vec())
}
fn read_nodes(&self, _hash: &Hash) -> Vec<Uuid> {
self.system
fn read_nodes(&self, _hash: &Hash) -> Result<Vec<Uuid>, Error> {
Ok(self
.system
.cluster_layout()
.read_version()
.read_version()?
.all_nodes()
.to_vec()
.to_vec())
}
fn read_quorum(&self) -> usize {
let layout = self.system.cluster_layout();
let nodes = layout.read_version().all_nodes();
nodes.len().div_euclid(2) + 1
}
fn write_sets(&self, _hash: &Hash) -> Self::WriteSets {
self.system.layout_manager.write_lock_with(write_sets)
}
fn write_quorum(&self) -> usize {
let layout = self.system.cluster_layout();
let min_len = layout
.versions()
.iter()
.map(|x| x.all_nodes().len())
.min()
.unwrap();
let max_quorum = layout
.versions()
.iter()
.map(|x| x.all_nodes().len().div_euclid(2) + 1)
.max()
.unwrap();
if min_len < max_quorum {
warn!("Write quorum will not be respected for TableFullReplication operations due to multiple active layout versions with vastly different number of nodes");
min_len
} else {
max_quorum
fn read_quorum(&self) -> Result<usize, Error> {
match self.consistency_mode {
ConsistencyMode::Dangerous | ConsistencyMode::Degraded => Ok(1),
ConsistencyMode::Consistent => {
let layout = self.system.cluster_layout();
let nodes = layout.read_version()?.all_nodes();
Ok(nodes.len().div_ceil(2))
}
}
}
fn partition_of(&self, _hash: &Hash) -> Partition {
0u16
fn write_sets(&self, _hash: &Hash) -> Result<Self::WriteSets, Error> {
self.system.layout_manager.write_lock_with(write_sets)
}
fn write_quorum(&self) -> Result<usize, Error> {
match self.consistency_mode {
ConsistencyMode::Dangerous => Ok(1),
ConsistencyMode::Degraded | ConsistencyMode::Consistent => {
let layout = self.system.cluster_layout();
let min_len = layout
.versions()?
.iter()
.map(|x| x.all_nodes().len())
.min()
.unwrap();
let max_quorum = layout
.versions()?
.iter()
.map(|x| x.all_nodes().len().div_euclid(2) + 1)
.max()
.unwrap();
if min_len < max_quorum {
warn!("Write quorum will not be respected for TableFullReplication operations due to multiple active layout versions with vastly different number of nodes");
Ok(std::cmp::max(1, min_len))
} else {
Ok(max_quorum)
}
}
}
}
fn sync_partitions(&self) -> SyncPartitions {
fn partition_of(&self, _hash: &Hash) -> Result<Partition, Error> {
Ok(0u16)
}
fn sync_partitions(&self) -> Result<SyncPartitions, Error> {
let layout = self.system.cluster_layout();
let layout_version = layout.ack_map_min();
@@ -87,19 +100,18 @@ impl TableReplication for TableFullReplication {
partition: 0u16,
first_hash: [0u8; 32].into(),
last_hash: [0xff; 32].into(),
storage_sets: write_sets(&layout),
storage_sets: write_sets(layout.versions()?),
}];
SyncPartitions {
Ok(SyncPartitions {
layout_version,
partitions,
}
})
}
}
fn write_sets(layout: &LayoutHelper) -> Vec<Vec<Uuid>> {
layout
.versions()
fn write_sets(layout_versions: &[LayoutVersion]) -> Vec<Vec<Uuid>> {
layout_versions
.iter()
.map(|x| x.all_nodes().to_vec())
.collect()
+8 -7
View File
@@ -2,6 +2,7 @@ use std::time::Duration;
use garage_rpc::layout::*;
use garage_util::data::*;
use garage_util::error::Error;
/// Trait to describe how a table shall be replicated
pub trait TableReplication: Send + Sync + 'static {
@@ -13,23 +14,23 @@ pub trait TableReplication: Send + Sync + 'static {
// To understand various replication methods
/// The entire list of all nodes that store a partition
fn storage_nodes(&self, hash: &Hash) -> Vec<Uuid>;
fn storage_nodes(&self, hash: &Hash) -> Result<Vec<Uuid>, Error>;
/// Which nodes to send read requests to
fn read_nodes(&self, hash: &Hash) -> Vec<Uuid>;
fn read_nodes(&self, hash: &Hash) -> Result<Vec<Uuid>, Error>;
/// Responses needed to consider a read successful
fn read_quorum(&self) -> usize;
fn read_quorum(&self) -> Result<usize, Error>;
/// Which nodes to send writes to
fn write_sets(&self, hash: &Hash) -> Self::WriteSets;
fn write_sets(&self, hash: &Hash) -> Result<Self::WriteSets, Error>;
/// Responses needed to consider a write successful in each set
fn write_quorum(&self) -> usize;
fn write_quorum(&self) -> Result<usize, Error>;
// Accessing partitions, for Merkle tree & sync
/// Get partition for data with given hash
fn partition_of(&self, hash: &Hash) -> Partition;
fn partition_of(&self, hash: &Hash) -> Result<Partition, Error>;
/// List of partitions and nodes to sync with in current layout
fn sync_partitions(&self) -> SyncPartitions;
fn sync_partitions(&self) -> Result<SyncPartitions, Error>;
}
#[derive(Debug)]
+33 -28
View File
@@ -4,6 +4,7 @@ use std::time::Duration;
use garage_rpc::layout::*;
use garage_rpc::replication_mode::ConsistencyMode;
use garage_util::data::*;
use garage_util::error::Error;
use crate::replication::sharded::manager::LayoutManager;
use crate::replication::*;
@@ -27,59 +28,64 @@ impl TableReplication for TableShardedReplication {
type WriteSets = WriteLock<Vec<Vec<Uuid>>>;
fn storage_nodes(&self, hash: &Hash) -> Vec<Uuid> {
fn storage_nodes(&self, hash: &Hash) -> Result<Vec<Uuid>, Error> {
let mut ret = vec![];
for version in self.layout_manager.layout().versions().iter() {
for version in self.layout_manager.layout().versions()?.iter() {
ret.extend(version.nodes_of(hash));
}
ret.sort();
ret.dedup();
ret
Ok(ret)
}
fn read_nodes(&self, hash: &Hash) -> Vec<Uuid> {
self.layout_manager
fn read_nodes(&self, hash: &Hash) -> Result<Vec<Uuid>, Error> {
Ok(self
.layout_manager
.layout()
.read_version()
.read_version()?
.nodes_of(hash)
.collect()
.collect())
}
fn read_quorum(&self) -> usize {
self.layout_manager
fn read_quorum(&self) -> Result<usize, Error> {
Ok(self
.layout_manager
.layout()
.read_version()
.read_quorum(self.consistency_mode)
.read_version()?
.read_quorum(self.consistency_mode))
}
fn write_sets(&self, hash: &Hash) -> Self::WriteSets {
self.layout_manager.write_lock_with(|l| write_sets(l, hash))
}
fn write_quorum(&self) -> usize {
fn write_sets(&self, hash: &Hash) -> Result<Self::WriteSets, Error> {
self.layout_manager
.write_lock_with(|lvs| write_sets(lvs, hash))
}
fn write_quorum(&self) -> Result<usize, Error> {
Ok(self
.layout_manager
.layout()
.current()
.write_quorum(self.consistency_mode)
.current()?
.write_quorum(self.consistency_mode))
}
fn partition_of(&self, hash: &Hash) -> Partition {
self.layout_manager.layout().current().partition_of(hash)
fn partition_of(&self, hash: &Hash) -> Result<Partition, Error> {
Ok(self.layout_manager.layout().current()?.partition_of(hash))
}
fn sync_partitions(&self) -> SyncPartitions {
fn sync_partitions(&self) -> Result<SyncPartitions, Error> {
let layout = self.layout_manager.layout();
let layout_versions = layout.versions()?;
let layout_version = layout.ack_map_min();
let mut partitions = layout
.current()
.current()?
.partitions()
.map(|(partition, first_hash)| {
SyncPartition {
partition,
first_hash,
last_hash: [0u8; 32].into(), // filled in just after
storage_sets: write_sets(&layout, &first_hash),
storage_sets: write_sets(layout_versions, &first_hash),
}
})
.collect::<Vec<_>>();
@@ -92,16 +98,15 @@ impl TableReplication for TableShardedReplication {
};
}
SyncPartitions {
Ok(SyncPartitions {
layout_version,
partitions,
}
})
}
}
fn write_sets(layout: &LayoutHelper, hash: &Hash) -> Vec<Vec<Uuid>> {
layout
.versions()
fn write_sets(layout_versions: &[LayoutVersion], hash: &Hash) -> Vec<Vec<Uuid>> {
layout_versions
.iter()
.map(|x| x.nodes_of(hash).collect())
.collect()

Some files were not shown because too many files have changed in this diff Show More