Compare commits

..

48 Commits

Author SHA1 Message Date
Alex Auvolat 8bccddb504 add garage node dump-table command for debugging purpose 2025-08-29 16:35:47 +02:00
Alex 0b901bf291 Merge pull request 'garage_db: reduce frequency of sqlite snapshot progress log (fix #1129)' (#1146) from fix-1129 into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1146
2025-08-27 22:26:32 +00:00
Alex Auvolat c8c20d6f47 garage_db: reduce frequency of sqlite snapshot progress log (fix #1129) 2025-08-28 00:07:35 +02:00
Alex e5db610e4c Merge pull request 'K2V client: allow custom HTTP client' (#731) from k2v/shared_http_client into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/731
Reviewed-by: maximilien <me@mricher.fr>
2025-08-27 21:21:09 +00:00
Alex 65c6f8adea Merge pull request 'garage_db: refactor open function' (#1142) from factor-db-open into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1142
2025-08-27 21:10:59 +00:00
Alex Auvolat 54b9bf02a3 garage_db: refactor open function 2025-08-27 23:03:09 +02:00
Alex 469153233f Merge pull request 'garage_db: rename len to approximate_len as it is used for stats only' (#1141) from db-approximate-len into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1141
2025-08-27 20:44:50 +00:00
Alex Auvolat 90bba5889a garage_db: rename len to approximate_len as it is used for stats only 2025-08-27 21:23:45 +02:00
Alex a64b567d43 Merge pull request 'Add experimental support for Fjall DB engine' (#906) from withings/garage:feat/fjall-db-engine into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/906
2025-08-27 19:09:40 +00:00
Alex Auvolat 6ea86db8cd document fjall db engine, remove flakey metadata_fsync implementation 2025-08-27 20:22:41 +02:00
Alex Auvolat aa69c06f2b fix potential race condition and naming bug in fjall adapter 2025-08-27 20:22:38 +02:00
Alex Auvolat a6c6c44310 nix: build and test fjall feature 2025-08-27 18:54:42 +02:00
Julien Kritter 96d7713915 Add support for an LSM-tree-based backend with Fjall 2025-08-27 18:54:34 +02:00
Alex d64498c3d3 Merge pull request 'log access keys' (#1122) from 1686a/log-access-key into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1122
2025-08-27 16:18:16 +00:00
trinity-1686a b340599e68 log access keys 2025-08-03 15:30:56 +02:00
Alex 5448012b27 Merge pull request 'Pixelfed_support' (#1118) from Lapineige/garage:Pixelfed_support into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1118
2025-08-02 15:03:57 +00:00
Alex ce34d11a65 Merge pull request 'don't die on SIGHUP' (#1121) from 1686a/handle-sighup into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1121
2025-08-02 14:53:58 +00:00
Alex 8cb7623ebd Merge pull request 'handle ECONNABORTED' (#1120) from 1686a/handle-econnaborted into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1120
2025-08-02 14:53:45 +00:00
trinity-1686a 5469c95877 handle ECONNABORTED 2025-08-02 13:14:01 +02:00
trinity-1686a f930c6f643 don't die on SIGHUP 2025-08-02 13:09:33 +02:00
Alex afcb22bf16 Merge pull request 'Fix typo in peertube buckets names' (#1117) from Lapineige/garage:main into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1117
2025-08-02 08:27:01 +00:00
Lapineige cc29a40d51 Actualiser doc/book/connect/apps/index.md 2025-08-01 21:35:15 +00:00
Lapineige 0f3f180c3e Merge branch 'main-v1' into main 2025-08-01 21:33:58 +00:00
Lapineige 70cf6004ae Fix typo in peertube buckets names 2025-08-01 21:32:59 +00:00
Alex c7571ff89b Merge pull request 'Fix some unsoundness in lmdb adapter unsafe' (#1099) from krtab/garage:fix_some_ub into main-v1
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1099
2025-07-31 19:38:23 +00:00
Arthur Carcano 1b42919bf7 Fix some unsoundness in lmdb adapter unsafe 2025-07-25 23:33:51 +02:00
Alex 3f4ab3a4a3 Merge pull request 'Garage v1.2.0' (#1068) from rel-1.2.0 into main
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1068
2025-06-13 16:12:29 +00:00
Alex Auvolat 3a4afc04a9 cargo: update crossbeam-channel to avoid yanked version 2025-06-13 17:22:47 +02:00
Alex Auvolat fbf03e9378 bump version to v1.2.0 2025-06-13 14:21:28 +02:00
Alex 9eb07d4c7b Merge pull request 'cli: mark block refs as deleted in garage block purge (fix #1055)' (#1067) from fix-1055 into main
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1067
2025-06-13 11:53:41 +00:00
Alex Auvolat 85ee4f5d8c cli: mark block refs as deleted in garage block purge 2025-06-13 13:52:02 +02:00
Alex 328072d122 Merge pull request 'put web error in a basic webpage' (#1064) from trinity-1686a/garage:1686a/non-xml-web-error into main
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1064
2025-06-12 06:06:38 +00:00
trinity-1686a 26bc807905 put web error in a basic webpage
before, it was a plain string, with an xml content type

this caused browsers to show very ugly and meaningless pages
2025-06-10 22:23:06 +02:00
Alex a9f5f242b2 Merge pull request 'feat: add log to journald feature' (#1056) from ragazenta/garage:feat/tracing-journald into main
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1056
2025-06-10 18:38:23 +00:00
maximilien ae98abca5c Merge pull request 'Add eddster2309/ansible-role-garage as deployment option' (#1057) from eddster2309/garage:main into main
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1057
Reviewed-by: maximilien <me@mricher.fr>
2025-06-08 11:56:31 +00:00
eddster2309 adfa44ad70 Add architecture support 2025-06-03 09:22:43 +00:00
eddster2309 47143b88ad Add eddster2309/ansible-role-garage as deployment option 2025-06-03 09:15:57 +00:00
Renjaya Raga Zenta 8843aa92fa feat: add log to journald feature
The systemd-journald is used in most major Linux distros that use systemd.
This enables logging using the systemd-journald native protocol, instead
of just writing to stderr.
2025-06-02 11:55:27 +07:00
Alex b601b3e46d Merge pull request 'documentation: Minor doc change to clarify why the capacity does not matter and how the zone name is used' (#1051) from ddxv/garage:docs-quick-start into main
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1051
2025-05-30 16:26:19 +00:00
Alex a19d2f16e2 Merge pull request 'api: s3: implement get bucket acl' (#1045) from ragazenta/garage:feat/dummy-acl into main
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1045
2025-05-30 16:25:04 +00:00
trinity-1686a fc8fc60f6d emit internal error when we detect race condition (#1053) (fix #1050)
i went with a `500`/`InternalError`/`Please try again.` because that is something i've seen AWS S3 report while developing other software, and i'm not convinced all clients would understand a 409 conflict properly (GET don't usually conflict)

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1053
Co-authored-by: trinity-1686a <trinity@deuxfleurs.fr>
Co-committed-by: trinity-1686a <trinity@deuxfleurs.fr>
2025-05-30 16:24:12 +00:00
Alex 77079a1498 Merge pull request '[1.1.x] speed up UploadPartCopy' (#1047) from yuka/garage:uploadpartcopy-v1 into main
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1047
2025-05-30 16:22:35 +00:00
James O'Claire 2a4f729b57 Minor doc change to clarify why the capacity does not matter and how the zone name is used 2025-05-28 09:49:50 +08:00
Renjaya Raga Zenta 1b042e379e api: s3: implement get bucket acl 2025-05-26 09:43:15 +07:00
Yureka ffbce0f689 speed up UploadPartCopy
(cherry picked from commit db54bf96c7)
2025-05-23 20:36:32 +02:00
Alex 37e5621dde Merge pull request 'documentation updates' (#1046) from doc-updates into main
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1046
2025-05-23 15:05:19 +00:00
Alex Auvolat 6529ff379a documentation updates 2025-05-23 17:02:23 +02:00
Quentin Dufour 8b35a946d9 Allow external HTTP client 2024-02-23 17:09:47 +01:00
68 changed files with 1669 additions and 1336 deletions
+5
View File
@@ -28,6 +28,11 @@ steps:
commands: commands:
- nix-build -j4 --attr flakePackages.tests-sqlite - nix-build -j4 --attr flakePackages.tests-sqlite
- name: unit + func tests (fjall)
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --attr flakePackages.tests-fjall
- name: integration tests - name: integration tests
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
commands: commands:
Generated
+225 -16
View File
@@ -687,6 +687,12 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d2c12f985c78475a6b8d629afd0c360260ef34cfef52efccdcfd31972f81c2e" checksum = "2d2c12f985c78475a6b8d629afd0c360260ef34cfef52efccdcfd31972f81c2e"
[[package]]
name = "byteview"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.2.16" version = "1.2.16"
@@ -798,6 +804,12 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
[[package]]
name = "compare"
version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7"
[[package]] [[package]]
name = "core-foundation" name = "core-foundation"
version = "0.9.4" version = "0.9.4"
@@ -867,9 +879,18 @@ dependencies = [
[[package]] [[package]]
name = "crossbeam-channel" name = "crossbeam-channel"
version = "0.5.14" version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [ dependencies = [
"crossbeam-utils", "crossbeam-utils",
] ]
@@ -883,6 +904,16 @@ dependencies = [
"crossbeam-utils", "crossbeam-utils",
] ]
[[package]]
name = "crossbeam-skiplist"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]] [[package]]
name = "crossbeam-utils" name = "crossbeam-utils"
version = "0.8.21" version = "0.8.21"
@@ -954,6 +985,20 @@ dependencies = [
"num_cpus", "num_cpus",
] ]
[[package]]
name = "dashmap"
version = "6.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
dependencies = [
"cfg-if",
"crossbeam-utils",
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core 0.9.10",
]
[[package]] [[package]]
name = "deranged" name = "deranged"
version = "0.4.0" version = "0.4.0"
@@ -996,6 +1041,12 @@ dependencies = [
"syn 2.0.100", "syn 2.0.100",
] ]
[[package]]
name = "double-ended-peekable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
[[package]] [[package]]
name = "dyn-clone" name = "dyn-clone"
version = "1.0.19" version = "1.0.19"
@@ -1017,6 +1068,18 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "enum_dispatch"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd"
dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn 2.0.100",
]
[[package]] [[package]]
name = "env_logger" name = "env_logger"
version = "0.10.2" version = "0.10.2"
@@ -1084,6 +1147,23 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "fjall"
version = "2.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc"
dependencies = [
"byteorder",
"byteview",
"dashmap 6.1.0",
"log",
"lsm-tree",
"path-absolutize",
"std-semaphore",
"tempfile",
"xxhash-rust",
]
[[package]] [[package]]
name = "fnv" name = "fnv"
version = "1.0.7" version = "1.0.7"
@@ -1200,7 +1280,7 @@ dependencies = [
[[package]] [[package]]
name = "garage" name = "garage"
version = "1.1.0" version = "1.2.0"
dependencies = [ dependencies = [
"assert-json-diff", "assert-json-diff",
"async-trait", "async-trait",
@@ -1241,6 +1321,7 @@ dependencies = [
"opentelemetry-prometheus", "opentelemetry-prometheus",
"parse_duration", "parse_duration",
"serde", "serde",
"serde_bytes",
"serde_json", "serde_json",
"sha1", "sha1",
"sha2", "sha2",
@@ -1250,12 +1331,13 @@ dependencies = [
"timeago", "timeago",
"tokio", "tokio",
"tracing", "tracing",
"tracing-journald",
"tracing-subscriber", "tracing-subscriber",
] ]
[[package]] [[package]]
name = "garage_api_admin" name = "garage_api_admin"
version = "1.1.0" version = "1.2.0"
dependencies = [ dependencies = [
"argon2", "argon2",
"async-trait", "async-trait",
@@ -1281,7 +1363,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_api_common" name = "garage_api_common"
version = "1.1.0" version = "1.2.0"
dependencies = [ dependencies = [
"base64 0.21.7", "base64 0.21.7",
"bytes", "bytes",
@@ -1315,7 +1397,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_api_k2v" name = "garage_api_k2v"
version = "1.1.0" version = "1.2.0"
dependencies = [ dependencies = [
"base64 0.21.7", "base64 0.21.7",
"err-derive", "err-derive",
@@ -1338,7 +1420,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_api_s3" name = "garage_api_s3"
version = "1.1.0" version = "1.2.0"
dependencies = [ dependencies = [
"aes-gcm", "aes-gcm",
"async-compression", "async-compression",
@@ -1383,7 +1465,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_block" name = "garage_block"
version = "1.1.0" version = "1.2.0"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-compression", "async-compression",
@@ -1408,11 +1490,13 @@ dependencies = [
[[package]] [[package]]
name = "garage_db" name = "garage_db"
version = "1.1.0" version = "1.2.0"
dependencies = [ dependencies = [
"err-derive", "err-derive",
"fjall",
"heed", "heed",
"mktemp", "mktemp",
"parking_lot 0.12.3",
"r2d2", "r2d2",
"r2d2_sqlite", "r2d2_sqlite",
"rusqlite", "rusqlite",
@@ -1421,7 +1505,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_model" name = "garage_model"
version = "1.1.0" version = "1.2.0"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"base64 0.21.7", "base64 0.21.7",
@@ -1448,7 +1532,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_net" name = "garage_net"
version = "1.1.0" version = "1.2.0"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"bytes", "bytes",
@@ -1473,7 +1557,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_rpc" name = "garage_rpc"
version = "1.1.0" version = "1.2.0"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-trait", "async-trait",
@@ -1505,7 +1589,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_table" name = "garage_table"
version = "1.1.0" version = "1.2.0"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-trait", "async-trait",
@@ -1526,7 +1610,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_util" name = "garage_util"
version = "1.1.0" version = "1.2.0"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-trait", "async-trait",
@@ -1558,7 +1642,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_web" name = "garage_web"
version = "1.1.0" version = "1.2.0"
dependencies = [ dependencies = [
"err-derive", "err-derive",
"garage_api_common", "garage_api_common",
@@ -1654,6 +1738,12 @@ dependencies = [
"syn 2.0.100", "syn 2.0.100",
] ]
[[package]]
name = "guardian"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f"
[[package]] [[package]]
name = "h2" name = "h2"
version = "0.3.26" version = "0.3.26"
@@ -2228,6 +2318,15 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "interval-heap"
version = "0.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6"
dependencies = [
"compare",
]
[[package]] [[package]]
name = "ipnet" name = "ipnet"
version = "2.11.0" version = "2.11.0"
@@ -2584,6 +2683,36 @@ dependencies = [
"hashbrown 0.15.2", "hashbrown 0.15.2",
] ]
[[package]]
name = "lsm-tree"
version = "2.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab73c02eadb3dc12c0024e5b61d6284e6d59064e67e74fbad77856caa56f62c7"
dependencies = [
"byteorder",
"crossbeam-skiplist",
"double-ended-peekable",
"enum_dispatch",
"guardian",
"interval-heap",
"log",
"lz4_flex",
"path-absolutize",
"quick_cache",
"rustc-hash",
"self_cell",
"tempfile",
"value-log",
"varint-rs",
"xxhash-rust",
]
[[package]]
name = "lz4_flex"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
[[package]] [[package]]
name = "matchers" name = "matchers"
version = "0.1.0" version = "0.1.0"
@@ -2838,7 +2967,7 @@ checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"crossbeam-channel", "crossbeam-channel",
"dashmap", "dashmap 4.0.2",
"fnv", "fnv",
"futures-channel", "futures-channel",
"futures-executor", "futures-executor",
@@ -2999,6 +3128,24 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "path-absolutize"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5"
dependencies = [
"path-dedot",
]
[[package]]
name = "path-dedot"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397"
dependencies = [
"once_cell",
]
[[package]] [[package]]
name = "pem" name = "pem"
version = "3.0.5" version = "3.0.5"
@@ -3294,6 +3441,16 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "quick_cache"
version = "0.6.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786"
dependencies = [
"equivalent",
"hashbrown 0.15.2",
]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.40" version = "1.0.40"
@@ -3531,6 +3688,12 @@ version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
[[package]]
name = "rustc-hash"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
[[package]] [[package]]
name = "rustc_version" name = "rustc_version"
version = "0.4.1" version = "0.4.1"
@@ -3773,6 +3936,12 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "self_cell"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749"
[[package]] [[package]]
name = "semver" name = "semver"
version = "1.0.26" version = "1.0.26"
@@ -3986,6 +4155,12 @@ dependencies = [
"syn 1.0.109", "syn 1.0.109",
] ]
[[package]]
name = "std-semaphore"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e"
[[package]] [[package]]
name = "strsim" name = "strsim"
version = "0.11.1" version = "0.11.1"
@@ -4514,6 +4689,17 @@ dependencies = [
"tracing", "tracing",
] ]
[[package]]
name = "tracing-journald"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc0b4143302cf1022dac868d521e36e8b27691f72c84b3311750d5188ebba657"
dependencies = [
"libc",
"tracing-core",
"tracing-subscriber",
]
[[package]] [[package]]
name = "tracing-log" name = "tracing-log"
version = "0.2.0" version = "0.2.0"
@@ -4652,6 +4838,29 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
[[package]]
name = "value-log"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c"
dependencies = [
"byteorder",
"byteview",
"interval-heap",
"log",
"path-absolutize",
"rustc-hash",
"tempfile",
"varint-rs",
"xxhash-rust",
]
[[package]]
name = "varint-rs"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]] [[package]]
name = "vcpkg" name = "vcpkg"
version = "0.2.15" version = "0.2.15"
+15 -12
View File
@@ -24,18 +24,18 @@ default-members = ["src/garage"]
# Internal Garage crates # Internal Garage crates
format_table = { version = "0.1.1", path = "src/format-table" } format_table = { version = "0.1.1", path = "src/format-table" }
garage_api_common = { version = "1.1.0", path = "src/api/common" } garage_api_common = { version = "1.2.0", path = "src/api/common" }
garage_api_admin = { version = "1.1.0", path = "src/api/admin" } garage_api_admin = { version = "1.2.0", path = "src/api/admin" }
garage_api_s3 = { version = "1.1.0", path = "src/api/s3" } garage_api_s3 = { version = "1.2.0", path = "src/api/s3" }
garage_api_k2v = { version = "1.1.0", path = "src/api/k2v" } garage_api_k2v = { version = "1.2.0", path = "src/api/k2v" }
garage_block = { version = "1.1.0", path = "src/block" } garage_block = { version = "1.2.0", path = "src/block" }
garage_db = { version = "1.1.0", path = "src/db", default-features = false } garage_db = { version = "1.2.0", path = "src/db", default-features = false }
garage_model = { version = "1.1.0", path = "src/model", default-features = false } garage_model = { version = "1.2.0", path = "src/model", default-features = false }
garage_net = { version = "1.1.0", path = "src/net" } garage_net = { version = "1.2.0", path = "src/net" }
garage_rpc = { version = "1.1.0", path = "src/rpc" } garage_rpc = { version = "1.2.0", path = "src/rpc" }
garage_table = { version = "1.1.0", path = "src/table" } garage_table = { version = "1.2.0", path = "src/table" }
garage_util = { version = "1.1.0", path = "src/util" } garage_util = { version = "1.2.0", path = "src/util" }
garage_web = { version = "1.1.0", path = "src/web" } garage_web = { version = "1.2.0", path = "src/web" }
k2v-client = { version = "0.0.4", path = "src/k2v-client" } k2v-client = { version = "0.0.4", path = "src/k2v-client" }
# External crates from crates.io # External crates from crates.io
@@ -65,6 +65,7 @@ md-5 = "0.10"
mktemp = "0.5" mktemp = "0.5"
nix = { version = "0.29", default-features = false, features = ["fs"] } nix = { version = "0.29", default-features = false, features = ["fs"] }
nom = "7.1" nom = "7.1"
parking_lot = "0.12"
parse_duration = "2.1" parse_duration = "2.1"
pin-project = "1.0.12" pin-project = "1.0.12"
pnet_datalink = "0.34" pnet_datalink = "0.34"
@@ -83,12 +84,14 @@ pretty_env_logger = "0.5"
structopt = { version = "0.3", default-features = false } structopt = { version = "0.3", default-features = false }
syslog-tracing = "0.3" syslog-tracing = "0.3"
tracing = "0.1" tracing = "0.1"
tracing-journald = "0.3.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] }
heed = { version = "0.11", default-features = false, features = ["lmdb"] } heed = { version = "0.11", default-features = false, features = ["lmdb"] }
rusqlite = "0.31.0" rusqlite = "0.31.0"
r2d2 = "0.8" r2d2 = "0.8"
r2d2_sqlite = "0.24" r2d2_sqlite = "0.24"
fjall = "2.4"
async-compression = { version = "0.4", features = ["tokio", "zstd"] } async-compression = { version = "0.4", features = ["tokio", "zstd"] }
zstd = { version = "0.13", default-features = false } zstd = { version = "0.13", default-features = false }
+4 -4
View File
@@ -12,7 +12,7 @@ In this section, we cover the following web applications:
| [Mastodon](#mastodon) | ✅ | Natively supported | | [Mastodon](#mastodon) | ✅ | Natively supported |
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` | | [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
| [ejabberd](#ejabberd) | ✅ | `mod_s3_upload` | | [ejabberd](#ejabberd) | ✅ | `mod_s3_upload` |
| [Pixelfed](#pixelfed) | | Not yet tested | | [Pixelfed](#pixelfed) | | Natively supported |
| [Pleroma](#pleroma) | ❓ | Not yet tested | | [Pleroma](#pleroma) | ❓ | Not yet tested |
| [Lemmy](#lemmy) | ✅ | Supported with pict-rs | | [Lemmy](#lemmy) | ✅ | Supported with pict-rs |
| [Funkwhale](#funkwhale) | ❓ | Not yet tested | | [Funkwhale](#funkwhale) | ❓ | Not yet tested |
@@ -191,10 +191,10 @@ garage key create peertube-key
Keep the Key ID and the Secret key in a pad, they will be needed later. Keep the Key ID and the Secret key in a pad, they will be needed later.
We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist). We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists).
```bash ```bash
garage bucket create peertube-videos garage bucket create peertube-videos
garage bucket create peertube-playlist garage bucket create peertube-playlists
``` ```
Now we allow our key to read and write on these buckets: Now we allow our key to read and write on these buckets:
@@ -253,7 +253,7 @@ object_storage:
proxify_private_files: false proxify_private_files: false
streaming_playlists: streaming_playlists:
bucket_name: 'peertube-playlist' bucket_name: 'peertube-playlists'
# Keep it empty for our example # Keep it empty for our example
prefix: '' prefix: ''
+24 -12
View File
@@ -8,18 +8,18 @@ have published Ansible roles. We list them and compare them below.
## Comparison of Ansible roles ## Comparison of Ansible roles
| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | | Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster ansible-role-garage](#eddster-ansible-role-garage) |
|------------------------------------|---------------------------------------------|---------------------------------------------------------------| |------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------|
| **Runtime** | Systemd | Docker | | **Runtime** | Systemd | Docker | Systemd |
| **Target OS** | Any Linux | Any Linux | | **Target OS** | Any Linux | Any Linux | Any Linux |
| **Architecture** | amd64, arm64, i686 | amd64, arm64 | | **Architecture** | amd64, arm64, i686 | amd64, arm64 | arm64, arm, 386, amd64 |
| **Additional software** | None | Traefik | | **Additional software** | None | Traefik | Ngnix and Keepalived (optional) |
| **Automatic node connection** | ❌ | ✅ | | **Automatic node connection** | ❌ | ✅ | ✅ |
| **Layout management** | ❌ | ✅ | | **Layout management** | ❌ | ✅ | ✅ |
| **Manage buckets & keys** | ❌ | ✅ (basic) | | **Manage buckets & keys** | ❌ | ✅ (basic) | ✅ |
| **Allow custom Garage config** | ✅ | ❌ | | **Allow custom Garage config** | ✅ | ❌ | ❌ |
| **Facilitate Garage upgrades** | ✅ | ❌ | | **Facilitate Garage upgrades** | ✅ | ❌ | ✅ |
| **Multiple instances on one host** | ✅ | ✅ | | **Multiple instances on one host** | ✅ | ✅ | ❌ |
## zorun/ansible-role-garage ## zorun/ansible-role-garage
@@ -49,3 +49,15 @@ structured DNS names, etc).
As a result, this role makes it easier to start with Garage on Ansible, As a result, this role makes it easier to start with Garage on Ansible,
but is less flexible. but is less flexible.
## eddster2309/ansible-role-garage
[Source code](https://github.com/eddster2309/ansible-role-garage), [Ansible galaxy](https://galaxy.ansible.com/ui/standalone/roles/eddster2309/garage/)
This role is a opinionated but customisable role using the official Garage
static binaries and only requires Systemd. As such it should work on any
Linux based host. It includes all the nesscary configuration to
automatically setup a clustered Garage deployment. Most Garage
configuration options are exposed through Ansible variables so while you
can't provide a custom config you can get very close. It can optionally
installed a HA nginx deployment with Keepalived.
+5 -5
View File
@@ -96,14 +96,14 @@ to store 2 TB of data in total.
## Get a Docker image ## Get a Docker image
Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
We encourage you to use a fixed tag (eg. `v1.1.0`) and not the `latest` tag. We encourage you to use a fixed tag (eg. `v1.2.0`) and not the `latest` tag.
For this example, we will use the latest published version at the time of the writing which is `v1.1.0` but it's up to you For this example, we will use the latest published version at the time of the writing which is `v1.2.0` but it's up to you
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
For example: For example:
``` ```
sudo docker pull dxflrs/garage:v1.1.0 sudo docker pull dxflrs/garage:v1.2.0
``` ```
## Deploying and configuring Garage ## Deploying and configuring Garage
@@ -171,7 +171,7 @@ docker run \
-v /etc/garage.toml:/etc/garage.toml \ -v /etc/garage.toml:/etc/garage.toml \
-v /var/lib/garage/meta:/var/lib/garage/meta \ -v /var/lib/garage/meta:/var/lib/garage/meta \
-v /var/lib/garage/data:/var/lib/garage/data \ -v /var/lib/garage/data:/var/lib/garage/data \
dxflrs/garage:v1.1.0 dxflrs/garage:v1.2.0
``` ```
With this command line, Garage should be started automatically at each boot. With this command line, Garage should be started automatically at each boot.
@@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
version: "3" version: "3"
services: services:
garage: garage:
image: dxflrs/garage:v1.1.0 image: dxflrs/garage:v1.2.0
network_mode: "host" network_mode: "host"
restart: unless-stopped restart: unless-stopped
volumes: volumes:
+6 -5
View File
@@ -132,7 +132,7 @@ docker run \
-v /path/to/garage.toml:/etc/garage.toml \ -v /path/to/garage.toml:/etc/garage.toml \
-v /path/to/garage/meta:/var/lib/garage/meta \ -v /path/to/garage/meta:/var/lib/garage/meta \
-v /path/to/garage/data:/var/lib/garage/data \ -v /path/to/garage/data:/var/lib/garage/data \
dxflrs/garage:v1.1.0 dxflrs/garage:v1.2.0
``` ```
Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903` Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903`
@@ -182,11 +182,12 @@ ID Hostname Address Tag Zone Capacit
## Creating a cluster layout ## Creating a cluster layout
Creating a cluster layout for a Garage deployment means informing Garage Creating a cluster layout for a Garage deployment means informing Garage
of the disk space available on each node of the cluster of the disk space available on each node of the cluster, `-c`,
as well as the zone (e.g. datacenter) each machine is located in. as well as the name of the zone (e.g. datacenter), `-z`, each machine is located in.
For our test deployment, we are using only one node. The way in which we configure For our test deployment, we are have only one node with zone named `dc1` and a
it does not matter, you can simply write: capacity of `1G`, though the capacity is ignored for a single node deployment
and can be changed later when adding new nodes.
```bash ```bash
garage layout assign -z dc1 -c 1G <node_id> garage layout assign -z dc1 -c 1G <node_id>
+23 -9
View File
@@ -93,30 +93,30 @@ The following gives details about each available configuration option.
[Environment variables](#env_variables). [Environment variables](#env_variables).
Top-level configuration options: Top-level configuration options, in alphabetical order:
[`allow_punycode`](#allow_punycode),
[`allow_world_readable_secrets`](#allow_world_readable_secrets), [`allow_world_readable_secrets`](#allow_world_readable_secrets),
[`block_ram_buffer_max`](#block_ram_buffer_max), [`block_ram_buffer_max`](#block_ram_buffer_max),
[`block_size`](#block_size), [`block_size`](#block_size),
[`bootstrap_peers`](#bootstrap_peers), [`bootstrap_peers`](#bootstrap_peers),
[`compression_level`](#compression_level), [`compression_level`](#compression_level),
[`consistency_mode`](#consistency_mode),
[`data_dir`](#data_dir), [`data_dir`](#data_dir),
[`data_fsync`](#data_fsync), [`data_fsync`](#data_fsync),
[`db_engine`](#db_engine), [`db_engine`](#db_engine),
[`disable_scrub`](#disable_scrub), [`disable_scrub`](#disable_scrub),
[`use_local_tz`](#use_local_tz),
[`lmdb_map_size`](#lmdb_map_size), [`lmdb_map_size`](#lmdb_map_size),
[`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval), [`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval),
[`metadata_dir`](#metadata_dir), [`metadata_dir`](#metadata_dir),
[`metadata_fsync`](#metadata_fsync), [`metadata_fsync`](#metadata_fsync),
[`metadata_snapshots_dir`](#metadata_snapshots_dir), [`metadata_snapshots_dir`](#metadata_snapshots_dir),
[`replication_factor`](#replication_factor), [`replication_factor`](#replication_factor),
[`consistency_mode`](#consistency_mode),
[`rpc_bind_addr`](#rpc_bind_addr), [`rpc_bind_addr`](#rpc_bind_addr),
[`rpc_bind_outgoing`](#rpc_bind_outgoing), [`rpc_bind_outgoing`](#rpc_bind_outgoing),
[`rpc_public_addr`](#rpc_public_addr), [`rpc_public_addr`](#rpc_public_addr),
[`rpc_public_addr_subnet`](#rpc_public_addr_subnet) [`rpc_public_addr_subnet`](#rpc_public_addr_subnet)
[`rpc_secret`/`rpc_secret_file`](#rpc_secret). [`rpc_secret`/`rpc_secret_file`](#rpc_secret),
[`allow_punycode`](#allow_punycode). [`use_local_tz`](#use_local_tz).
The `[consul_discovery]` section: The `[consul_discovery]` section:
[`api`](#consul_api), [`api`](#consul_api),
@@ -153,13 +153,17 @@ The `[admin]` section:
### Environment variables {#env_variables} ### Environment variables {#env_variables}
The following configuration parameter must be specified as an environment The following configuration parameters must be specified as environment variables,
variable, it does not exist in the configuration file: they do not exist in the configuration file:
- `GARAGE_LOG_TO_SYSLOG` (since `v0.9.4`): set this to `1` or `true` to make the - `GARAGE_LOG_TO_SYSLOG` (since `v0.9.4`): set this to `1` or `true` to make the
Garage daemon send its logs to `syslog` (using the libc `syslog` function) Garage daemon send its logs to `syslog` (using the libc `syslog` function)
instead of printing to stderr. instead of printing to stderr.
- `GARAGE_LOG_TO_JOURNALD` (since `v1.2.0`): set this to `1` or `true` to make the
Garage daemon send its logs to `journald` (using the native protocol of `systemd-journald`)
instead of printing to stderr.
The following environment variables can be used to override the corresponding The following environment variables can be used to override the corresponding
values in the configuration file: values in the configuration file:
@@ -171,7 +175,7 @@ values in the configuration file:
### Top-level configuration options ### Top-level configuration options
#### `replication_factor` {#replication_factor} #### `replication_factor` (since `v1.0.0`) {#replication_factor}
The replication factor can be any positive integer smaller or equal the node count in your cluster. The replication factor can be any positive integer smaller or equal the node count in your cluster.
The chosen replication factor has a big impact on the cluster's failure tolerancy and performance characteristics. The chosen replication factor has a big impact on the cluster's failure tolerancy and performance characteristics.
@@ -219,7 +223,7 @@ is in progress. In theory, no data should be lost as rebalancing is a
routine operation for Garage, although we cannot guarantee you that everything routine operation for Garage, although we cannot guarantee you that everything
will go right in such an extreme scenario. will go right in such an extreme scenario.
#### `consistency_mode` {#consistency_mode} #### `consistency_mode` (since `v1.0.0`) {#consistency_mode}
The consistency mode setting determines the read and write behaviour of your cluster. The consistency mode setting determines the read and write behaviour of your cluster.
@@ -329,6 +333,7 @@ Since `v0.8.0`, Garage can use alternative storage backends as follows:
| --------- | ----------------- | ------------- | | --------- | ----------------- | ------------- |
| [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` | | [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
| [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` | | [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
| [Fjall](https://github.com/fjall-rs/fjall) (**experimental support** since `v1.3.0`) | `"fjall"` | `<metadata_dir>/db.fjall/` |
| [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` | | [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` |
Sled was supported until Garage v0.9.x, and was removed in Garage v1.0. Sled was supported until Garage v0.9.x, and was removed in Garage v1.0.
@@ -365,6 +370,14 @@ LMDB works very well, but is known to have the following limitations:
so it is not the best choice for high-performance storage clusters, so it is not the best choice for high-performance storage clusters,
but it should work fine in many cases. but it should work fine in many cases.
- Fjall: a storage engine based on LSM trees, which theoretically allow for
higher write throughput than other storage engines that are based on B-trees.
Using Fjall could potentially improve Garage's performance significantly in
write-heavy workloads. **Support for Fjall is experimental at this point**,
we have added it to Garage for evaluation purposes only. **Do not use it for
production-critical workloads.**
It is possible to convert Garage's metadata directory from one format to another It is possible to convert Garage's metadata directory from one format to another
using the `garage convert-db` command, which should be used as follows: using the `garage convert-db` command, which should be used as follows:
@@ -402,6 +415,7 @@ Here is how this option impacts the different database engines:
|----------|------------------------------------|-------------------------------| |----------|------------------------------------|-------------------------------|
| Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` | | Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` |
| LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` | | LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` |
| Fjall | default options | not supported |
Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`). Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).
@@ -23,7 +23,6 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the
- 2022-05-25 - Many Ceph S3 endpoints are not documented but implemented. Following a notification from the Ceph community, we added them. - 2022-05-25 - Many Ceph S3 endpoints are not documented but implemented. Following a notification from the Ceph community, we added them.
## High-level features ## High-level features
| Feature | Garage | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) | | Feature | Garage | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) |
@@ -34,6 +33,7 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the
| [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) | ✅ Implemented | ❌| ✅| ✅ | ✅ | | [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) | ✅ Implemented | ❌| ✅| ✅ | ✅ |
| [Presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html) | ✅ Implemented | ❌| ✅ | ✅ | ✅(❓) | | [Presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html) | ✅ Implemented | ❌| ✅ | ✅ | ✅(❓) |
| [SSE-C encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) | ✅ Implemented | ❓ | ✅ | ❌ | ✅ | | [SSE-C encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) | ✅ Implemented | ❓ | ✅ | ❌ | ✅ |
| [Bucket versioning](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html) | ❌ Missing | ✅ | ✅ | ❌ | ✅ |
*Note:* OpenIO does not says if it supports presigned URLs. Because it is part *Note:* OpenIO does not says if it supports presigned URLs. Because it is part
of signature v4 and they claim they support it without additional precisions, of signature v4 and they claim they support it without additional precisions,
+1 -1
View File
@@ -70,7 +70,7 @@ Example response body:
```json ```json
{ {
"node": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df", "node": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df",
"garageVersion": "v1.1.0", "garageVersion": "v1.2.0",
"garageFeatures": [ "garageFeatures": [
"k2v", "k2v",
"lmdb", "lmdb",
+3
View File
@@ -53,6 +53,9 @@
tests-sqlite = testWith { tests-sqlite = testWith {
GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite"; GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite";
}; };
tests-fjall = testWith {
GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall";
};
}; };
# ---- developpment shell, for making native builds only ---- # ---- developpment shell, for making native builds only ----
+2 -1
View File
@@ -68,12 +68,13 @@ let
rootFeatures = if features != null then rootFeatures = if features != null then
features features
else else
([ "bundled-libs" "lmdb" "sqlite" "k2v" ] ++ (lib.optionals release [ ([ "bundled-libs" "lmdb" "sqlite" "fjall" "k2v" ] ++ (lib.optionals release [
"consul-discovery" "consul-discovery"
"kubernetes-discovery" "kubernetes-discovery"
"metrics" "metrics"
"telemetry-otlp" "telemetry-otlp"
"syslog" "syslog"
"journald"
])); ]));
featuresStr = lib.concatStringsSep "," rootFeatures; featuresStr = lib.concatStringsSep "," rootFeatures;
+3 -3
View File
@@ -2,8 +2,8 @@ apiVersion: v2
name: garage name: garage
description: S3-compatible object store for small self-hosted geo-distributed deployments description: S3-compatible object store for small self-hosted geo-distributed deployments
type: application type: application
version: 0.7.0 version: 0.7.1
appVersion: "v1.1.0" appVersion: "v1.2.0"
home: https://garagehq.deuxfleurs.fr/ home: https://garagehq.deuxfleurs.fr/
icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg
@@ -15,4 +15,4 @@ keywords:
sources: sources:
- https://git.deuxfleurs.fr/Deuxfleurs/garage.git - https://git.deuxfleurs.fr/Deuxfleurs/garage.git
maintainers: [] maintainers: []
+1 -1
View File
@@ -1,6 +1,6 @@
# garage # garage
![Version: 0.7.0](https://img.shields.io/badge/Version-0.7.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.1.0](https://img.shields.io/badge/AppVersion-v1.1.0-informational?style=flat-square) ![Version: 0.7.1](https://img.shields.io/badge/Version-0.7.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.2.0](https://img.shields.io/badge/AppVersion-v1.2.0-informational?style=flat-square)
S3-compatible object store for small self-hosted geo-distributed deployments S3-compatible object store for small self-hosted geo-distributed deployments
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_api_admin" name = "garage_api_admin"
version = "1.1.0" version = "1.2.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
-6
View File
@@ -419,17 +419,11 @@ pub async fn handle_update_bucket(
if let Some(wa) = req.website_access { if let Some(wa) = req.website_access {
if wa.enabled { if wa.enabled {
let (redirect_all, routing_rules) = match state.website_config.get() {
Some(wc) => (wc.redirect_all.clone(), wc.routing_rules.clone()),
None => (None, Vec::new()),
};
state.website_config.update(Some(WebsiteConfig { state.website_config.update(Some(WebsiteConfig {
index_document: wa.index_document.ok_or_bad_request( index_document: wa.index_document.ok_or_bad_request(
"Please specify indexDocument when enabling website access.", "Please specify indexDocument when enabling website access.",
)?, )?,
error_document: wa.error_document, error_document: wa.error_document,
redirect_all,
routing_rules,
})); }));
} else { } else {
if wa.index_document.is_some() || wa.error_document.is_some() { if wa.index_document.is_some() || wa.error_document.is_some() {
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_api_common" name = "garage_api_common"
version = "1.1.0" version = "1.2.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+23 -11
View File
@@ -33,6 +33,7 @@ use garage_util::metrics::{gen_trace_id, RecordDuration};
use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_util::socket_address::UnixOrTCPSocketAddress;
use crate::helpers::{BoxBody, ErrorBody}; use crate::helpers::{BoxBody, ErrorBody};
use crate::signature::payload::Authorization;
pub trait ApiEndpoint: Send + Sync + 'static { pub trait ApiEndpoint: Send + Sync + 'static {
fn name(&self) -> &'static str; fn name(&self) -> &'static str;
@@ -58,6 +59,12 @@ pub trait ApiHandler: Send + Sync + 'static {
req: Request<IncomingBody>, req: Request<IncomingBody>,
endpoint: Self::Endpoint, endpoint: Self::Endpoint,
) -> impl Future<Output = Result<Response<BoxBody<Self::Error>>, Self::Error>> + Send; ) -> impl Future<Output = Result<Response<BoxBody<Self::Error>>, Self::Error>> + Send;
/// Returns the key id used to authenticate this request. The ID returned must be safe to
/// log.
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
None
}
} }
pub struct ApiServer<A: ApiHandler> { pub struct ApiServer<A: ApiHandler> {
@@ -142,19 +149,20 @@ impl<A: ApiHandler> ApiServer<A> {
) -> Result<Response<BoxBody<A::Error>>, http::Error> { ) -> Result<Response<BoxBody<A::Error>>, http::Error> {
let uri = req.uri().clone(); let uri = req.uri().clone();
if let Ok(forwarded_for_ip_addr) = let source = if let Ok(forwarded_for_ip_addr) =
forwarded_headers::handle_forwarded_for_headers(req.headers()) forwarded_headers::handle_forwarded_for_headers(req.headers())
{ {
info!( format!("{forwarded_for_ip_addr} (via {addr})")
"{} (via {}) {} {}",
forwarded_for_ip_addr,
addr,
req.method(),
uri
);
} else { } else {
info!("{} {} {}", addr, req.method(), uri); format!("{addr}")
} };
// we only do this to log the access key, so we can discard any error
let key = self
.api_handler
.key_id_from_request(&req)
.map(|k| format!("(key {k}) "))
.unwrap_or_default();
info!("{source} {key}{} {uri}", req.method());
debug!("{:?}", req); debug!("{:?}", req);
let tracer = opentelemetry::global::tracer("garage"); let tracer = opentelemetry::global::tracer("garage");
@@ -343,7 +351,11 @@ where
while !*must_exit.borrow() { while !*must_exit.borrow() {
let (stream, client_addr) = tokio::select! { let (stream, client_addr) = tokio::select! {
acc = listener.accept() => acc?, acc = listener.accept() => match acc {
Ok(r) => r,
Err(e) if e.kind() == std::io::ErrorKind::ConnectionAborted => continue,
Err(e) => return Err(e.into()),
},
_ = must_exit.changed() => continue, _ = must_exit.changed() => continue,
}; };
+2 -2
View File
@@ -417,7 +417,7 @@ pub async fn verify_v4(
// ============ Authorization header, or X-Amz-* query params ========= // ============ Authorization header, or X-Amz-* query params =========
pub struct Authorization { pub struct Authorization {
key_id: String, pub key_id: String,
scope: String, scope: String,
signed_headers: String, signed_headers: String,
signature: String, signature: String,
@@ -426,7 +426,7 @@ pub struct Authorization {
} }
impl Authorization { impl Authorization {
fn parse_header(headers: &HeaderMap) -> Result<Self, Error> { pub fn parse_header(headers: &HeaderMap) -> Result<Self, Error> {
let authorization = headers let authorization = headers
.get(AUTHORIZATION) .get(AUTHORIZATION)
.ok_or_bad_request("Missing authorization header")? .ok_or_bad_request("Missing authorization header")?
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_api_k2v" name = "garage_api_k2v"
version = "1.1.0" version = "1.2.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+6
View File
@@ -176,6 +176,12 @@ impl ApiHandler for K2VApiServer {
Ok(resp_ok) Ok(resp_ok)
} }
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
garage_api_common::signature::payload::Authorization::parse_header(req.headers())
.map(|auth| auth.key_id)
.ok()
}
} }
impl ApiEndpoint for K2VApiEndpoint { impl ApiEndpoint for K2VApiEndpoint {
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_api_s3" name = "garage_api_s3"
version = "1.1.0" version = "1.2.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+7
View File
@@ -226,6 +226,7 @@ impl ApiHandler for S3ApiServer {
Endpoint::DeleteBucket {} => handle_delete_bucket(ctx).await, Endpoint::DeleteBucket {} => handle_delete_bucket(ctx).await,
Endpoint::GetBucketLocation {} => handle_get_bucket_location(ctx), Endpoint::GetBucketLocation {} => handle_get_bucket_location(ctx),
Endpoint::GetBucketVersioning {} => handle_get_bucket_versioning(), Endpoint::GetBucketVersioning {} => handle_get_bucket_versioning(),
Endpoint::GetBucketAcl {} => handle_get_bucket_acl(ctx),
Endpoint::ListObjects { Endpoint::ListObjects {
delimiter, delimiter,
encoding_type, encoding_type,
@@ -342,6 +343,12 @@ impl ApiHandler for S3ApiServer {
Ok(resp_ok) Ok(resp_ok)
} }
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
garage_api_common::signature::payload::Authorization::parse_header(req.headers())
.map(|auth| auth.key_id)
.ok()
}
} }
impl ApiEndpoint for S3ApiEndpoint { impl ApiEndpoint for S3ApiEndpoint {
+59 -1
View File
@@ -5,7 +5,7 @@ use hyper::{Request, Response, StatusCode};
use garage_model::bucket_alias_table::*; use garage_model::bucket_alias_table::*;
use garage_model::bucket_table::Bucket; use garage_model::bucket_table::Bucket;
use garage_model::garage::Garage; use garage_model::garage::Garage;
use garage_model::key_table::Key; use garage_model::key_table::{Key, KeyParams};
use garage_model::permission::BucketKeyPerm; use garage_model::permission::BucketKeyPerm;
use garage_table::util::*; use garage_table::util::*;
use garage_util::crdt::*; use garage_util::crdt::*;
@@ -44,6 +44,55 @@ pub fn handle_get_bucket_versioning() -> Result<Response<ResBody>, Error> {
.body(string_body(xml))?) .body(string_body(xml))?)
} }
pub fn handle_get_bucket_acl(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
let ReqCtx {
bucket_id, api_key, ..
} = ctx;
let key_p = api_key.params().ok_or_internal_error(
"Key should not be in deleted state at this point (in handle_get_bucket_acl)",
)?;
let mut grants: Vec<s3_xml::Grant> = vec![];
let kp = api_key.bucket_permissions(&bucket_id);
if kp.allow_owner {
grants.push(s3_xml::Grant {
grantee: create_grantee(&key_p, &api_key),
permission: s3_xml::Value("FULL_CONTROL".to_string()),
});
} else {
if kp.allow_read {
grants.push(s3_xml::Grant {
grantee: create_grantee(&key_p, &api_key),
permission: s3_xml::Value("READ".to_string()),
});
grants.push(s3_xml::Grant {
grantee: create_grantee(&key_p, &api_key),
permission: s3_xml::Value("READ_ACP".to_string()),
});
}
if kp.allow_write {
grants.push(s3_xml::Grant {
grantee: create_grantee(&key_p, &api_key),
permission: s3_xml::Value("WRITE".to_string()),
});
}
}
let access_control_policy = s3_xml::AccessControlPolicy {
xmlns: (),
owner: None,
acl: s3_xml::AccessControlList { entries: grants },
};
let xml = s3_xml::to_xml_with_header(&access_control_policy)?;
trace!("xml: {}", xml);
Ok(Response::builder()
.header("Content-Type", "application/xml")
.body(string_body(xml))?)
}
pub async fn handle_list_buckets( pub async fn handle_list_buckets(
garage: &Garage, garage: &Garage,
api_key: &Key, api_key: &Key,
@@ -311,6 +360,15 @@ fn parse_create_bucket_xml(xml_bytes: &[u8]) -> Option<Option<String>> {
Some(ret) Some(ret)
} }
fn create_grantee(key_params: &KeyParams, api_key: &Key) -> s3_xml::Grantee {
s3_xml::Grantee {
xmlns_xsi: (),
typ: "CanonicalUser".to_string(),
display_name: Some(s3_xml::Value(key_params.name.get().to_string())),
id: Some(s3_xml::Value(api_key.key_id.to_string())),
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
+49 -28
View File
@@ -26,7 +26,7 @@ use garage_api_common::signature::checksum::*;
use crate::api_server::{ReqBody, ResBody}; use crate::api_server::{ReqBody, ResBody};
use crate::encryption::EncryptionParams; use crate::encryption::EncryptionParams;
use crate::error::*; use crate::error::*;
use crate::get::{full_object_byte_stream, PreconditionHeaders}; use crate::get::{check_version_not_deleted, full_object_byte_stream, PreconditionHeaders};
use crate::multipart; use crate::multipart;
use crate::put::{extract_metadata_headers, save_stream, ChecksumMode, SaveStreamResult}; use crate::put::{extract_metadata_headers, save_stream, ChecksumMode, SaveStreamResult};
use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION; use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION;
@@ -237,6 +237,7 @@ async fn handle_copy_metaonly(
.get(&source_version.uuid, &EmptyKey) .get(&source_version.uuid, &EmptyKey)
.await?; .await?;
let source_version = source_version.ok_or(Error::NoSuchKey)?; let source_version = source_version.ok_or(Error::NoSuchKey)?;
check_version_not_deleted(&source_version)?;
// Write an "uploading" marker in Object table // Write an "uploading" marker in Object table
// This holds a reference to the object in the Version table // This holds a reference to the object in the Version table
@@ -428,6 +429,7 @@ pub async fn handle_upload_part_copy(
.get(&source_object_version.uuid, &EmptyKey) .get(&source_object_version.uuid, &EmptyKey)
.await? .await?
.ok_or(Error::NoSuchKey)?; .ok_or(Error::NoSuchKey)?;
check_version_not_deleted(&source_version)?;
// We want to reuse blocks from the source version as much as possible. // We want to reuse blocks from the source version as much as possible.
// However, we still need to get the data from these blocks // However, we still need to get the data from these blocks
@@ -559,6 +561,7 @@ pub async fn handle_upload_part_copy(
let mut current_offset = 0; let mut current_offset = 0;
let mut next_block = defragmenter.next().await?; let mut next_block = defragmenter.next().await?;
let mut blocks_to_dup = dest_version.clone();
// TODO this could be optimized similarly to read_and_put_blocks // TODO this could be optimized similarly to read_and_put_blocks
// low priority because uploadpartcopy is rarely used // low priority because uploadpartcopy is rarely used
@@ -588,8 +591,7 @@ pub async fn handle_upload_part_copy(
.unwrap()?; .unwrap()?;
checksummer = checksummer_updated; checksummer = checksummer_updated;
dest_version.blocks.clear(); let (version_block_key, version_block) = (
dest_version.blocks.put(
VersionBlockKey { VersionBlockKey {
part_number, part_number,
offset: current_offset, offset: current_offset,
@@ -601,37 +603,56 @@ pub async fn handle_upload_part_copy(
); );
current_offset += data_len; current_offset += data_len;
let block_ref = BlockRef { let next = if let Some(final_data) = data_to_upload {
block: final_hash, dest_version.blocks.clear();
version: dest_version_id, dest_version.blocks.put(version_block_key, version_block);
deleted: false.into(), let block_ref = BlockRef {
block: final_hash,
version: dest_version_id,
deleted: false.into(),
};
let (_, _, _, next) = futures::try_join!(
// Thing 1: if the block is not exactly a block that existed before,
// we need to insert that data as a new block.
garage.block_manager.rpc_put_block(
final_hash,
final_data,
dest_encryption.is_encrypted(),
None
),
// Thing 2: we need to insert the block in the version
garage.version_table.insert(&dest_version),
// Thing 3: we need to add a block reference
garage.block_ref_table.insert(&block_ref),
// Thing 4: we need to read the next block
defragmenter.next(),
)?;
next
} else {
blocks_to_dup.blocks.put(version_block_key, version_block);
defragmenter.next().await?
}; };
let (_, _, _, next) = futures::try_join!(
// Thing 1: if the block is not exactly a block that existed before,
// we need to insert that data as a new block.
async {
if let Some(final_data) = data_to_upload {
garage
.block_manager
.rpc_put_block(final_hash, final_data, dest_encryption.is_encrypted(), None)
.await
} else {
Ok(())
}
},
// Thing 2: we need to insert the block in the version
garage.version_table.insert(&dest_version),
// Thing 3: we need to add a block reference
garage.block_ref_table.insert(&block_ref),
// Thing 4: we need to read the next block
defragmenter.next(),
)?;
next_block = next; next_block = next;
} }
assert_eq!(current_offset, source_range.length); assert_eq!(current_offset, source_range.length);
// Put the duplicated blocks into the version & block_refs tables
let block_refs_to_put = blocks_to_dup
.blocks
.items()
.iter()
.map(|b| BlockRef {
block: b.1.hash,
version: dest_version_id,
deleted: false.into(),
})
.collect::<Vec<_>>();
futures::try_join!(
garage.version_table.insert(&blocks_to_dup),
garage.block_ref_table.insert_many(&block_refs_to_put[..]),
)?;
let checksums = checksummer.finalize(); let checksums = checksummer.finalize();
let etag = dest_encryption.etag_from_md5(&checksums.md5); let etag = dest_encryption.etag_from_md5(&checksums.md5);
let checksum = checksums.extract(dest_object_checksum_algorithm); let checksum = checksums.extract(dest_object_checksum_algorithm);
+30 -2
View File
@@ -19,12 +19,13 @@ use garage_net::stream::ByteStream;
use garage_rpc::rpc_helper::OrderTag; use garage_rpc::rpc_helper::OrderTag;
use garage_table::EmptyKey; use garage_table::EmptyKey;
use garage_util::data::*; use garage_util::data::*;
use garage_util::error::OkOrMessage; use garage_util::error::{Error as UtilError, OkOrMessage};
use garage_model::garage::Garage; use garage_model::garage::Garage;
use garage_model::s3::object_table::*; use garage_model::s3::object_table::*;
use garage_model::s3::version_table::*; use garage_model::s3::version_table::*;
use garage_api_common::common_error::CommonError;
use garage_api_common::helpers::*; use garage_api_common::helpers::*;
use garage_api_common::signature::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE}; use garage_api_common::signature::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE};
@@ -215,6 +216,7 @@ pub async fn handle_head_without_ctx(
.get(&object_version.uuid, &EmptyKey) .get(&object_version.uuid, &EmptyKey)
.await? .await?
.ok_or(Error::NoSuchKey)?; .ok_or(Error::NoSuchKey)?;
check_version_not_deleted(&version)?;
let (part_offset, part_end) = let (part_offset, part_end) =
calculate_part_bounds(&version, pn).ok_or(Error::InvalidPart)?; calculate_part_bounds(&version, pn).ok_or(Error::InvalidPart)?;
@@ -365,6 +367,21 @@ pub async fn handle_get_without_ctx(
} }
} }
pub(crate) fn check_version_not_deleted(version: &Version) -> Result<(), Error> {
if version.deleted.get() {
// the version was deleted between when the object_table was consulted
// and now, this could mean the object was deleted, or overriden.
// Rather than say the key doesn't exist, return a transient error
// to signal the client to try again.
return Err(CommonError::InternalError(UtilError::Message(
"conflict/inconsistency between object and version state, version is deleted"
.to_string(),
))
.into());
}
Ok(())
}
async fn handle_get_full( async fn handle_get_full(
garage: Arc<Garage>, garage: Arc<Garage>,
version: &ObjectVersion, version: &ObjectVersion,
@@ -431,6 +448,7 @@ pub fn full_object_byte_stream(
.ok_or_message("channel closed")?; .ok_or_message("channel closed")?;
let version = version_fut.await.unwrap()?.ok_or(Error::NoSuchKey)?; let version = version_fut.await.unwrap()?.ok_or(Error::NoSuchKey)?;
check_version_not_deleted(&version)?;
for (i, (_, vb)) in version.blocks.items().iter().enumerate().skip(1) { for (i, (_, vb)) in version.blocks.items().iter().enumerate().skip(1) {
let stream_block_i = encryption let stream_block_i = encryption
.get_block(&garage, &vb.hash, Some(order_stream.order(i as u64))) .get_block(&garage, &vb.hash, Some(order_stream.order(i as u64)))
@@ -446,6 +464,14 @@ pub fn full_object_byte_stream(
{ {
Ok(()) => (), Ok(()) => (),
Err(e) => { Err(e) => {
// TODO i think this is a bad idea, we should log
// an error and stop there. If the error happens to
// be exactly the size of what hasn't been streamed
// yet, the client will see the request as a
// success
// instead truncating the output notify the client
// something happened with their download, so that
// they can retry it
let _ = tx.send(error_stream_item(e)).await; let _ = tx.send(error_stream_item(e)).await;
} }
} }
@@ -497,7 +523,7 @@ async fn handle_get_range(
.get(&version.uuid, &EmptyKey) .get(&version.uuid, &EmptyKey)
.await? .await?
.ok_or(Error::NoSuchKey)?; .ok_or(Error::NoSuchKey)?;
check_version_not_deleted(&version)?;
let body = let body =
body_from_blocks_range(garage, encryption, version.blocks.items(), begin, end); body_from_blocks_range(garage, encryption, version.blocks.items(), begin, end);
Ok(resp_builder.body(body)?) Ok(resp_builder.body(body)?)
@@ -548,6 +574,8 @@ async fn handle_get_part(
.await? .await?
.ok_or(Error::NoSuchKey)?; .ok_or(Error::NoSuchKey)?;
check_version_not_deleted(&version)?;
let (begin, end) = let (begin, end) =
calculate_part_bounds(&version, part_number).ok_or(Error::InvalidPart)?; calculate_part_bounds(&version, part_number).ok_or(Error::InvalidPart)?;
+54 -172
View File
@@ -3,7 +3,7 @@ use quick_xml::de::from_reader;
use hyper::{header::HeaderName, Request, Response, StatusCode}; use hyper::{header::HeaderName, Request, Response, StatusCode};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use garage_model::bucket_table::{self, *}; use garage_model::bucket_table::*;
use garage_api_common::helpers::*; use garage_api_common::helpers::*;
@@ -26,28 +26,7 @@ pub async fn handle_get_website(ctx: ReqCtx) -> Result<Response<ResBody>, Error>
suffix: Value(website.index_document.to_string()), suffix: Value(website.index_document.to_string()),
}), }),
redirect_all_requests_to: None, redirect_all_requests_to: None,
routing_rules: RoutingRules { routing_rules: None,
rules: website
.routing_rules
.clone()
.into_iter()
.map(|rule| RoutingRule {
condition: rule.condition.map(|cond| Condition {
http_error_code: cond.http_error_code.map(|c| IntValue(c as i64)),
prefix: cond.prefix.map(Value),
}),
redirect: Redirect {
hostname: rule.redirect.hostname.map(Value),
http_redirect_code: Some(IntValue(
rule.redirect.http_redirect_code as i64,
)),
protocol: rule.redirect.protocol.map(Value),
replace_full: rule.redirect.replace_key.map(Value),
replace_prefix: rule.redirect.replace_key_prefix.map(Value),
},
})
.collect(),
},
}; };
let xml = to_xml_with_header(&wc)?; let xml = to_xml_with_header(&wc)?;
Ok(Response::builder() Ok(Response::builder()
@@ -118,28 +97,18 @@ pub struct WebsiteConfiguration {
pub index_document: Option<Suffix>, pub index_document: Option<Suffix>,
#[serde(rename = "RedirectAllRequestsTo")] #[serde(rename = "RedirectAllRequestsTo")]
pub redirect_all_requests_to: Option<Target>, pub redirect_all_requests_to: Option<Target>,
#[serde( #[serde(rename = "RoutingRules")]
rename = "RoutingRules", pub routing_rules: Option<Vec<RoutingRule>>,
default,
skip_serializing_if = "RoutingRules::is_empty"
)]
pub routing_rules: RoutingRules,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Default)]
pub struct RoutingRules {
#[serde(rename = "RoutingRule")]
pub rules: Vec<RoutingRule>,
}
impl RoutingRules {
fn is_empty(&self) -> bool {
self.rules.is_empty()
}
} }
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
pub struct RoutingRule { pub struct RoutingRule {
#[serde(rename = "RoutingRule")]
pub inner: RoutingRuleInner,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
pub struct RoutingRuleInner {
#[serde(rename = "Condition")] #[serde(rename = "Condition")]
pub condition: Option<Condition>, pub condition: Option<Condition>,
#[serde(rename = "Redirect")] #[serde(rename = "Redirect")]
@@ -193,7 +162,7 @@ impl WebsiteConfiguration {
if self.redirect_all_requests_to.is_some() if self.redirect_all_requests_to.is_some()
&& (self.error_document.is_some() && (self.error_document.is_some()
|| self.index_document.is_some() || self.index_document.is_some()
|| !self.routing_rules.is_empty()) || self.routing_rules.is_some())
{ {
return Err(Error::bad_request( return Err(Error::bad_request(
"Bad XML: can't have RedirectAllRequestsTo and other fields", "Bad XML: can't have RedirectAllRequestsTo and other fields",
@@ -208,15 +177,10 @@ impl WebsiteConfiguration {
if let Some(ref rart) = self.redirect_all_requests_to { if let Some(ref rart) = self.redirect_all_requests_to {
rart.validate()?; rart.validate()?;
} }
for rr in &self.routing_rules.rules { if let Some(ref rrs) = self.routing_rules {
rr.validate()?; for rr in rrs {
} rr.inner.validate()?;
if self.routing_rules.rules.len() > 1000 { }
// we will do linear scans, best to avoid overly long configuration. The
// limit was choosen arbitrarily
return Err(Error::bad_request(
"Bad XML: RoutingRules can't have more than 1000 child elements",
));
} }
Ok(()) Ok(())
@@ -225,7 +189,11 @@ impl WebsiteConfiguration {
pub fn into_garage_website_config(self) -> Result<WebsiteConfig, Error> { pub fn into_garage_website_config(self) -> Result<WebsiteConfig, Error> {
if self.redirect_all_requests_to.is_some() { if self.redirect_all_requests_to.is_some() {
Err(Error::NotImplemented( Err(Error::NotImplemented(
"RedirectAllRequestsTo is not currently implemented in Garage, however its effect can be emulated using a single inconditional RoutingRule.".into(), "S3 website redirects are not currently implemented in Garage.".into(),
))
} else if self.routing_rules.map(|x| !x.is_empty()).unwrap_or(false) {
Err(Error::NotImplemented(
"S3 routing rules are not currently implemented in Garage.".into(),
)) ))
} else { } else {
Ok(WebsiteConfig { Ok(WebsiteConfig {
@@ -234,36 +202,6 @@ impl WebsiteConfiguration {
.map(|x| x.suffix.0) .map(|x| x.suffix.0)
.unwrap_or_else(|| "index.html".to_string()), .unwrap_or_else(|| "index.html".to_string()),
error_document: self.error_document.map(|x| x.key.0), error_document: self.error_document.map(|x| x.key.0),
redirect_all: None,
routing_rules: self
.routing_rules
.rules
.into_iter()
.map(|rule| {
bucket_table::RoutingRule {
condition: rule.condition.map(|condition| {
bucket_table::RedirectCondition {
http_error_code: condition.http_error_code.map(|c| c.0 as u16),
prefix: condition.prefix.map(|p| p.0),
}
}),
redirect: bucket_table::Redirect {
hostname: rule.redirect.hostname.map(|h| h.0),
protocol: rule.redirect.protocol.map(|p| p.0),
// aws default to 301, which i find punitive in case of
// missconfiguration (can be permanently cached on the
// user agent)
http_redirect_code: rule
.redirect
.http_redirect_code
.map(|c| c.0 as u16)
.unwrap_or(302),
replace_key_prefix: rule.redirect.replace_prefix.map(|k| k.0),
replace_key: rule.redirect.replace_full.map(|k| k.0),
},
}
})
.collect(),
}) })
} }
} }
@@ -304,69 +242,37 @@ impl Target {
} }
} }
impl RoutingRule { impl RoutingRuleInner {
pub fn validate(&self) -> Result<(), Error> { pub fn validate(&self) -> Result<(), Error> {
if let Some(condition) = &self.condition { let has_prefix = self
condition.validate()?; .condition
} .as_ref()
self.redirect.validate() .and_then(|c| c.prefix.as_ref())
} .is_some();
} self.redirect.validate(has_prefix)
impl Condition {
pub fn validate(&self) -> Result<bool, Error> {
if let Some(ref error_code) = self.http_error_code {
// TODO do other error codes make sense? Aws only allows 4xx and 5xx
if error_code.0 != 404 {
return Err(Error::bad_request(
"Bad XML: HttpErrorCodeReturnedEquals must be 404 or absent",
));
}
}
Ok(self.prefix.is_some())
} }
} }
impl Redirect { impl Redirect {
pub fn validate(&self) -> Result<(), Error> { pub fn validate(&self, has_prefix: bool) -> Result<(), Error> {
if self.replace_prefix.is_some() && self.replace_full.is_some() { if self.replace_prefix.is_some() {
return Err(Error::bad_request( if self.replace_full.is_some() {
"Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set", return Err(Error::bad_request(
)); "Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set",
));
}
if !has_prefix {
return Err(Error::bad_request(
"Bad XML: ReplaceKeyPrefixWith is set, but KeyPrefixEquals isn't",
));
}
} }
if let Some(ref protocol) = self.protocol { if let Some(ref protocol) = self.protocol {
if protocol.0 != "http" && protocol.0 != "https" { if protocol.0 != "http" && protocol.0 != "https" {
return Err(Error::bad_request("Bad XML: invalid protocol")); return Err(Error::bad_request("Bad XML: invalid protocol"));
} }
} }
if let Some(ref http_redirect_code) = self.http_redirect_code { // TODO there are probably more invalid cases, but which ones?
match http_redirect_code.0 {
// aws allows all 3xx except 300, but some are non-sensical (not modified,
// use proxy...)
301 | 302 | 303 | 307 | 308 => {
if self.hostname.is_none() && self.protocol.is_some() {
return Err(Error::bad_request(
"Bad XML: HostName must be set if Protocol is set",
));
}
}
// aws doesn't allow these codes, but netlify does, and it seems like a
// cool feature (change the page seen without changing the url shown by the
// user agent)
200 | 404 => {
if self.hostname.is_some() || self.protocol.is_some() {
// hostname would mean different bucket, protocol doesn't make
// sense
return Err(Error::bad_request(
"Bad XML: an HttpRedirectCode of 200 is not acceptable alongside HostName or Protocol",
));
}
}
_ => {
return Err(Error::bad_request("Bad XML: invalid HttpRedirectCode"));
}
}
}
Ok(()) Ok(())
} }
} }
@@ -405,15 +311,6 @@ mod tests {
<ReplaceKeyWith>fullkey</ReplaceKeyWith> <ReplaceKeyWith>fullkey</ReplaceKeyWith>
</Redirect> </Redirect>
</RoutingRule> </RoutingRule>
<RoutingRule>
<Condition>
<KeyPrefixEquals></KeyPrefixEquals>
</Condition>
<Redirect>
<HttpRedirectCode>404</HttpRedirectCode>
<ReplaceKeyWith>missing</ReplaceKeyWith>
</Redirect>
</RoutingRule>
</RoutingRules> </RoutingRules>
</WebsiteConfiguration>"#; </WebsiteConfiguration>"#;
let conf: WebsiteConfiguration = from_str(message).unwrap(); let conf: WebsiteConfiguration = from_str(message).unwrap();
@@ -429,36 +326,21 @@ mod tests {
hostname: Value("garage.tld".to_owned()), hostname: Value("garage.tld".to_owned()),
protocol: Some(Value("https".to_owned())), protocol: Some(Value("https".to_owned())),
}), }),
routing_rules: RoutingRules { routing_rules: Some(vec![RoutingRule {
rules: vec![ inner: RoutingRuleInner {
RoutingRule { condition: Some(Condition {
condition: Some(Condition { http_error_code: Some(IntValue(404)),
http_error_code: Some(IntValue(404)), prefix: Some(Value("prefix1".to_owned())),
prefix: Some(Value("prefix1".to_owned())), }),
}), redirect: Redirect {
redirect: Redirect { hostname: Some(Value("gara.ge".to_owned())),
hostname: Some(Value("gara.ge".to_owned())), protocol: Some(Value("http".to_owned())),
protocol: Some(Value("http".to_owned())), http_redirect_code: Some(IntValue(303)),
http_redirect_code: Some(IntValue(303)), replace_prefix: Some(Value("prefix2".to_owned())),
replace_prefix: Some(Value("prefix2".to_owned())), replace_full: Some(Value("fullkey".to_owned())),
replace_full: Some(Value("fullkey".to_owned())),
},
}, },
RoutingRule { },
condition: Some(Condition { }]),
http_error_code: None,
prefix: Some(Value("".to_owned())),
}),
redirect: Redirect {
hostname: None,
protocol: None,
http_redirect_code: Some(IntValue(404)),
replace_prefix: None,
replace_full: Some(Value("missing".to_owned())),
},
},
],
},
}; };
assert_eq! { assert_eq! {
ref_value, ref_value,
+77
View File
@@ -13,6 +13,10 @@ pub fn xmlns_tag<S: Serializer>(_v: &(), s: S) -> Result<S::Ok, S::Error> {
s.serialize_str("http://s3.amazonaws.com/doc/2006-03-01/") s.serialize_str("http://s3.amazonaws.com/doc/2006-03-01/")
} }
pub fn xmlns_xsi_tag<S: Serializer>(_v: &(), s: S) -> Result<S::Ok, S::Error> {
s.serialize_str("http://www.w3.org/2001/XMLSchema-instance")
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
pub struct Value(#[serde(rename = "$value")] pub String); pub struct Value(#[serde(rename = "$value")] pub String);
@@ -319,6 +323,42 @@ pub struct PostObject {
pub etag: Value, pub etag: Value,
} }
#[derive(Debug, Serialize, PartialEq, Eq)]
pub struct Grantee {
#[serde(rename = "xmlns:xsi", serialize_with = "xmlns_xsi_tag")]
pub xmlns_xsi: (),
#[serde(rename = "xsi:type")]
pub typ: String,
#[serde(rename = "DisplayName")]
pub display_name: Option<Value>,
#[serde(rename = "ID")]
pub id: Option<Value>,
}
#[derive(Debug, Serialize, PartialEq, Eq)]
pub struct Grant {
#[serde(rename = "Grantee")]
pub grantee: Grantee,
#[serde(rename = "Permission")]
pub permission: Value,
}
#[derive(Debug, Serialize, PartialEq, Eq)]
pub struct AccessControlList {
#[serde(rename = "Grant")]
pub entries: Vec<Grant>,
}
#[derive(Debug, Serialize, PartialEq, Eq)]
pub struct AccessControlPolicy {
#[serde(serialize_with = "xmlns_tag")]
pub xmlns: (),
#[serde(rename = "Owner")]
pub owner: Option<Owner>,
#[serde(rename = "AccessControlList")]
pub acl: AccessControlList,
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@@ -427,6 +467,43 @@ mod tests {
Ok(()) Ok(())
} }
#[test]
fn get_bucket_acl_result() -> Result<(), ApiError> {
let grant = Grant {
grantee: Grantee {
xmlns_xsi: (),
typ: "CanonicalUser".to_string(),
display_name: Some(Value("owner_name".to_string())),
id: Some(Value("qsdfjklm".to_string())),
},
permission: Value("FULL_CONTROL".to_string()),
};
let get_bucket_acl = AccessControlPolicy {
xmlns: (),
owner: None,
acl: AccessControlList {
entries: vec![grant],
},
};
assert_eq!(
to_xml_with_header(&get_bucket_acl)?,
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<AccessControlPolicy xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
<AccessControlList>\
<Grant>\
<Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"CanonicalUser\">\
<DisplayName>owner_name</DisplayName>\
<ID>qsdfjklm</ID>\
</Grantee>\
<Permission>FULL_CONTROL</Permission>\
</Grant>\
</AccessControlList>\
</AccessControlPolicy>"
);
Ok(())
}
#[test] #[test]
fn delete_result() -> Result<(), ApiError> { fn delete_result() -> Result<(), ApiError> {
let delete_result = DeleteResult { let delete_result = DeleteResult {
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_block" name = "garage_block"
version = "1.1.0" version = "1.2.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+3 -3
View File
@@ -408,8 +408,8 @@ impl BlockManager {
} }
/// Get number of items in the refcount table /// Get number of items in the refcount table
pub fn rc_len(&self) -> Result<usize, Error> { pub fn rc_approximate_len(&self) -> Result<usize, Error> {
Ok(self.rc.rc_table.len()?) Ok(self.rc.rc_table.approximate_len()?)
} }
/// Send command to start/stop/manager scrub worker /// Send command to start/stop/manager scrub worker
@@ -427,7 +427,7 @@ impl BlockManager {
/// List all resync errors /// List all resync errors
pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> { pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> {
let mut blocks = Vec::with_capacity(self.resync.errors.len()?); let mut blocks = Vec::with_capacity(self.resync.errors.approximate_len()?);
for ent in self.resync.errors.iter()? { for ent in self.resync.errors.iter()? {
let (hash, cnt) = ent?; let (hash, cnt) = ent?;
let cnt = ErrorCounter::decode(&cnt); let cnt = ErrorCounter::decode(&cnt);
+3 -3
View File
@@ -50,7 +50,7 @@ impl BlockManagerMetrics {
.init(), .init(),
_rc_size: meter _rc_size: meter
.u64_value_observer("block.rc_size", move |observer| { .u64_value_observer("block.rc_size", move |observer| {
if let Ok(value) = rc_tree.len() { if let Ok(value) = rc_tree.approximate_len() {
observer.observe(value as u64, &[]) observer.observe(value as u64, &[])
} }
}) })
@@ -58,7 +58,7 @@ impl BlockManagerMetrics {
.init(), .init(),
_resync_queue_len: meter _resync_queue_len: meter
.u64_value_observer("block.resync_queue_length", move |observer| { .u64_value_observer("block.resync_queue_length", move |observer| {
if let Ok(value) = resync_queue.len() { if let Ok(value) = resync_queue.approximate_len() {
observer.observe(value as u64, &[]); observer.observe(value as u64, &[]);
} }
}) })
@@ -68,7 +68,7 @@ impl BlockManagerMetrics {
.init(), .init(),
_resync_errored_blocks: meter _resync_errored_blocks: meter
.u64_value_observer("block.resync_errored_blocks", move |observer| { .u64_value_observer("block.resync_errored_blocks", move |observer| {
if let Ok(value) = resync_errors.len() { if let Ok(value) = resync_errors.approximate_len() {
observer.observe(value as u64, &[]); observer.observe(value as u64, &[]);
} }
}) })
+8 -6
View File
@@ -106,13 +106,13 @@ impl BlockResyncManager {
} }
/// Get length of resync queue /// Get length of resync queue
pub fn queue_len(&self) -> Result<usize, Error> { pub fn queue_approximate_len(&self) -> Result<usize, Error> {
Ok(self.queue.len()?) Ok(self.queue.approximate_len()?)
} }
/// Get number of blocks that have an error /// Get number of blocks that have an error
pub fn errors_len(&self) -> Result<usize, Error> { pub fn errors_approximate_len(&self) -> Result<usize, Error> {
Ok(self.errors.len()?) Ok(self.errors.approximate_len()?)
} }
/// Clear the error counter for a block and put it in queue immediately /// Clear the error counter for a block and put it in queue immediately
@@ -548,9 +548,11 @@ impl Worker for ResyncWorker {
} }
WorkerStatus { WorkerStatus {
queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64), queue_length: Some(self.manager.resync.queue_approximate_len().unwrap_or(0) as u64),
tranquility: Some(tranquility), tranquility: Some(tranquility),
persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64), persistent_errors: Some(
self.manager.resync.errors_approximate_len().unwrap_or(0) as u64
),
..Default::default() ..Default::default()
} }
} }
+6 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_db" name = "garage_db"
version = "1.1.0" version = "1.2.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@@ -16,10 +16,14 @@ err-derive.workspace = true
tracing.workspace = true tracing.workspace = true
heed = { workspace = true, optional = true } heed = { workspace = true, optional = true }
rusqlite = { workspace = true, optional = true, features = ["backup"] } rusqlite = { workspace = true, optional = true, features = ["backup"] }
r2d2 = { workspace = true, optional = true } r2d2 = { workspace = true, optional = true }
r2d2_sqlite = { workspace = true, optional = true } r2d2_sqlite = { workspace = true, optional = true }
fjall = { workspace = true, optional = true }
parking_lot = { workspace = true, optional = true }
[dev-dependencies] [dev-dependencies]
mktemp.workspace = true mktemp.workspace = true
@@ -27,4 +31,5 @@ mktemp.workspace = true
default = [ "lmdb", "sqlite" ] default = [ "lmdb", "sqlite" ]
bundled-libs = [ "rusqlite?/bundled" ] bundled-libs = [ "rusqlite?/bundled" ]
lmdb = [ "heed" ] lmdb = [ "heed" ]
fjall = [ "dep:fjall", "dep:parking_lot" ]
sqlite = [ "rusqlite", "r2d2", "r2d2_sqlite" ] sqlite = [ "rusqlite", "r2d2", "r2d2_sqlite" ]
+453
View File
@@ -0,0 +1,453 @@
use core::ops::Bound;
use std::path::PathBuf;
use std::sync::Arc;
use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard};
use fjall::{
PartitionCreateOptions, PersistMode, TransactionalKeyspace, TransactionalPartitionHandle,
WriteTransaction,
};
use crate::{
open::{Engine, OpenOpt},
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
TxResult, TxValueIter, Value, ValueIter,
};
pub use fjall;
// --
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
info!("Opening Fjall database at: {}", path.display());
if opt.fsync {
return Err(Error(
"metadata_fsync is not supported with the Fjall database engine".into(),
));
}
let mut config = fjall::Config::new(path);
if let Some(block_cache_size) = opt.fjall_block_cache_size {
config = config.cache_size(block_cache_size as u64);
}
let keyspace = config.open_transactional()?;
Ok(FjallDb::init(keyspace))
}
// -- err
impl From<fjall::Error> for Error {
fn from(e: fjall::Error) -> Error {
Error(format!("fjall: {}", e).into())
}
}
impl From<fjall::LsmError> for Error {
fn from(e: fjall::LsmError) -> Error {
Error(format!("fjall lsm_tree: {}", e).into())
}
}
impl From<fjall::Error> for TxOpError {
fn from(e: fjall::Error) -> TxOpError {
TxOpError(e.into())
}
}
// -- db
pub struct FjallDb {
keyspace: TransactionalKeyspace,
trees: RwLock<Vec<(String, TransactionalPartitionHandle)>>,
}
type ByteRefRangeBound<'r> = (Bound<&'r [u8]>, Bound<&'r [u8]>);
impl FjallDb {
pub fn init(keyspace: TransactionalKeyspace) -> Db {
let s = Self {
keyspace,
trees: RwLock::new(Vec::new()),
};
Db(Arc::new(s))
}
fn get_tree(
&self,
i: usize,
) -> Result<MappedRwLockReadGuard<'_, TransactionalPartitionHandle>> {
RwLockReadGuard::try_map(self.trees.read(), |trees: &Vec<_>| {
trees.get(i).map(|tup| &tup.1)
})
.map_err(|_| Error("invalid tree id".into()))
}
}
impl IDb for FjallDb {
fn engine(&self) -> String {
"Fjall (EXPERIMENTAL!)".into()
}
fn open_tree(&self, name: &str) -> Result<usize> {
let mut trees = self.trees.write();
let safe_name = encode_name(name)?;
if let Some(i) = trees.iter().position(|(name, _)| *name == safe_name) {
Ok(i)
} else {
let tree = self
.keyspace
.open_partition(&safe_name, PartitionCreateOptions::default())?;
let i = trees.len();
trees.push((safe_name, tree));
Ok(i)
}
}
fn list_trees(&self) -> Result<Vec<String>> {
Ok(self
.keyspace
.list_partitions()
.iter()
.map(|n| decode_name(&n))
.collect::<Result<Vec<_>>>()?)
}
fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
std::fs::create_dir_all(base_path)?;
let path = Engine::Fjall.db_path(base_path);
let source_state = self.keyspace.read_tx();
let copy_keyspace = fjall::Config::new(path).open()?;
for partition_name in self.keyspace.list_partitions() {
let source_partition = self
.keyspace
.open_partition(&partition_name, PartitionCreateOptions::default())?;
let copy_partition =
copy_keyspace.open_partition(&partition_name, PartitionCreateOptions::default())?;
for entry in source_state.iter(&source_partition) {
let (key, value) = entry?;
copy_partition.insert(key, value)?;
}
}
copy_keyspace.persist(PersistMode::SyncAll)?;
Ok(())
}
// ----
fn get(&self, tree_idx: usize, key: &[u8]) -> Result<Option<Value>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
let val = tx.get(&tree, key)?;
match val {
None => Ok(None),
Some(v) => Ok(Some(v.to_vec())),
}
}
fn approximate_len(&self, tree_idx: usize) -> Result<usize> {
let tree = self.get_tree(tree_idx)?;
Ok(tree.approximate_len())
}
fn is_empty(&self, tree_idx: usize) -> Result<bool> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(tx.is_empty(&tree)?)
}
fn insert(&self, tree_idx: usize, key: &[u8], value: &[u8]) -> Result<()> {
let tree = self.get_tree(tree_idx)?;
let mut tx = self.keyspace.write_tx();
tx.insert(&tree, key, value);
tx.commit()?;
Ok(())
}
fn remove(&self, tree_idx: usize, key: &[u8]) -> Result<()> {
let tree = self.get_tree(tree_idx)?;
let mut tx = self.keyspace.write_tx();
tx.remove(&tree, key);
tx.commit()?;
Ok(())
}
fn clear(&self, tree_idx: usize) -> Result<()> {
let mut trees = self.trees.write();
if tree_idx >= trees.len() {
return Err(Error("invalid tree id".into()));
}
let (name, tree) = trees.remove(tree_idx);
self.keyspace.delete_partition(tree)?;
let tree = self
.keyspace
.open_partition(&name, PartitionCreateOptions::default())?;
trees.insert(tree_idx, (name, tree));
Ok(())
}
fn iter(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(tx.iter(&tree).map(iterator_remap)))
}
fn iter_rev(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(tx.iter(&tree).rev().map(iterator_remap)))
}
fn range<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(
tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high))
.map(iterator_remap),
))
}
fn range_rev<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let tx = self.keyspace.read_tx();
Ok(Box::new(
tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high))
.rev()
.map(iterator_remap),
))
}
// ----
fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()> {
let trees = self.trees.read();
let mut tx = FjallTx {
trees: &trees[..],
tx: self.keyspace.write_tx(),
};
let res = f.try_on(&mut tx);
match res {
TxFnResult::Ok(on_commit) => {
tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?;
Ok(on_commit)
}
TxFnResult::Abort => {
tx.tx.rollback();
Err(TxError::Abort(()))
}
TxFnResult::DbErr => {
tx.tx.rollback();
Err(TxError::Db(Error(
"(this message will be discarded)".into(),
)))
}
}
}
}
// ----
struct FjallTx<'a> {
trees: &'a [(String, TransactionalPartitionHandle)],
tx: WriteTransaction<'a>,
}
impl<'a> FjallTx<'a> {
fn get_tree(&self, i: usize) -> TxOpResult<&TransactionalPartitionHandle> {
self.trees.get(i).map(|tup| &tup.1).ok_or_else(|| {
TxOpError(Error(
"invalid tree id (it might have been openned after the transaction started)".into(),
))
})
}
}
impl<'a> ITx for FjallTx<'a> {
fn get(&self, tree_idx: usize, key: &[u8]) -> TxOpResult<Option<Value>> {
let tree = self.get_tree(tree_idx)?;
match self.tx.get(tree, key)? {
Some(v) => Ok(Some(v.to_vec())),
None => Ok(None),
}
}
fn len(&self, tree_idx: usize) -> TxOpResult<usize> {
let tree = self.get_tree(tree_idx)?;
Ok(self.tx.len(tree)? as usize)
}
fn insert(&mut self, tree_idx: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> {
let tree = self.get_tree(tree_idx)?.clone();
self.tx.insert(&tree, key, value);
Ok(())
}
fn remove(&mut self, tree_idx: usize, key: &[u8]) -> TxOpResult<()> {
let tree = self.get_tree(tree_idx)?.clone();
self.tx.remove(&tree, key);
Ok(())
}
fn clear(&mut self, _tree_idx: usize) -> TxOpResult<()> {
unimplemented!("LSM tree clearing in cross-partition transaction is not supported")
}
fn iter(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?.clone();
Ok(Box::new(self.tx.iter(&tree).map(iterator_remap_tx)))
}
fn iter_rev(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?.clone();
Ok(Box::new(self.tx.iter(&tree).rev().map(iterator_remap_tx)))
}
fn range<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let low = clone_bound(low);
let high = clone_bound(high);
Ok(Box::new(
self.tx
.range::<Vec<u8>, ByteVecRangeBounds>(&tree, (low, high))
.map(iterator_remap_tx),
))
}
fn range_rev<'r>(
&self,
tree_idx: usize,
low: Bound<&'r [u8]>,
high: Bound<&'r [u8]>,
) -> TxOpResult<TxValueIter<'_>> {
let tree = self.get_tree(tree_idx)?;
let low = clone_bound(low);
let high = clone_bound(high);
Ok(Box::new(
self.tx
.range::<Vec<u8>, ByteVecRangeBounds>(&tree, (low, high))
.rev()
.map(iterator_remap_tx),
))
}
}
// -- maps fjall's (k, v) to ours
fn iterator_remap(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> Result<(Value, Value)> {
r.map(|(k, v)| (k.to_vec(), v.to_vec()))
.map_err(|e| e.into())
}
fn iterator_remap_tx(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> TxOpResult<(Value, Value)> {
r.map(|(k, v)| (k.to_vec(), v.to_vec()))
.map_err(|e| e.into())
}
// -- utils to deal with Garage's tightness on Bound lifetimes
type ByteVecBound = Bound<Vec<u8>>;
type ByteVecRangeBounds = (ByteVecBound, ByteVecBound);
fn clone_bound(bound: Bound<&[u8]>) -> ByteVecBound {
let value = match bound {
Bound::Excluded(v) | Bound::Included(v) => v.to_vec(),
Bound::Unbounded => vec![],
};
match bound {
Bound::Included(_) => Bound::Included(value),
Bound::Excluded(_) => Bound::Excluded(value),
Bound::Unbounded => Bound::Unbounded,
}
}
// -- utils to encode table names --
fn encode_name(s: &str) -> Result<String> {
let base = 'A' as u32;
let mut ret = String::with_capacity(s.len() + 10);
for c in s.chars() {
if c.is_alphanumeric() || c == '_' || c == '-' || c == '#' {
ret.push(c);
} else if c <= u8::MAX as char {
ret.push('$');
let c_hi = c as u32 / 16;
let c_lo = c as u32 % 16;
ret.push(char::from_u32(base + c_hi).unwrap());
ret.push(char::from_u32(base + c_lo).unwrap());
} else {
return Err(Error(
format!("table name {} could not be safely encoded", s).into(),
));
}
}
Ok(ret)
}
fn decode_name(s: &str) -> Result<String> {
use std::convert::TryFrom;
let errfn = || Error(format!("encoded table name {} is invalid", s).into());
let c_map = |c: char| {
let c = c as u32;
let base = 'A' as u32;
if (base..base + 16).contains(&c) {
Some(c - base)
} else {
None
}
};
let mut ret = String::with_capacity(s.len());
let mut it = s.chars();
while let Some(c) = it.next() {
if c == '$' {
let c_hi = it.next().and_then(c_map).ok_or_else(errfn)?;
let c_lo = it.next().and_then(c_map).ok_or_else(errfn)?;
let c_dec = char::try_from(c_hi * 16 + c_lo).map_err(|_| errfn())?;
ret.push(c_dec);
} else {
ret.push(c);
}
}
Ok(ret)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_encdec_name() {
for name in [
"testname",
"test_name",
"test name",
"test$name",
"test:name@help.me$get/this**right",
] {
let encname = encode_name(name).unwrap();
assert!(!encname.contains(' '));
assert!(!encname.contains('.'));
assert!(!encname.contains('*'));
assert_eq!(*name, decode_name(&encname).unwrap());
}
}
}
+11 -4
View File
@@ -1,6 +1,8 @@
#[macro_use] #[macro_use]
extern crate tracing; extern crate tracing;
#[cfg(feature = "fjall")]
pub mod fjall_adapter;
#[cfg(feature = "lmdb")] #[cfg(feature = "lmdb")]
pub mod lmdb_adapter; pub mod lmdb_adapter;
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]
@@ -152,7 +154,7 @@ impl Db {
let tree_names = other.list_trees()?; let tree_names = other.list_trees()?;
for name in tree_names { for name in tree_names {
let tree = self.open_tree(&name)?; let tree = self.open_tree(&name)?;
if tree.len()? > 0 { if !tree.is_empty()? {
return Err(Error(format!("tree {} already contains data", name).into())); return Err(Error(format!("tree {} already contains data", name).into()));
} }
@@ -194,8 +196,12 @@ impl Tree {
self.0.get(self.1, key.as_ref()) self.0.get(self.1, key.as_ref())
} }
#[inline] #[inline]
pub fn len(&self) -> Result<usize> { pub fn approximate_len(&self) -> Result<usize> {
self.0.len(self.1) self.0.approximate_len(self.1)
}
#[inline]
pub fn is_empty(&self) -> Result<bool> {
self.0.is_empty(self.1)
} }
#[inline] #[inline]
@@ -333,7 +339,8 @@ pub(crate) trait IDb: Send + Sync {
fn snapshot(&self, path: &PathBuf) -> Result<()>; fn snapshot(&self, path: &PathBuf) -> Result<()>;
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>; fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
fn len(&self, tree: usize) -> Result<usize>; fn approximate_len(&self, tree: usize) -> Result<usize>;
fn is_empty(&self, tree: usize) -> Result<bool>;
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>; fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>;
fn remove(&self, tree: usize, key: &[u8]) -> Result<()>; fn remove(&self, tree: usize, key: &[u8]) -> Result<()>;
+94 -29
View File
@@ -1,8 +1,8 @@
use core::ops::Bound; use core::ops::Bound;
use core::ptr::NonNull;
use std::collections::HashMap; use std::collections::HashMap;
use std::convert::TryInto; use std::convert::TryInto;
use std::marker::PhantomPinned;
use std::path::PathBuf; use std::path::PathBuf;
use std::pin::Pin; use std::pin::Pin;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
@@ -11,12 +11,55 @@ use heed::types::ByteSlice;
use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database}; use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database};
use crate::{ use crate::{
open::{Engine, OpenOpt},
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult, Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
TxResult, TxValueIter, Value, ValueIter, TxResult, TxValueIter, Value, ValueIter,
}; };
pub use heed; pub use heed;
// ---- top-level open function
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
info!("Opening LMDB database at: {}", path.display());
if let Err(e) = std::fs::create_dir_all(&path) {
return Err(Error(
format!("Unable to create LMDB data directory: {}", e).into(),
));
}
let map_size = match opt.lmdb_map_size {
None => recommended_map_size(),
Some(v) => v - (v % 4096),
};
let mut env_builder = heed::EnvOpenOptions::new();
env_builder.max_dbs(100);
env_builder.map_size(map_size);
env_builder.max_readers(2048);
unsafe {
env_builder.flag(heed::flags::Flags::MdbNoRdAhead);
env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
if !opt.fsync {
env_builder.flag(heed::flags::Flags::MdbNoSync);
}
}
match env_builder.open(&path) {
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
return Err(Error(
"OutOfMemory error while trying to open LMDB database. This can happen \
if your operating system is not allowing you to use sufficient virtual \
memory address space. Please check that no limit is set (ulimit -v). \
You may also try to set a smaller `lmdb_map_size` configuration parameter. \
On 32-bit machines, you should probably switch to another database engine."
.into(),
))
}
Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
Ok(db) => Ok(LmdbDb::init(db)),
}
}
// -- err // -- err
impl From<heed::Error> for Error { impl From<heed::Error> for Error {
@@ -104,10 +147,9 @@ impl IDb for LmdbDb {
Ok(ret2) Ok(ret2)
} }
fn snapshot(&self, to: &PathBuf) -> Result<()> { fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
std::fs::create_dir_all(to)?; std::fs::create_dir_all(base_path)?;
let mut path = to.clone(); let path = Engine::Lmdb.db_path(base_path);
path.push("data.mdb");
self.db self.db
.copy_to_path(path, heed::CompactionOption::Enabled)?; .copy_to_path(path, heed::CompactionOption::Enabled)?;
Ok(()) Ok(())
@@ -126,11 +168,16 @@ impl IDb for LmdbDb {
} }
} }
fn len(&self, tree: usize) -> Result<usize> { fn approximate_len(&self, tree: usize) -> Result<usize> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?; let tx = self.db.read_txn()?;
Ok(tree.len(&tx)?.try_into().unwrap()) Ok(tree.len(&tx)?.try_into().unwrap())
} }
fn is_empty(&self, tree: usize) -> Result<bool> {
let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?;
Ok(tree.is_empty(&tx)?)
}
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
@@ -159,13 +206,15 @@ impl IDb for LmdbDb {
fn iter(&self, tree: usize) -> Result<ValueIter<'_>> { fn iter(&self, tree: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?; let tx = self.db.read_txn()?;
TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) // Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) }
} }
fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> { fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?; let tx = self.db.read_txn()?;
TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?)) // Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?)) }
} }
fn range<'r>( fn range<'r>(
@@ -176,7 +225,8 @@ impl IDb for LmdbDb {
) -> Result<ValueIter<'_>> { ) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?; let tx = self.db.read_txn()?;
TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?)) // Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?)) }
} }
fn range_rev<'r>( fn range_rev<'r>(
&self, &self,
@@ -186,7 +236,8 @@ impl IDb for LmdbDb {
) -> Result<ValueIter<'_>> { ) -> Result<ValueIter<'_>> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
let tx = self.db.read_txn()?; let tx = self.db.read_txn()?;
TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?)) // Safety: the cloture does not store its argument anywhere,
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?)) }
} }
// ---- // ----
@@ -316,28 +367,41 @@ where
{ {
tx: RoTxn<'a>, tx: RoTxn<'a>,
iter: Option<I>, iter: Option<I>,
_pin: PhantomPinned,
} }
impl<'a, I> TxAndIterator<'a, I> impl<'a, I> TxAndIterator<'a, I>
where where
I: Iterator<Item = IteratorItem<'a>> + 'a, I: Iterator<Item = IteratorItem<'a>> + 'a,
{ {
fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>> fn iter(self: Pin<&mut Self>) -> &mut Option<I> {
// Safety: iter is not structural
unsafe { &mut self.get_unchecked_mut().iter }
}
/// Safety: iterfun must not store its argument anywhere but in its result.
unsafe fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>>
where where
F: FnOnce(&'a RoTxn<'a>) -> Result<I>, F: FnOnce(&'a RoTxn<'a>) -> Result<I>,
{ {
let res = TxAndIterator { tx, iter: None }; let res = TxAndIterator {
tx,
iter: None,
_pin: PhantomPinned,
};
let mut boxed = Box::pin(res); let mut boxed = Box::pin(res);
// This unsafe allows us to bypass lifetime checks let tx_lifetime_overextended: &'a RoTxn<'a> = {
let tx = unsafe { NonNull::from(&boxed.tx).as_ref() }; let tx = &boxed.tx;
let iter = iterfun(tx)?; // Safety: Artificially extending the lifetime because
// this reference will only be stored and accessed from the
// returned ValueIter which guarantees that it is destroyed
// before the tx it is pointing to.
unsafe { &*&raw const *tx }
};
let iter = iterfun(&tx_lifetime_overextended)?;
let mut_ref = Pin::as_mut(&mut boxed); *boxed.as_mut().iter() = Some(iter);
// This unsafe allows us to write in a field of the pinned struct
unsafe {
Pin::get_unchecked_mut(mut_ref).iter = Some(iter);
}
Ok(Box::new(TxAndIteratorPin(boxed))) Ok(Box::new(TxAndIteratorPin(boxed)))
} }
@@ -348,8 +412,10 @@ where
I: Iterator<Item = IteratorItem<'a>> + 'a, I: Iterator<Item = IteratorItem<'a>> + 'a,
{ {
fn drop(&mut self) { fn drop(&mut self) {
// ensure the iterator is dropped before the RoTxn it references // Safety: `new_unchecked` is okay because we know this value is never
drop(self.iter.take()); // used again after being dropped.
let this = unsafe { Pin::new_unchecked(self) };
drop(this.iter().take());
} }
} }
@@ -365,13 +431,12 @@ where
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
let mut_ref = Pin::as_mut(&mut self.0); let mut_ref = Pin::as_mut(&mut self.0);
// This unsafe allows us to mutably access the iterator field let next = mut_ref.iter().as_mut()?.next()?;
let next = unsafe { Pin::get_unchecked_mut(mut_ref).iter.as_mut()?.next() }; let res = match next {
match next { Err(e) => Err(e.into()),
None => None, Ok((k, v)) => Ok((k.to_vec(), v.to_vec())),
Some(Err(e)) => Some(Err(e.into())), };
Some(Ok((k, v))) => Some(Ok((k.to_vec(), v.to_vec()))), Some(res)
}
} }
} }
+28 -44
View File
@@ -11,6 +11,7 @@ use crate::{Db, Error, Result};
pub enum Engine { pub enum Engine {
Lmdb, Lmdb,
Sqlite, Sqlite,
Fjall,
} }
impl Engine { impl Engine {
@@ -19,8 +20,26 @@ impl Engine {
match self { match self {
Self::Lmdb => "lmdb", Self::Lmdb => "lmdb",
Self::Sqlite => "sqlite", Self::Sqlite => "sqlite",
Self::Fjall => "fjall",
} }
} }
/// Return engine-specific DB path from base path
pub fn db_path(&self, base_path: &PathBuf) -> PathBuf {
let mut ret = base_path.clone();
match self {
Self::Lmdb => {
ret.push("db.lmdb");
}
Self::Sqlite => {
ret.push("db.sqlite");
}
Self::Fjall => {
ret.push("db.fjall");
}
}
ret
}
} }
impl std::fmt::Display for Engine { impl std::fmt::Display for Engine {
@@ -36,10 +55,11 @@ impl std::str::FromStr for Engine {
match text { match text {
"lmdb" | "heed" => Ok(Self::Lmdb), "lmdb" | "heed" => Ok(Self::Lmdb),
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite), "sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
"fjall" => Ok(Self::Fjall),
"sled" => Err(Error("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into())), "sled" => Err(Error("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into())),
kind => Err(Error( kind => Err(Error(
format!( format!(
"Invalid DB engine: {} (options are: lmdb, sqlite)", "Invalid DB engine: {} (options are: lmdb, sqlite, fjall)",
kind kind
) )
.into(), .into(),
@@ -51,6 +71,7 @@ impl std::str::FromStr for Engine {
pub struct OpenOpt { pub struct OpenOpt {
pub fsync: bool, pub fsync: bool,
pub lmdb_map_size: Option<usize>, pub lmdb_map_size: Option<usize>,
pub fjall_block_cache_size: Option<usize>,
} }
impl Default for OpenOpt { impl Default for OpenOpt {
@@ -58,6 +79,7 @@ impl Default for OpenOpt {
Self { Self {
fsync: false, fsync: false,
lmdb_map_size: None, lmdb_map_size: None,
fjall_block_cache_size: None,
} }
} }
} }
@@ -66,53 +88,15 @@ pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
match engine { match engine {
// ---- Sqlite DB ---- // ---- Sqlite DB ----
#[cfg(feature = "sqlite")] #[cfg(feature = "sqlite")]
Engine::Sqlite => { Engine::Sqlite => crate::sqlite_adapter::open_db(path, opt),
info!("Opening Sqlite database at: {}", path.display());
let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
Ok(crate::sqlite_adapter::SqliteDb::new(manager, opt.fsync)?)
}
// ---- LMDB DB ---- // ---- LMDB DB ----
#[cfg(feature = "lmdb")] #[cfg(feature = "lmdb")]
Engine::Lmdb => { Engine::Lmdb => crate::lmdb_adapter::open_db(path, opt),
info!("Opening LMDB database at: {}", path.display());
if let Err(e) = std::fs::create_dir_all(&path) {
return Err(Error(
format!("Unable to create LMDB data directory: {}", e).into(),
));
}
let map_size = match opt.lmdb_map_size { // ---- Fjall DB ----
None => crate::lmdb_adapter::recommended_map_size(), #[cfg(feature = "fjall")]
Some(v) => v - (v % 4096), Engine::Fjall => crate::fjall_adapter::open_db(path, opt),
};
let mut env_builder = heed::EnvOpenOptions::new();
env_builder.max_dbs(100);
env_builder.map_size(map_size);
env_builder.max_readers(2048);
unsafe {
env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoRdAhead);
env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
if !opt.fsync {
env_builder.flag(heed::flags::Flags::MdbNoSync);
}
}
match env_builder.open(&path) {
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
return Err(Error(
"OutOfMemory error while trying to open LMDB database. This can happen \
if your operating system is not allowing you to use sufficient virtual \
memory address space. Please check that no limit is set (ulimit -v). \
You may also try to set a smaller `lmdb_map_size` configuration parameter. \
On 32-bit machines, you should probably switch to another database engine."
.into(),
))
}
Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
Ok(db) => Ok(crate::lmdb_adapter::LmdbDb::init(db)),
}
}
// Pattern is unreachable when all supported DB engines are compiled into binary. The allow // Pattern is unreachable when all supported DB engines are compiled into binary. The allow
// attribute is added so that we won't have to change this match in case stop building // attribute is added so that we won't have to change this match in case stop building
+37 -7
View File
@@ -11,12 +11,23 @@ use r2d2_sqlite::SqliteConnectionManager;
use rusqlite::{params, Rows, Statement, Transaction}; use rusqlite::{params, Rows, Statement, Transaction};
use crate::{ use crate::{
open::{Engine, OpenOpt},
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult, Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
TxResult, TxValueIter, Value, ValueIter, TxResult, TxValueIter, Value, ValueIter,
}; };
pub use rusqlite; pub use rusqlite;
// ---- top-level open function
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
info!("Opening Sqlite database at: {}", path.display());
let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
Ok(SqliteDb::new(manager, opt.fsync)?)
}
// ----
type Connection = r2d2::PooledConnection<SqliteConnectionManager>; type Connection = r2d2::PooledConnection<SqliteConnectionManager>;
// --- err // --- err
@@ -139,17 +150,32 @@ impl IDb for SqliteDb {
Ok(trees) Ok(trees)
} }
fn snapshot(&self, to: &PathBuf) -> Result<()> { fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
fn progress(p: rusqlite::backup::Progress) { fn progress(p: rusqlite::backup::Progress) {
let percent = (p.pagecount - p.remaining) * 100 / p.pagecount; use std::sync::atomic::{AtomicU64, Ordering};
info!("Sqlite snapshot progress: {}%", percent); use std::time::{SystemTime, UNIX_EPOCH};
static LAST_LOG_TIME: AtomicU64 = AtomicU64::new(0);
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Fix your clock :o")
.as_millis() as u64;
if now >= LAST_LOG_TIME.load(Ordering::Relaxed) + 10 * 1000 {
let percent = (p.pagecount - p.remaining) * 100 / p.pagecount;
info!("Sqlite snapshot progress: {}%", percent);
LAST_LOG_TIME.fetch_max(now, Ordering::Relaxed);
}
} }
std::fs::create_dir_all(to)?;
let mut path = to.clone(); std::fs::create_dir_all(base_path)?;
path.push("db.sqlite"); let path = Engine::Sqlite.db_path(&base_path);
self.db self.db
.get()? .get()?
.backup(rusqlite::DatabaseName::Main, path, Some(progress))?; .backup(rusqlite::DatabaseName::Main, path, Some(progress))?;
Ok(()) Ok(())
} }
@@ -160,7 +186,7 @@ impl IDb for SqliteDb {
self.internal_get(&self.db.get()?, &tree, key) self.internal_get(&self.db.get()?, &tree, key)
} }
fn len(&self, tree: usize) -> Result<usize> { fn approximate_len(&self, tree: usize) -> Result<usize> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
let db = self.db.get()?; let db = self.db.get()?;
@@ -172,6 +198,10 @@ impl IDb for SqliteDb {
} }
} }
fn is_empty(&self, tree: usize) -> Result<bool> {
Ok(self.approximate_len(tree)? == 0)
}
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> { fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
let tree = self.get_tree(tree)?; let tree = self.get_tree(tree)?;
let db = self.db.get()?; let db = self.db.get()?;
+14 -2
View File
@@ -1,7 +1,7 @@
use crate::*; use crate::*;
fn test_suite(db: Db) { fn test_suite(db: Db) {
let tree = db.open_tree("tree").unwrap(); let tree = db.open_tree("tree:this_is_a_tree").unwrap();
let ka: &[u8] = &b"test"[..]; let ka: &[u8] = &b"test"[..];
let kb: &[u8] = &b"zwello"[..]; let kb: &[u8] = &b"zwello"[..];
@@ -14,7 +14,7 @@ fn test_suite(db: Db) {
assert!(tree.insert(ka, va).is_ok()); assert!(tree.insert(ka, va).is_ok());
assert_eq!(tree.get(ka).unwrap().unwrap(), va); assert_eq!(tree.get(ka).unwrap().unwrap(), va);
assert_eq!(tree.len().unwrap(), 1); assert_eq!(tree.iter().unwrap().count(), 1);
// ---- test transaction logic ---- // ---- test transaction logic ----
@@ -148,3 +148,15 @@ fn test_sqlite_db() {
let db = SqliteDb::new(manager, false).unwrap(); let db = SqliteDb::new(manager, false).unwrap();
test_suite(db); test_suite(db);
} }
#[test]
#[cfg(feature = "fjall")]
fn test_fjall_db() {
use crate::fjall_adapter::{fjall, FjallDb};
let path = mktemp::Temp::new_dir().unwrap();
let config = fjall::Config::new(path).temporary(true);
let keyspace = config.open_transactional().unwrap();
let db = FjallDb::init(keyspace);
test_suite(db);
}
+7 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage" name = "garage"
version = "1.1.0" version = "1.2.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@@ -49,6 +49,8 @@ structopt.workspace = true
git-version.workspace = true git-version.workspace = true
serde.workspace = true serde.workspace = true
serde_json.workspace = true
serde_bytes.workspace = true
futures.workspace = true futures.workspace = true
tokio.workspace = true tokio.workspace = true
@@ -57,6 +59,7 @@ opentelemetry.workspace = true
opentelemetry-prometheus = { workspace = true, optional = true } opentelemetry-prometheus = { workspace = true, optional = true }
opentelemetry-otlp = { workspace = true, optional = true } opentelemetry-otlp = { workspace = true, optional = true }
syslog-tracing = { workspace = true, optional = true } syslog-tracing = { workspace = true, optional = true }
tracing-journald = { workspace = true, optional = true }
[dev-dependencies] [dev-dependencies]
garage_api_common.workspace = true garage_api_common.workspace = true
@@ -90,6 +93,7 @@ k2v = [ "garage_util/k2v", "garage_api_k2v" ]
# Database engines # Database engines
lmdb = [ "garage_model/lmdb" ] lmdb = [ "garage_model/lmdb" ]
sqlite = [ "garage_model/sqlite" ] sqlite = [ "garage_model/sqlite" ]
fjall = [ "garage_model/fjall" ]
# Automatic registration and discovery via Consul API # Automatic registration and discovery via Consul API
consul-discovery = [ "garage_rpc/consul-discovery" ] consul-discovery = [ "garage_rpc/consul-discovery" ]
@@ -101,6 +105,8 @@ metrics = [ "garage_api_admin/metrics", "opentelemetry-prometheus" ]
telemetry-otlp = [ "opentelemetry-otlp" ] telemetry-otlp = [ "opentelemetry-otlp" ]
# Logging to syslog # Logging to syslog
syslog = [ "syslog-tracing" ] syslog = [ "syslog-tracing" ]
# Logging to journald
journald = [ "tracing-journald" ]
# NOTE: bundled-libs and system-libs should be treat as mutually exclusive; # NOTE: bundled-libs and system-libs should be treat as mutually exclusive;
# exactly one of them should be enabled. # exactly one of them should be enabled.
+9 -1
View File
@@ -101,6 +101,7 @@ impl AdminRpcHandler {
let mut obj_dels = 0; let mut obj_dels = 0;
let mut mpu_dels = 0; let mut mpu_dels = 0;
let mut ver_dels = 0; let mut ver_dels = 0;
let mut br_dels = 0;
for hash in blocks { for hash in blocks {
let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?; let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
@@ -131,12 +132,19 @@ impl AdminRpcHandler {
ver_dels += 1; ver_dels += 1;
} }
} }
if !br.deleted.get() {
let mut br = br;
br.deleted.set();
self.garage.block_ref_table.insert(&br).await?;
br_dels += 1;
}
} }
} }
Ok(AdminRpc::Ok(format!( Ok(AdminRpc::Ok(format!(
"Purged {} blocks, {} versions, {} objects, {} multipart uploads", "Purged {} blocks: marked {} block refs, {} versions, {} objects and {} multipart uploads as deleted",
blocks.len(), blocks.len(),
br_dels,
ver_dels, ver_dels,
obj_dels, obj_dels,
mpu_dels, mpu_dels,
-6
View File
@@ -390,15 +390,9 @@ impl AdminRpcHandler {
} }
let website = if query.allow { let website = if query.allow {
let (redirect_all, routing_rules) = match bucket_state.website_config.get() {
Some(wc) => (wc.redirect_all.clone(), wc.routing_rules.clone()),
None => (None, Vec::new()),
};
Some(WebsiteConfig { Some(WebsiteConfig {
index_document: query.index_document.clone(), index_document: query.index_document.clone(),
error_document: query.error_document.clone(), error_document: query.error_document.clone(),
redirect_all,
routing_rules,
}) })
} else { } else {
None None
+12 -7
View File
@@ -219,7 +219,7 @@ impl AdminRpcHandler {
// Gather block manager statistics // Gather block manager statistics
writeln!(&mut ret, "\nBlock manager stats:").unwrap(); writeln!(&mut ret, "\nBlock manager stats:").unwrap();
let rc_len = self.garage.block_manager.rc_len()?.to_string(); let rc_len = self.garage.block_manager.rc_approximate_len()?.to_string();
writeln!( writeln!(
&mut ret, &mut ret,
@@ -230,13 +230,13 @@ impl AdminRpcHandler {
writeln!( writeln!(
&mut ret, &mut ret,
" resync queue length: {}", " resync queue length: {}",
self.garage.block_manager.resync.queue_len()? self.garage.block_manager.resync.queue_approximate_len()?
) )
.unwrap(); .unwrap();
writeln!( writeln!(
&mut ret, &mut ret,
" blocks with resync errors: {}", " blocks with resync errors: {}",
self.garage.block_manager.resync.errors_len()? self.garage.block_manager.resync.errors_approximate_len()?
) )
.unwrap(); .unwrap();
@@ -346,16 +346,21 @@ impl AdminRpcHandler {
F: TableSchema + 'static, F: TableSchema + 'static,
R: TableReplication + 'static, R: TableReplication + 'static,
{ {
let data_len = t.data.store.len().map_err(GarageError::from)?.to_string(); let data_len = t
let mkl_len = t.merkle_updater.merkle_tree_len()?.to_string(); .data
.store
.approximate_len()
.map_err(GarageError::from)?
.to_string();
let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?.to_string();
Ok(format!( Ok(format!(
" {}\t{}\t{}\t{}\t{}", " {}\t{}\t{}\t{}\t{}",
F::TABLE_NAME, F::TABLE_NAME,
data_len, data_len,
mkl_len, mkl_len,
t.merkle_updater.todo_len()?, t.merkle_updater.todo_approximate_len()?,
t.data.gc_todo_len()? t.data.gc_todo_approximate_len()?
)) ))
} }
+10
View File
@@ -71,6 +71,10 @@ pub enum NodeOperation {
/// Connect to Garage node that is currently isolated from the system /// Connect to Garage node that is currently isolated from the system
#[structopt(name = "connect", version = garage_version())] #[structopt(name = "connect", version = garage_version())]
Connect(ConnectNodeOpt), Connect(ConnectNodeOpt),
/// Dump the content of a metadata table as JSON lines
#[structopt(name = "dump", version = garage_version())]
Dump(DumpNodeOpt),
} }
#[derive(StructOpt, Debug)] #[derive(StructOpt, Debug)]
@@ -88,6 +92,12 @@ pub struct ConnectNodeOpt {
pub(crate) node: String, pub(crate) node: String,
} }
#[derive(StructOpt, Debug)]
pub struct DumpNodeOpt {
/// Name of the data table to dump
pub(crate) what: String,
}
#[derive(StructOpt, Debug)] #[derive(StructOpt, Debug)]
pub enum LayoutOperation { pub enum LayoutOperation {
/// Assign role to Garage node /// Assign role to Garage node
+41 -1
View File
@@ -145,11 +145,14 @@ async fn main() {
let res = match opt.cmd { let res = match opt.cmd {
Command::Server => server::run_server(opt.config_file, opt.secrets).await, Command::Server => server::run_server(opt.config_file, opt.secrets).await,
Command::OfflineRepair(repair_opt) => { Command::OfflineRepair(repair_opt) => {
repair::offline::offline_repair(opt.config_file, opt.secrets, repair_opt).await repair::offline::offline_repair(opt.config_file, opt.secrets, repair_opt)
} }
Command::ConvertDb(conv_opt) => { Command::ConvertDb(conv_opt) => {
cli::convert_db::do_conversion(conv_opt).map_err(From::from) cli::convert_db::do_conversion(conv_opt).map_err(From::from)
} }
Command::Node(NodeOperation::Dump(dump_opt)) => {
repair::offline::dump(opt.config_file, opt.secrets, dump_opt)
}
Command::Node(NodeOperation::NodeId(node_id_opt)) => { Command::Node(NodeOperation::NodeId(node_id_opt)) => {
node_id_command(opt.config_file, node_id_opt.quiet) node_id_command(opt.config_file, node_id_opt.quiet)
} }
@@ -208,6 +211,43 @@ fn init_logging(opt: &Opt) {
} }
} }
if std::env::var("GARAGE_LOG_TO_JOURNALD")
.map(|x| x == "1" || x == "true")
.unwrap_or(false)
{
#[cfg(feature = "journald")]
{
use tracing_journald::{Priority, PriorityMappings};
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
let registry = tracing_subscriber::registry()
.with(tracing_subscriber::fmt::layer().with_writer(std::io::sink))
.with(env_filter);
match tracing_journald::layer() {
Ok(layer) => {
registry
.with(layer.with_priority_mappings(PriorityMappings {
info: Priority::Informational,
debug: Priority::Debug,
..PriorityMappings::new()
}))
.init();
}
Err(e) => {
eprintln!("Couldn't connect to journald: {}.", e);
std::process::exit(1);
}
}
return;
}
#[cfg(not(feature = "journald"))]
{
eprintln!("Journald support is not enabled in this build.");
std::process::exit(1);
}
}
tracing_subscriber::fmt() tracing_subscriber::fmt()
.with_writer(std::io::stderr) .with_writer(std::io::stderr)
.with_env_filter(env_filter) .with_env_filter(env_filter)
+102 -1
View File
@@ -1,14 +1,18 @@
use std::io::Write;
use std::path::PathBuf; use std::path::PathBuf;
use serde::Serialize;
use garage_util::config::*; use garage_util::config::*;
use garage_util::error::*; use garage_util::error::*;
use garage_model::garage::Garage; use garage_model::garage::Garage;
use garage_table::{replication::TableReplication, *};
use crate::cli::structs::*; use crate::cli::structs::*;
use crate::secrets::{fill_secrets, Secrets}; use crate::secrets::{fill_secrets, Secrets};
pub async fn offline_repair( pub fn offline_repair(
config_file: PathBuf, config_file: PathBuf,
secrets: Secrets, secrets: Secrets,
opt: OfflineRepairOpt, opt: OfflineRepairOpt,
@@ -45,3 +49,100 @@ pub async fn offline_repair(
Ok(()) Ok(())
} }
pub fn dump(config_file: PathBuf, secrets: Secrets, opt: DumpNodeOpt) -> Result<(), Error> {
let what = opt.what.as_str();
info!("Loading configuration...");
let config = fill_secrets(read_config(config_file)?, secrets)?;
info!("Initializing Garage main data store...");
let garage = Garage::new(config)?;
match what {
"bucket" | "buckets" => dump_table_inner(&garage.bucket_table),
"bucket_alias" | "bucket_aliases" => dump_table_inner(&garage.bucket_alias_table),
"key" | "keys" => dump_table_inner(&garage.key_table),
"object" | "objects" => dump_table_inner(&garage.object_table),
"object_counter" | "object_counters" => Err(Error::Message(
"object_counters cannot be JSON-serialized".into(),
)),
"mpu" => dump_table_inner(&garage.mpu_table),
"mpu_counter" | "mpu_counters" => Err(Error::Message(
"mpu_counters cannot be JSON-serialized".into(),
)),
"version" | "versions" => dump_table_inner(&garage.version_table),
"block_ref" | "block_refs" => dump_table_inner(&garage.block_ref_table),
#[cfg(feature = "k2v")]
"k2v_item" | "k2v_items" => dump_table_inner(&garage.k2v.item_table),
//#[cfg(feature = "k2v")]
"k2v_counter" | "k2v_counters" => Err(Error::Message(
"k2v_counters cannot be JSON-serialized".into(),
)),
other => {
let mut stdout = std::io::stdout().lock();
match other {
"cluster_layout" => Err(Error::Message(
"cluster_layout cannot be JSON-serialized".into(),
)),
_ => Err(Error::Message(format!("invalid thing to dump: {}", what))),
}
}
}
}
#[derive(Serialize)]
struct DumpEntry<'a, T: Serialize> {
#[serde(with = "serde_bytes")]
partition_key: &'a [u8],
#[serde(with = "serde_bytes")]
sort_key: &'a [u8],
entry: &'a T,
}
fn dump_table_inner<F, R>(table: &Table<F, R>) -> Result<(), Error>
where
F: TableSchema,
R: TableReplication,
{
eprintln!("Dumping table {}...", F::TABLE_NAME);
let mut stdout = std::io::stdout().lock();
for line in table.data.store.iter()? {
let (_k, v) = line?;
let v_dec = table.data.decode_entry(&v)?;
let pkh = v_dec.partition_key().hash();
let dump_entry = DumpEntry {
partition_key: pkh.as_slice(),
sort_key: v_dec.sort_key().sort_key(),
entry: &v_dec,
};
dump_line(&mut stdout, dump_entry)?;
}
stdout.flush()?;
Ok(())
}
fn dump_line<T: Serialize>(
mut stdout: &mut std::io::StdoutLock<'static>,
dump_entry: T,
) -> Result<(), Error> {
let mut ser = serde_json::ser::Serializer::with_formatter(&mut stdout, DumpFormatter);
dump_entry.serialize(&mut ser)?;
stdout.write_all(b"\n")?;
Ok(())
}
struct DumpFormatter;
impl serde_json::ser::Formatter for DumpFormatter {
fn write_byte_array<W>(&mut self, writer: &mut W, value: &[u8]) -> std::io::Result<()>
where
W: ?Sized + std::io::Write,
{
writer.write_all(b"\"")?;
writer.write_all(hex::encode(&value).as_bytes())?;
writer.write_all(b"\"")
}
}
+15 -4
View File
@@ -183,10 +183,21 @@ fn watch_shutdown_signal() -> watch::Receiver<bool> {
let mut sigterm = let mut sigterm =
signal(SignalKind::terminate()).expect("Failed to install SIGTERM handler"); signal(SignalKind::terminate()).expect("Failed to install SIGTERM handler");
let mut sighup = signal(SignalKind::hangup()).expect("Failed to install SIGHUP handler"); let mut sighup = signal(SignalKind::hangup()).expect("Failed to install SIGHUP handler");
tokio::select! { loop {
_ = sigint.recv() => info!("Received SIGINT, shutting down."), tokio::select! {
_ = sigterm.recv() => info!("Received SIGTERM, shutting down."), _ = sigint.recv() => {
_ = sighup.recv() => info!("Received SIGHUP, shutting down."), info!("Received SIGINT, shutting down.");
break
}
_ = sigterm.recv() => {
info!("Received SIGTERM, shutting down.");
break
}
_ = sighup.recv() => {
info!("Received SIGHUP, reload not supported.");
continue
}
}
} }
send_cancel.send(true).unwrap(); send_cancel.send(true).unwrap();
}); });
+43 -445
View File
@@ -5,10 +5,7 @@ use crate::json_body;
use assert_json_diff::assert_json_eq; use assert_json_diff::assert_json_eq;
use aws_sdk_s3::{ use aws_sdk_s3::{
primitives::ByteStream, primitives::ByteStream,
types::{ types::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration},
Condition, CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, Protocol, Redirect,
RoutingRule, WebsiteConfiguration,
},
}; };
use http::{Request, StatusCode}; use http::{Request, StatusCode};
use http_body_util::BodyExt; use http_body_util::BodyExt;
@@ -537,447 +534,6 @@ async fn test_website_check_domain() {
); );
} }
#[tokio::test]
async fn test_website_redirect_full_bucket() {
const BCKT_NAME: &str = "my-redirect-full";
let ctx = common::context();
let bucket = ctx.create_bucket(BCKT_NAME);
let conf = WebsiteConfiguration::builder()
.routing_rules(
RoutingRule::builder()
.condition(Condition::builder().key_prefix_equals("").build())
.redirect(
Redirect::builder()
.protocol(Protocol::Https)
.host_name("other.tld")
.replace_key_prefix_with("")
.build(),
)
.build(),
)
.build();
ctx.client
.put_bucket_website()
.bucket(&bucket)
.website_configuration(conf)
.send()
.await
.unwrap();
let req = Request::builder()
.method("GET")
.uri(format!("http://127.0.0.1:{}/my-path", ctx.garage.web_port))
.header("Host", format!("{}.web.garage", BCKT_NAME))
.body(Body::new(Bytes::new()))
.unwrap();
let client = Client::builder(TokioExecutor::new()).build_http();
let resp = client.request(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::FOUND);
assert_eq!(
resp.headers()
.get(hyper::header::LOCATION)
.unwrap()
.to_str()
.unwrap(),
"https://other.tld/my-path"
);
}
#[tokio::test]
async fn test_website_redirect() {
const BCKT_NAME: &str = "my-redirect";
let ctx = common::context();
let bucket = ctx.create_bucket(BCKT_NAME);
ctx.client
.put_object()
.bucket(&bucket)
.key("index.html")
.body(ByteStream::from_static(b"index"))
.send()
.await
.unwrap();
ctx.client
.put_object()
.bucket(&bucket)
.key("404.html")
.body(ByteStream::from_static(b"main 404"))
.send()
.await
.unwrap();
ctx.client
.put_object()
.bucket(&bucket)
.key("static-file")
.body(ByteStream::from_static(b"static file"))
.send()
.await
.unwrap();
let mut conf = WebsiteConfiguration::builder()
.index_document(
IndexDocument::builder()
.suffix("home.html")
.build()
.unwrap(),
)
.error_document(ErrorDocument::builder().key("404.html").build().unwrap());
for (prefix, condition) in [("unconditional", false), ("conditional", true)] {
let code = condition.then(|| "404".to_string());
conf = conf
// simple redirect
.routing_rules(
RoutingRule::builder()
.condition(
Condition::builder()
.set_http_error_code_returned_equals(code.clone())
.key_prefix_equals(format!("{prefix}/redirect-prefix/"))
.build(),
)
.redirect(
Redirect::builder()
.http_redirect_code("302")
.replace_key_prefix_with("other-prefix/")
.build(),
)
.build(),
)
.routing_rules(
RoutingRule::builder()
.condition(
Condition::builder()
.set_http_error_code_returned_equals(code.clone())
.key_prefix_equals(format!("{prefix}/redirect-prefix-307/"))
.build(),
)
.redirect(
Redirect::builder()
.http_redirect_code("307")
.replace_key_prefix_with("other-prefix/")
.build(),
)
.build(),
)
// simple redirect
.routing_rules(
RoutingRule::builder()
.condition(
Condition::builder()
.set_http_error_code_returned_equals(code.clone())
.key_prefix_equals(format!("{prefix}/redirect-fixed/"))
.build(),
)
.redirect(
Redirect::builder()
.http_redirect_code("302")
.replace_key_with("fixed_key")
.build(),
)
.build(),
)
// stream other file
.routing_rules(
RoutingRule::builder()
.condition(
Condition::builder()
.set_http_error_code_returned_equals(code.clone())
.key_prefix_equals(format!("{prefix}/stream-fixed/"))
.build(),
)
.redirect(
Redirect::builder()
.http_redirect_code("200")
.replace_key_with("static-file")
.build(),
)
.build(),
)
// stream other file as error
.routing_rules(
RoutingRule::builder()
.condition(
Condition::builder()
.set_http_error_code_returned_equals(code.clone())
.key_prefix_equals(format!("{prefix}/stream-404/"))
.build(),
)
.redirect(
Redirect::builder()
.http_redirect_code("404")
.replace_key_with("static-file")
.build(),
)
.build(),
)
// fail to stream other file
.routing_rules(
RoutingRule::builder()
.condition(
Condition::builder()
.set_http_error_code_returned_equals(code.clone())
.key_prefix_equals(format!("{prefix}/stream-missing/"))
.build(),
)
.redirect(
Redirect::builder()
.http_redirect_code("200")
.replace_key_with("missing-file")
.build(),
)
.build(),
);
}
let conf = conf.build();
ctx.client
.put_bucket_website()
.bucket(&bucket)
.website_configuration(conf.clone())
.send()
.await
.unwrap();
let stored_cfg = ctx
.client
.get_bucket_website()
.bucket(&bucket)
.send()
.await
.unwrap();
assert_eq!(stored_cfg.index_document, conf.index_document);
assert_eq!(stored_cfg.error_document, conf.error_document);
assert_eq!(stored_cfg.routing_rules, conf.routing_rules);
let req = |path| {
Request::builder()
.method("GET")
.uri(format!(
"http://127.0.0.1:{}/{}/path",
ctx.garage.web_port, path
))
.header("Host", format!("{}.web.garage", BCKT_NAME))
.body(Body::new(Bytes::new()))
.unwrap()
};
test_redirect_helper("unconditional", true, &req).await;
test_redirect_helper("conditional", true, &req).await;
for prefix in ["unconditional", "conditional"] {
for rule_path in [
"redirect-prefix",
"redirect-prefix-307",
"redirect-fixed",
"stream-fixed",
"stream-404",
"stream-missing",
] {
ctx.client
.put_object()
.bucket(&bucket)
.key(format!("{prefix}/{rule_path}/path"))
.body(ByteStream::from_static(b"i exist"))
.send()
.await
.unwrap();
}
}
test_redirect_helper("unconditional", true, &req).await;
test_redirect_helper("conditional", false, &req).await;
}
async fn test_redirect_helper(
prefix: &str,
should_see_redirect: bool,
req: impl Fn(String) -> Request<http_body_util::Full<Bytes>>,
) {
use http::header;
let client = Client::builder(TokioExecutor::new()).build_http();
let expected_body = b"i exist".as_ref();
let resp = client
.request(req(format!("{prefix}/redirect-prefix")))
.await
.unwrap();
if should_see_redirect {
assert_eq!(resp.status(), StatusCode::FOUND);
assert_eq!(
resp.headers()
.get(header::LOCATION)
.unwrap()
.to_str()
.unwrap(),
"/other-prefix/path"
);
assert!(resp
.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.is_empty());
} else {
assert_eq!(resp.status(), StatusCode::OK);
assert!(resp.headers().get(header::LOCATION).is_none());
assert_eq!(
resp.into_body().collect().await.unwrap().to_bytes(),
expected_body,
);
}
let resp = client
.request(req(format!("{prefix}/redirect-prefix-307")))
.await
.unwrap();
if should_see_redirect {
assert_eq!(resp.status(), StatusCode::TEMPORARY_REDIRECT);
assert_eq!(
resp.headers()
.get(header::LOCATION)
.unwrap()
.to_str()
.unwrap(),
"/other-prefix/path"
);
assert!(resp
.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.is_empty());
} else {
assert_eq!(resp.status(), StatusCode::OK);
assert!(resp.headers().get(header::LOCATION).is_none());
assert_eq!(
resp.into_body().collect().await.unwrap().to_bytes(),
expected_body,
);
}
let resp = client
.request(req(format!("{prefix}/redirect-fixed")))
.await
.unwrap();
if should_see_redirect {
assert_eq!(resp.status(), StatusCode::FOUND);
assert_eq!(
resp.headers()
.get(header::LOCATION)
.unwrap()
.to_str()
.unwrap(),
"/fixed_key"
);
assert!(resp
.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.is_empty());
} else {
assert_eq!(resp.status(), StatusCode::OK);
assert!(resp.headers().get(header::LOCATION).is_none());
assert_eq!(
resp.into_body().collect().await.unwrap().to_bytes(),
expected_body,
);
}
let resp = client
.request(req(format!("{prefix}/stream-fixed")))
.await
.unwrap();
if should_see_redirect {
assert_eq!(resp.status(), StatusCode::OK);
assert!(resp.headers().get(header::LOCATION).is_none());
assert_eq!(
resp.into_body().collect().await.unwrap().to_bytes(),
b"static file".as_ref(),
);
} else {
assert_eq!(resp.status(), StatusCode::OK);
assert!(resp.headers().get(header::LOCATION).is_none());
assert_eq!(
resp.into_body().collect().await.unwrap().to_bytes(),
expected_body,
);
}
let resp = client
.request(req(format!("{prefix}/stream-404")))
.await
.unwrap();
if should_see_redirect {
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
assert!(resp.headers().get(header::LOCATION).is_none());
assert_eq!(
resp.into_body().collect().await.unwrap().to_bytes(),
b"static file".as_ref(),
);
} else {
assert_eq!(resp.status(), StatusCode::OK);
assert!(resp.headers().get(header::LOCATION).is_none());
assert_eq!(
resp.into_body().collect().await.unwrap().to_bytes(),
expected_body,
);
}
let resp = client
.request(req(format!("{prefix}/stream-404")))
.await
.unwrap();
if should_see_redirect {
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
assert!(resp.headers().get(header::LOCATION).is_none());
assert_eq!(
resp.into_body().collect().await.unwrap().to_bytes(),
b"static file".as_ref(),
);
} else {
assert_eq!(resp.status(), StatusCode::OK);
assert!(resp.headers().get(header::LOCATION).is_none());
assert_eq!(
resp.into_body().collect().await.unwrap().to_bytes(),
expected_body,
);
}
}
#[tokio::test]
async fn test_website_invalid_redirect() {
const BCKT_NAME: &str = "my-invalid-redirect";
let ctx = common::context();
let bucket = ctx.create_bucket(BCKT_NAME);
let conf = WebsiteConfiguration::builder()
.routing_rules(
RoutingRule::builder()
.condition(Condition::builder().key_prefix_equals("").build())
.redirect(
Redirect::builder()
.protocol(Protocol::Https)
.host_name("other.tld")
.replace_key_prefix_with("")
// we don't allow 200 with hostname
.http_redirect_code("200")
.build(),
)
.build(),
)
.build();
ctx.client
.put_bucket_website()
.bucket(&bucket)
.website_configuration(conf)
.send()
.await
.unwrap_err();
}
#[tokio::test] #[tokio::test]
async fn test_website_puny() { async fn test_website_puny() {
const BCKT_NAME: &str = "xn--pda.eu"; const BCKT_NAME: &str = "xn--pda.eu";
@@ -1050,3 +606,45 @@ async fn test_website_puny() {
); );
} }
} }
#[tokio::test]
async fn test_website_object_not_found() {
const BCKT_NAME: &str = "not-found";
let ctx = common::context();
let _bucket = ctx.create_bucket(BCKT_NAME);
let client = Client::builder(TokioExecutor::new()).build_http();
let req = |suffix| {
Request::builder()
.method("GET")
.uri(format!("http://127.0.0.1:{}/", ctx.garage.web_port))
.header("Host", format!("{}{}", BCKT_NAME, suffix))
.body(Body::new(Bytes::new()))
.unwrap()
};
ctx.garage
.command()
.args(["bucket", "website", "--allow", BCKT_NAME])
.quiet()
.expect_success_status("Could not allow website on bucket");
let resp = client.request(req("")).await.unwrap();
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
// the error we return by default are *not* xml
assert_eq!(
resp.headers().get(http::header::CONTENT_TYPE).unwrap(),
"text/html; charset=utf-8"
);
let result = String::from_utf8(
resp.into_body()
.collect()
.await
.unwrap()
.to_bytes()
.to_vec(),
)
.unwrap();
assert!(result.contains("not found"));
}
+10
View File
@@ -72,6 +72,16 @@ impl K2vClient {
.enable_http2() .enable_http2()
.build(); .build();
let client = HttpClient::builder(TokioExecutor::new()).build(connector); let client = HttpClient::builder(TokioExecutor::new()).build(connector);
Self::new_with_client(config, client)
}
/// Create a new K2V client with an external client.
/// Useful for example if you plan on creating many clients but you want to mutualize the
/// underlying thread pools & co.
pub fn new_with_client(
config: K2vClientConfig,
client: HttpClient<HttpsConnector<HttpConnector>, Body>,
) -> Result<Self, Error> {
let user_agent: std::borrow::Cow<str> = match &config.user_agent { let user_agent: std::borrow::Cow<str> = match &config.user_agent {
Some(ua) => ua.into(), Some(ua) => ua.into(),
None => format!("k2v/{}", env!("CARGO_PKG_VERSION")).into(), None => format!("k2v/{}", env!("CARGO_PKG_VERSION")).into(),
+2 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_model" name = "garage_model"
version = "1.1.0" version = "1.2.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@@ -44,3 +44,4 @@ default = [ "lmdb", "sqlite" ]
k2v = [ "garage_util/k2v" ] k2v = [ "garage_util/k2v" ]
lmdb = [ "garage_db/lmdb" ] lmdb = [ "garage_db/lmdb" ]
sqlite = [ "garage_db/sqlite" ] sqlite = [ "garage_db/sqlite" ]
fjall = [ "garage_db/fjall" ]
+1 -116
View File
@@ -119,122 +119,7 @@ mod v08 {
impl garage_util::migrate::InitialFormat for Bucket {} impl garage_util::migrate::InitialFormat for Bucket {}
} }
mod v2 { pub use v08::*;
use crate::permission::BucketKeyPerm;
use garage_util::crdt;
use garage_util::data::Uuid;
use serde::{Deserialize, Serialize};
use super::v08;
pub use v08::{BucketQuotas, CorsRule, LifecycleExpiration, LifecycleFilter, LifecycleRule};
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
pub struct Bucket {
/// ID of the bucket
pub id: Uuid,
/// State, and configuration if not deleted, of the bucket
pub state: crdt::Deletable<BucketParams>,
}
/// Configuration for a bucket
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
pub struct BucketParams {
/// Bucket's creation date
pub creation_date: u64,
/// Map of key with access to the bucket, and what kind of access they give
pub authorized_keys: crdt::Map<String, BucketKeyPerm>,
/// Map of aliases that are or have been given to this bucket
/// in the global namespace
/// (not authoritative: this is just used as an indication to
/// map back to aliases when doing ListBuckets)
pub aliases: crdt::LwwMap<String, bool>,
/// Map of aliases that are or have been given to this bucket
/// in namespaces local to keys
/// key = (access key id, alias name)
pub local_aliases: crdt::LwwMap<(String, String), bool>,
/// Whether this bucket is allowed for website access
/// (under all of its global alias names),
/// and if so, the website configuration XML document
pub website_config: crdt::Lww<Option<WebsiteConfig>>,
/// CORS rules
pub cors_config: crdt::Lww<Option<Vec<CorsRule>>>,
/// Lifecycle configuration
pub lifecycle_config: crdt::Lww<Option<Vec<LifecycleRule>>>,
/// Bucket quotas
pub quotas: crdt::Lww<BucketQuotas>,
}
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
pub struct WebsiteConfig {
pub index_document: String,
pub error_document: Option<String>,
// this field is currently unused, but present so adding it in the future doesn't
// need a new migration
pub redirect_all: Option<RedirectAll>,
pub routing_rules: Vec<RoutingRule>,
}
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
pub struct RedirectAll {
pub hostname: String,
pub protocol: String,
}
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
pub struct RoutingRule {
pub condition: Option<RedirectCondition>,
pub redirect: Redirect,
}
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
pub struct RedirectCondition {
pub http_error_code: Option<u16>,
pub prefix: Option<String>,
}
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
pub struct Redirect {
pub hostname: Option<String>,
pub http_redirect_code: u16,
pub protocol: Option<String>,
pub replace_key_prefix: Option<String>,
pub replace_key: Option<String>,
}
impl garage_util::migrate::Migrate for Bucket {
const VERSION_MARKER: &'static [u8] = b"G2bkt";
type Previous = v08::Bucket;
fn migrate(old: v08::Bucket) -> Bucket {
Bucket {
id: old.id,
state: old.state.map(|x| BucketParams {
creation_date: x.creation_date,
authorized_keys: x.authorized_keys,
aliases: x.aliases,
local_aliases: x.local_aliases,
website_config: x.website_config.map(|wc_opt| {
wc_opt.map(|wc| WebsiteConfig {
index_document: wc.index_document,
error_document: wc.error_document,
redirect_all: None,
routing_rules: vec![],
})
}),
cors_config: x.cors_config,
lifecycle_config: x.lifecycle_config,
quotas: x.quotas,
}),
}
}
}
}
pub use v2::*;
impl AutoCrdt for BucketQuotas { impl AutoCrdt for BucketQuotas {
const WARN_IF_DIFFERENT: bool = true; const WARN_IF_DIFFERENT: bool = true;
+5 -9
View File
@@ -116,21 +116,17 @@ impl Garage {
info!("Opening database..."); info!("Opening database...");
let db_engine = db::Engine::from_str(&config.db_engine) let db_engine = db::Engine::from_str(&config.db_engine)
.ok_or_message("Invalid `db_engine` value in configuration file")?; .ok_or_message("Invalid `db_engine` value in configuration file")?;
let mut db_path = config.metadata_dir.clone(); let db_path = db_engine.db_path(&config.metadata_dir);
match db_engine {
db::Engine::Sqlite => {
db_path.push("db.sqlite");
}
db::Engine::Lmdb => {
db_path.push("db.lmdb");
}
}
let db_opt = db::OpenOpt { let db_opt = db::OpenOpt {
fsync: config.metadata_fsync, fsync: config.metadata_fsync,
lmdb_map_size: match config.lmdb_map_size { lmdb_map_size: match config.lmdb_map_size {
v if v == usize::default() => None, v if v == usize::default() => None,
v => Some(v), v => Some(v),
}, },
fjall_block_cache_size: match config.fjall_block_cache_size {
v if v == usize::default() => None,
v => Some(v),
},
}; };
let db = db::open_db(&db_path, db_engine, &db_opt) let db = db::open_db(&db_path, db_engine, &db_opt)
.ok_or_message("Unable to open metadata db")?; .ok_or_message("Unable to open metadata db")?;
+3 -3
View File
@@ -121,13 +121,13 @@ impl Worker for LifecycleWorker {
mpu_aborted, mpu_aborted,
.. ..
} => { } => {
let n_objects = self.garage.object_table.data.store.len().ok(); let n_objects = self.garage.object_table.data.store.approximate_len().ok();
let progress = match n_objects { let progress = match n_objects {
None => "...".to_string(), Some(total) if total > 0 => format!(
Some(total) => format!(
"~{:.2}%", "~{:.2}%",
100. * std::cmp::min(*counter, total) as f32 / total as f32 100. * std::cmp::min(*counter, total) as f32 / total as f32
), ),
_ => "...".to_string(),
}; };
WorkerStatus { WorkerStatus {
progress: Some(progress), progress: Some(progress),
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_net" name = "garage_net"
version = "1.1.0" version = "1.2.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_rpc" name = "garage_rpc"
version = "1.1.0" version = "1.2.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_table" name = "garage_table"
version = "1.1.0" version = "1.2.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+2 -2
View File
@@ -367,7 +367,7 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
} }
} }
pub fn gc_todo_len(&self) -> Result<usize, Error> { pub fn gc_todo_approximate_len(&self) -> Result<usize, Error> {
Ok(self.gc_todo.len()?) Ok(self.gc_todo.approximate_len()?)
} }
} }
+1 -1
View File
@@ -313,7 +313,7 @@ impl<F: TableSchema, R: TableReplication> Worker for GcWorker<F, R> {
fn status(&self) -> WorkerStatus { fn status(&self) -> WorkerStatus {
WorkerStatus { WorkerStatus {
queue_length: Some(self.gc.data.gc_todo_len().unwrap_or(0) as u64), queue_length: Some(self.gc.data.gc_todo_approximate_len().unwrap_or(0) as u64),
..Default::default() ..Default::default()
} }
} }
+5 -5
View File
@@ -287,12 +287,12 @@ impl<F: TableSchema, R: TableReplication> MerkleUpdater<F, R> {
MerkleNode::decode_opt(&ent) MerkleNode::decode_opt(&ent)
} }
pub fn merkle_tree_len(&self) -> Result<usize, Error> { pub fn merkle_tree_approximate_len(&self) -> Result<usize, Error> {
Ok(self.data.merkle_tree.len()?) Ok(self.data.merkle_tree.approximate_len()?)
} }
pub fn todo_len(&self) -> Result<usize, Error> { pub fn todo_approximate_len(&self) -> Result<usize, Error> {
Ok(self.data.merkle_todo.len()?) Ok(self.data.merkle_todo.approximate_len()?)
} }
} }
@@ -306,7 +306,7 @@ impl<F: TableSchema, R: TableReplication> Worker for MerkleWorker<F, R> {
fn status(&self) -> WorkerStatus { fn status(&self) -> WorkerStatus {
WorkerStatus { WorkerStatus {
queue_length: Some(self.0.todo_len().unwrap_or(0) as u64), queue_length: Some(self.0.todo_approximate_len().unwrap_or(0) as u64),
..Default::default() ..Default::default()
} }
} }
+4 -4
View File
@@ -34,7 +34,7 @@ impl TableMetrics {
.u64_value_observer( .u64_value_observer(
"table.size", "table.size",
move |observer| { move |observer| {
if let Ok(value) = store.len() { if let Ok(value) = store.approximate_len() {
observer.observe( observer.observe(
value as u64, value as u64,
&[KeyValue::new("table_name", table_name)], &[KeyValue::new("table_name", table_name)],
@@ -48,7 +48,7 @@ impl TableMetrics {
.u64_value_observer( .u64_value_observer(
"table.merkle_tree_size", "table.merkle_tree_size",
move |observer| { move |observer| {
if let Ok(value) = merkle_tree.len() { if let Ok(value) = merkle_tree.approximate_len() {
observer.observe( observer.observe(
value as u64, value as u64,
&[KeyValue::new("table_name", table_name)], &[KeyValue::new("table_name", table_name)],
@@ -62,7 +62,7 @@ impl TableMetrics {
.u64_value_observer( .u64_value_observer(
"table.merkle_updater_todo_queue_length", "table.merkle_updater_todo_queue_length",
move |observer| { move |observer| {
if let Ok(v) = merkle_todo.len() { if let Ok(v) = merkle_todo.approximate_len() {
observer.observe( observer.observe(
v as u64, v as u64,
&[KeyValue::new("table_name", table_name)], &[KeyValue::new("table_name", table_name)],
@@ -76,7 +76,7 @@ impl TableMetrics {
.u64_value_observer( .u64_value_observer(
"table.gc_todo_queue_length", "table.gc_todo_queue_length",
move |observer| { move |observer| {
if let Ok(value) = gc_todo.len() { if let Ok(value) = gc_todo.approximate_len() {
observer.observe( observer.observe(
value as u64, value as u64,
&[KeyValue::new("table_name", table_name)], &[KeyValue::new("table_name", table_name)],
+1 -1
View File
@@ -27,7 +27,7 @@ impl<F: TableSchema, R: TableReplication> Worker for InsertQueueWorker<F, R> {
fn status(&self) -> WorkerStatus { fn status(&self) -> WorkerStatus {
WorkerStatus { WorkerStatus {
queue_length: Some(self.0.data.insert_queue.len().unwrap_or(0) as u64), queue_length: Some(self.0.data.insert_queue.approximate_len().unwrap_or(0) as u64),
..Default::default() ..Default::default()
} }
} }
+6 -3
View File
@@ -43,10 +43,13 @@ impl TableReplication for TableFullReplication {
} }
fn write_quorum(&self) -> usize { fn write_quorum(&self) -> usize {
let nmembers = self.system.cluster_layout().current().all_nodes().len(); let nmembers = self.system.cluster_layout().current().all_nodes().len();
if nmembers < 3 {
1 let max_faults = if nmembers > 1 { 1 } else { 0 };
if nmembers > max_faults {
nmembers - max_faults
} else { } else {
nmembers.div_euclid(2) + 1 1
} }
} }
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_util" name = "garage_util"
version = "1.1.0" version = "1.2.0"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+4
View File
@@ -122,6 +122,10 @@ pub struct Config {
#[serde(deserialize_with = "deserialize_capacity", default)] #[serde(deserialize_with = "deserialize_capacity", default)]
pub lmdb_map_size: usize, pub lmdb_map_size: usize,
/// Fjall block cache size
#[serde(deserialize_with = "deserialize_capacity", default)]
pub fjall_block_cache_size: usize,
// -- APIs // -- APIs
/// Configuration for S3 api /// Configuration for S3 api
pub s3_api: S3ApiConfig, pub s3_api: S3ApiConfig,
-10
View File
@@ -9,16 +9,6 @@ pub enum Deletable<T> {
Deleted, Deleted,
} }
impl<T> Deletable<T> {
/// Map value, used for migrations
pub fn map<U, F: FnOnce(T) -> U>(self, f: F) -> Deletable<U> {
match self {
Self::Present(x) => Deletable::<U>::Present(f(x)),
Self::Deleted => Deletable::<U>::Deleted,
}
}
}
impl<T: Crdt> Deletable<T> { impl<T: Crdt> Deletable<T> {
/// Create a new deletable object that isn't deleted /// Create a new deletable object that isn't deleted
pub fn present(v: T) -> Self { pub fn present(v: T) -> Self {
-10
View File
@@ -43,16 +43,6 @@ pub struct Lww<T> {
v: T, v: T,
} }
impl<T> Lww<T> {
/// Map value, used for migrations
pub fn map<U, F: FnOnce(T) -> U>(self, f: F) -> Lww<U> {
Lww::<U> {
ts: self.ts,
v: f(self.v),
}
}
}
impl<T> Lww<T> impl<T> Lww<T>
where where
T: Crdt, T: Crdt,
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "garage_web" name = "garage_web"
version = "1.1.0" version = "1.2.0"
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"] authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
+94 -304
View File
@@ -25,14 +25,12 @@ use garage_api_common::cors::{
}; };
use garage_api_common::generic_server::{server_loop, UnixListenerOn}; use garage_api_common::generic_server::{server_loop, UnixListenerOn};
use garage_api_common::helpers::*; use garage_api_common::helpers::*;
use garage_api_s3::api_server::ResBody;
use garage_api_s3::error::{ use garage_api_s3::error::{
CommonErrorDerivative, Error as ApiError, OkOrBadRequest, OkOrInternalError, CommonErrorDerivative, Error as ApiError, OkOrBadRequest, OkOrInternalError,
}; };
use garage_api_s3::get::{handle_get_without_ctx, handle_head_without_ctx}; use garage_api_s3::get::{handle_get_without_ctx, handle_head_without_ctx};
use garage_api_s3::website::X_AMZ_WEBSITE_REDIRECT_LOCATION; use garage_api_s3::website::X_AMZ_WEBSITE_REDIRECT_LOCATION;
use garage_model::bucket_table::{self, RoutingRule};
use garage_model::garage::Garage; use garage_model::garage::Garage;
use garage_table::*; use garage_table::*;
@@ -262,71 +260,45 @@ impl WebServer {
// Get path // Get path
let path = req.uri().path().to_string(); let path = req.uri().path().to_string();
let index = &website_config.index_document; let index = &website_config.index_document;
let routing_result = path_to_keys(&path, index, &website_config.routing_rules)?; let (key, may_redirect) = path_to_keys(&path, index)?;
debug!( debug!(
"Selected bucket: \"{}\" {:?}, routing to {:?}", "Selected bucket: \"{}\" {:?}, target key: \"{}\", may redirect to: {:?}",
bucket_name, bucket_id, routing_result, bucket_name, bucket_id, key, may_redirect
); );
let ret_doc = match (req.method(), routing_result.main_target()) { let ret_doc = match *req.method() {
(&Method::OPTIONS, _) => handle_options_for_bucket(req, &bucket_params) Method::OPTIONS => handle_options_for_bucket(req, &bucket_params)
.map_err(ApiError::from) .map_err(ApiError::from)
.map(|res| res.map(|_empty_body: EmptyBody| empty_body())), .map(|res| res.map(|_empty_body: EmptyBody| empty_body())),
(_, Err((url, code))) => Ok(Response::builder() Method::HEAD => {
.status(code) handle_head_without_ctx(self.garage.clone(), req, bucket_id, &key, None).await
.header("Location", url)
.body(empty_body())
.unwrap()),
(_, Ok((key, code))) => {
handle_inner(self.garage.clone(), req, bucket_id, key, code).await
} }
}; Method::GET => {
handle_get_without_ctx(
// Try handling errors if bucket configuration provided fallbacks
let ret_doc_with_redir = match (&ret_doc, &routing_result) {
(
Err(ApiError::NoSuchKey),
RoutingResult::LoadOrRedirect {
redirect_if_exists,
redirect_url,
redirect_code,
..
},
) => {
let redirect = if let Some(redirect_key) = redirect_if_exists {
self.check_key_exists(bucket_id, redirect_key.as_str())
.await?
} else {
true
};
if redirect {
Ok(Response::builder()
.status(redirect_code)
.header("Location", redirect_url)
.body(empty_body())
.unwrap())
} else {
ret_doc
}
}
(
Err(ApiError::NoSuchKey),
RoutingResult::LoadOrAlternativeError {
redirect_key,
redirect_code,
..
},
) => {
handle_inner(
self.garage.clone(), self.garage.clone(),
req, req,
bucket_id, bucket_id,
redirect_key, &key,
*redirect_code, None,
Default::default(),
) )
.await .await
} }
_ => Err(ApiError::bad_request("HTTP method not supported")),
};
// Try implicit redirect on error
let ret_doc_with_redir = match (&ret_doc, may_redirect) {
(Err(ApiError::NoSuchKey), ImplicitRedirect::To { key, url })
if self.check_key_exists(bucket_id, key.as_str()).await? =>
{
Ok(Response::builder()
.status(StatusCode::FOUND)
.header(LOCATION, url)
.body(empty_body())
.unwrap())
}
(Ok(ret), _) if ret.headers().contains_key(X_AMZ_WEBSITE_REDIRECT_LOCATION) => { (Ok(ret), _) if ret.headers().contains_key(X_AMZ_WEBSITE_REDIRECT_LOCATION) => {
let redirect_location = ret.headers().get(X_AMZ_WEBSITE_REDIRECT_LOCATION).unwrap(); let redirect_location = ret.headers().get(X_AMZ_WEBSITE_REDIRECT_LOCATION).unwrap();
Ok(Response::builder() Ok(Response::builder()
@@ -360,17 +332,17 @@ impl WebServer {
// We want to return the error document // We want to return the error document
// Create a fake HTTP request with path = the error document // Create a fake HTTP request with path = the error document
let req2 = Request::builder() let req2 = Request::builder()
.method("GET")
.uri(format!("http://{}/{}", host, &error_document)) .uri(format!("http://{}/{}", host, &error_document))
.body(()) .body(())
.unwrap(); .unwrap();
match handle_inner( match handle_get_without_ctx(
self.garage.clone(), self.garage.clone(),
&req2, &req2,
bucket_id, bucket_id,
&error_document, &error_document,
error.http_status_code(), None,
Default::default(),
) )
.await .await
{ {
@@ -385,6 +357,8 @@ impl WebServer {
error error
); );
*error_doc.status_mut() = error.http_status_code();
// Preserve error message in a special header // Preserve error message in a special header
for error_line in error.to_string().split('\n') { for error_line in error.to_string().split('\n') {
if let Ok(v) = HeaderValue::from_bytes(error_line.as_bytes()) { if let Ok(v) = HeaderValue::from_bytes(error_line.as_bytes()) {
@@ -415,52 +389,6 @@ impl WebServer {
} }
} }
async fn handle_inner(
garage: Arc<Garage>,
req: &Request<()>,
bucket_id: Uuid,
key: &str,
status_code: StatusCode,
) -> Result<Response<ResBody>, ApiError> {
if status_code != StatusCode::OK {
// If we are returning an error document, discard all headers from
// the original request that would have influenced the result:
// - Range header, we don't want to return a subrange of the error document
// - Caching directives such as If-None-Match, etc, which are not relevant
let cleaned_req = Request::builder().uri(req.uri()).body(()).unwrap();
let mut ret = match req.method() {
&Method::HEAD => {
handle_head_without_ctx(garage, &cleaned_req, bucket_id, key, None).await?
}
&Method::GET => {
handle_get_without_ctx(
garage,
&cleaned_req,
bucket_id,
key,
None,
Default::default(),
)
.await?
}
_ => return Err(ApiError::bad_request("HTTP method not supported")),
};
*ret.status_mut() = status_code;
Ok(ret)
} else {
match req.method() {
&Method::HEAD => handle_head_without_ctx(garage, req, bucket_id, key, None).await,
&Method::GET => {
handle_get_without_ctx(garage, req, bucket_id, key, None, Default::default()).await
}
_ => Err(ApiError::bad_request("HTTP method not supported")),
}
}
}
fn error_to_res(e: Error) -> Response<BoxBody<Error>> { fn error_to_res(e: Error) -> Response<BoxBody<Error>> {
// If we are here, it is either that: // If we are here, it is either that:
// - there was an error before trying to get the requested URL // - there was an error before trying to get the requested URL
@@ -469,52 +397,37 @@ fn error_to_res(e: Error) -> Response<BoxBody<Error>> {
// was a HEAD request or we couldn't get the error document) // was a HEAD request or we couldn't get the error document)
// We do NOT enter this code path when returning the bucket's // We do NOT enter this code path when returning the bucket's
// error document (this is handled in serve_file) // error document (this is handled in serve_file)
let body = string_body(format!("{}\n", e)); let mut body_str = format!(
let mut http_error = Response::new(body); r"<title>{http_code} {code_text}</title>
<h1>{http_code} {code_text}</h1>",
http_code = e.http_status_code().as_u16(),
code_text = e.http_status_code().canonical_reason().unwrap_or("Unknown"),
);
if let Error::ApiError(ref err) = e {
body_str.push_str(&format!(
r"
<ul>
<li>Code: {s3_code}</li>
<li>Message: {s3_message}.</li>
</ul>",
s3_code = err.aws_code(),
s3_message = err,
));
}
let mut http_error = Response::new(string_body(body_str));
*http_error.status_mut() = e.http_status_code(); *http_error.status_mut() = e.http_status_code();
e.add_headers(http_error.headers_mut()); e.add_headers(http_error.headers_mut());
http_error.headers_mut().insert(
http::header::CONTENT_TYPE,
"text/html; charset=utf-8".parse().unwrap(),
);
http_error http_error
} }
#[derive(Debug, PartialEq)] #[derive(Debug, PartialEq)]
enum RoutingResult { enum ImplicitRedirect {
// Load a key and use `code` as status, or fallback to normal 404 handler if not found No,
LoadKey { To { key: String, url: String },
key: String,
code: StatusCode,
},
// Load a key and use `200` as status, or fallback with a redirection using `redirect_code`
// as status
LoadOrRedirect {
key: String,
redirect_if_exists: Option<String>,
redirect_url: String,
redirect_code: StatusCode,
},
// Load a key and use `200` as status, or fallback by loading a different key and use
// `redirect_code` as status
LoadOrAlternativeError {
key: String,
redirect_key: String,
redirect_code: StatusCode,
},
// Send an http redirect with `code` as status
Redirect {
url: String,
code: StatusCode,
},
}
impl RoutingResult {
// return Ok((key_to_deref, status_code)) or Err((redirect_target, status_code))
fn main_target(&self) -> Result<(&str, StatusCode), (&str, StatusCode)> {
match self {
RoutingResult::LoadKey { key, code } => Ok((key, *code)),
RoutingResult::LoadOrRedirect { key, .. } => Ok((key, StatusCode::OK)),
RoutingResult::LoadOrAlternativeError { key, .. } => Ok((key, StatusCode::OK)),
RoutingResult::Redirect { url, code } => Err((url, *code)),
}
}
} }
/// Path to key /// Path to key
@@ -524,154 +437,35 @@ impl RoutingResult {
/// which is also AWS S3 behavior. /// which is also AWS S3 behavior.
/// ///
/// Check: https://docs.aws.amazon.com/AmazonS3/latest/userguide/IndexDocumentSupport.html /// Check: https://docs.aws.amazon.com/AmazonS3/latest/userguide/IndexDocumentSupport.html
fn path_to_keys( fn path_to_keys<'a>(path: &'a str, index: &str) -> Result<(String, ImplicitRedirect), Error> {
path: &str,
index: &str,
routing_rules: &[RoutingRule],
) -> Result<RoutingResult, Error> {
let path_utf8 = percent_encoding::percent_decode_str(path).decode_utf8()?; let path_utf8 = percent_encoding::percent_decode_str(path).decode_utf8()?;
let base_key = match path_utf8.strip_prefix("/") { let base_key = match path_utf8.strip_prefix("/") {
Some(bk) => bk, Some(bk) => bk,
None => return Err(Error::BadRequest("Path must start with a / (slash)".into())), None => return Err(Error::BadRequest("Path must start with a / (slash)".into())),
}; };
let is_bucket_root = base_key.len() == 0;
let is_bucket_root = base_key.is_empty();
let is_trailing_slash = path_utf8.ends_with("/"); let is_trailing_slash = path_utf8.ends_with("/");
let key = if is_bucket_root || is_trailing_slash { match (is_bucket_root, is_trailing_slash) {
// we can't store anything at the root, so we need to query the index // It is not possible to store something at the root of the bucket (ie. empty key),
// if the key end with a slash, we always query the index // the only option is to fetch the index
format!("{base_key}{index}") (true, _) => Ok((index.to_string(), ImplicitRedirect::No)),
} else {
// if the key doesn't end with `/`, leave it unmodified
base_key.to_string()
};
let mut routing_rules_iter = routing_rules.iter(); // "If you create a folder structure in your bucket, you must have an index document at each level. In each folder, the index document must have the same name, for example, index.html. When a user specifies a URL that resembles a folder lookup, the presence or absence of a trailing slash determines the behavior of the website. For example, the following URL, with a trailing slash, returns the photos/index.html index document."
let key = loop { (false, true) => Ok((format!("{base_key}{index}"), ImplicitRedirect::No)),
let Some(routing_rule) = routing_rules_iter.next() else {
break key;
};
let Ok(status_code) = StatusCode::from_u16(routing_rule.redirect.http_redirect_code) else { // "However, if you exclude the trailing slash from the preceding URL, Amazon S3 first looks for an object photos in the bucket. If the photos object is not found, it searches for an index document, photos/index.html. If that document is found, Amazon S3 returns a 302 Found message and points to the photos/ key. For subsequent requests to photos/, Amazon S3 returns photos/index.html. If the index document is not found, Amazon S3 returns an error."
continue; (false, false) => Ok((
}; base_key.to_string(),
if let Some(condition) = &routing_rule.condition { ImplicitRedirect::To {
let suffix = if let Some(prefix) = &condition.prefix { key: format!("{base_key}/{index}"),
let Some(suffix) = key.strip_prefix(prefix) else { url: format!("{path}/"),
continue; },
}; )),
Some(suffix)
} else {
None
};
let mut target = compute_redirect_target(&routing_rule.redirect, suffix);
let query_alternative_key =
status_code == StatusCode::OK || status_code == StatusCode::NOT_FOUND;
let redirect_on_error =
condition.http_error_code == Some(StatusCode::NOT_FOUND.as_u16());
match (query_alternative_key, redirect_on_error) {
(false, false) => {
return Ok(RoutingResult::Redirect {
url: target,
code: status_code,
})
}
(true, false) => {
// we need to remove the leading /
target.remove(0);
if status_code == StatusCode::OK {
break target;
} else {
return Ok(RoutingResult::LoadKey {
key: target,
code: status_code,
});
}
}
(false, true) => {
return Ok(RoutingResult::LoadOrRedirect {
key,
redirect_if_exists: None,
redirect_url: target,
redirect_code: status_code,
});
}
(true, true) => {
target.remove(0);
return Ok(RoutingResult::LoadOrAlternativeError {
key,
redirect_key: target,
redirect_code: status_code,
});
}
}
} else {
let target = compute_redirect_target(&routing_rule.redirect, None);
return Ok(RoutingResult::Redirect {
url: target,
code: status_code,
});
}
};
if is_bucket_root || is_trailing_slash {
Ok(RoutingResult::LoadKey {
key,
code: StatusCode::OK,
})
} else {
Ok(RoutingResult::LoadOrRedirect {
redirect_if_exists: Some(format!("{key}/{index}")),
// we can't use `path` because key might have changed substentially in case of
// routing rules
redirect_url: percent_encoding::percent_encode(
format!("/{key}/").as_bytes(),
PATH_ENCODING_SET,
)
.to_string(),
key,
redirect_code: StatusCode::FOUND,
})
} }
} }
// per https://url.spec.whatwg.org/#path-percent-encode-set
const PATH_ENCODING_SET: &percent_encoding::AsciiSet = &percent_encoding::CONTROLS
.add(b' ')
.add(b'"')
.add(b'#')
.add(b'<')
.add(b'>')
.add(b'?')
.add(b'`')
.add(b'{')
.add(b'}');
fn compute_redirect_target(redirect: &bucket_table::Redirect, suffix: Option<&str>) -> String {
let mut res = String::new();
if let Some(hostname) = &redirect.hostname {
if let Some(protocol) = &redirect.protocol {
res.push_str(protocol);
res.push_str("://");
} else {
res.push_str("//");
}
res.push_str(hostname);
}
res.push('/');
if let Some(replace_key_prefix) = &redirect.replace_key_prefix {
res.push_str(replace_key_prefix);
if let Some(suffix) = suffix {
res.push_str(suffix)
}
} else if let Some(replace_key) = &redirect.replace_key {
res.push_str(replace_key)
}
res
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@@ -679,39 +473,35 @@ mod tests {
#[test] #[test]
fn path_to_keys_test() -> Result<(), Error> { fn path_to_keys_test() -> Result<(), Error> {
assert_eq!( assert_eq!(
path_to_keys("/file%20.jpg", "index.html", &[])?, path_to_keys("/file%20.jpg", "index.html")?,
RoutingResult::LoadOrRedirect { (
key: "file .jpg".to_string(), "file .jpg".to_string(),
redirect_url: "/file%20.jpg/".to_string(), ImplicitRedirect::To {
redirect_if_exists: Some("file .jpg/index.html".to_string()), key: "file .jpg/index.html".to_string(),
redirect_code: StatusCode::FOUND, url: "/file%20.jpg/".to_string()
} }
)
); );
assert_eq!( assert_eq!(
path_to_keys("/%20t/", "index.html", &[])?, path_to_keys("/%20t/", "index.html")?,
RoutingResult::LoadKey { (" t/index.html".to_string(), ImplicitRedirect::No)
key: " t/index.html".to_string(),
code: StatusCode::OK
}
); );
assert_eq!( assert_eq!(
path_to_keys("/", "index.html", &[])?, path_to_keys("/", "index.html")?,
RoutingResult::LoadKey { ("index.html".to_string(), ImplicitRedirect::No)
key: "index.html".to_string(),
code: StatusCode::OK
}
); );
assert_eq!( assert_eq!(
path_to_keys("/hello", "index.html", &[])?, path_to_keys("/hello", "index.html")?,
RoutingResult::LoadOrRedirect { (
key: "hello".to_string(), "hello".to_string(),
redirect_url: "/hello/".to_string(), ImplicitRedirect::To {
redirect_if_exists: Some("hello/index.html".to_string()), key: "hello/index.html".to_string(),
redirect_code: StatusCode::FOUND, url: "/hello/".to_string()
} }
)
); );
assert!(path_to_keys("", "index.html", &[]).is_err()); assert!(path_to_keys("", "index.html").is_err());
assert!(path_to_keys("i/am/relative", "index.html", &[]).is_err()); assert!(path_to_keys("i/am/relative", "index.html").is_err());
Ok(()) Ok(())
} }
} }