mirror of
https://github.com/deuxfleurs-org/garage.git
synced 2026-08-16 16:58:19 +00:00
Compare commits
48 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8bccddb504 | |||
| 0b901bf291 | |||
| c8c20d6f47 | |||
| e5db610e4c | |||
| 65c6f8adea | |||
| 54b9bf02a3 | |||
| 469153233f | |||
| 90bba5889a | |||
| a64b567d43 | |||
| 6ea86db8cd | |||
| aa69c06f2b | |||
| a6c6c44310 | |||
| 96d7713915 | |||
| d64498c3d3 | |||
| b340599e68 | |||
| 5448012b27 | |||
| ce34d11a65 | |||
| 8cb7623ebd | |||
| 5469c95877 | |||
| f930c6f643 | |||
| afcb22bf16 | |||
| cc29a40d51 | |||
| 0f3f180c3e | |||
| 70cf6004ae | |||
| c7571ff89b | |||
| 1b42919bf7 | |||
| 3f4ab3a4a3 | |||
| 3a4afc04a9 | |||
| fbf03e9378 | |||
| 9eb07d4c7b | |||
| 85ee4f5d8c | |||
| 328072d122 | |||
| 26bc807905 | |||
| a9f5f242b2 | |||
| ae98abca5c | |||
| adfa44ad70 | |||
| 47143b88ad | |||
| 8843aa92fa | |||
| b601b3e46d | |||
| a19d2f16e2 | |||
| fc8fc60f6d | |||
| 77079a1498 | |||
| 2a4f729b57 | |||
| 1b042e379e | |||
| ffbce0f689 | |||
| 37e5621dde | |||
| 6529ff379a | |||
| 8b35a946d9 |
@@ -28,6 +28,11 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- nix-build -j4 --attr flakePackages.tests-sqlite
|
- nix-build -j4 --attr flakePackages.tests-sqlite
|
||||||
|
|
||||||
|
- name: unit + func tests (fjall)
|
||||||
|
image: nixpkgs/nix:nixos-22.05
|
||||||
|
commands:
|
||||||
|
- nix-build -j4 --attr flakePackages.tests-fjall
|
||||||
|
|
||||||
- name: integration tests
|
- name: integration tests
|
||||||
image: nixpkgs/nix:nixos-22.05
|
image: nixpkgs/nix:nixos-22.05
|
||||||
commands:
|
commands:
|
||||||
|
|||||||
Generated
+225
-16
@@ -687,6 +687,12 @@ version = "1.3.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2d2c12f985c78475a6b8d629afd0c360260ef34cfef52efccdcfd31972f81c2e"
|
checksum = "2d2c12f985c78475a6b8d629afd0c360260ef34cfef52efccdcfd31972f81c2e"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "byteview"
|
||||||
|
version = "0.6.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cc"
|
name = "cc"
|
||||||
version = "1.2.16"
|
version = "1.2.16"
|
||||||
@@ -798,6 +804,12 @@ version = "1.0.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
|
checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "compare"
|
||||||
|
version = "0.0.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "core-foundation"
|
name = "core-foundation"
|
||||||
version = "0.9.4"
|
version = "0.9.4"
|
||||||
@@ -867,9 +879,18 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crossbeam-channel"
|
name = "crossbeam-channel"
|
||||||
version = "0.5.14"
|
version = "0.5.15"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471"
|
checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
|
||||||
|
dependencies = [
|
||||||
|
"crossbeam-utils",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crossbeam-epoch"
|
||||||
|
version = "0.9.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"crossbeam-utils",
|
"crossbeam-utils",
|
||||||
]
|
]
|
||||||
@@ -883,6 +904,16 @@ dependencies = [
|
|||||||
"crossbeam-utils",
|
"crossbeam-utils",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "crossbeam-skiplist"
|
||||||
|
version = "0.1.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b"
|
||||||
|
dependencies = [
|
||||||
|
"crossbeam-epoch",
|
||||||
|
"crossbeam-utils",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crossbeam-utils"
|
name = "crossbeam-utils"
|
||||||
version = "0.8.21"
|
version = "0.8.21"
|
||||||
@@ -954,6 +985,20 @@ dependencies = [
|
|||||||
"num_cpus",
|
"num_cpus",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "dashmap"
|
||||||
|
version = "6.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"crossbeam-utils",
|
||||||
|
"hashbrown 0.14.5",
|
||||||
|
"lock_api",
|
||||||
|
"once_cell",
|
||||||
|
"parking_lot_core 0.9.10",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "deranged"
|
name = "deranged"
|
||||||
version = "0.4.0"
|
version = "0.4.0"
|
||||||
@@ -996,6 +1041,12 @@ dependencies = [
|
|||||||
"syn 2.0.100",
|
"syn 2.0.100",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "double-ended-peekable"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "dyn-clone"
|
name = "dyn-clone"
|
||||||
version = "1.0.19"
|
version = "1.0.19"
|
||||||
@@ -1017,6 +1068,18 @@ dependencies = [
|
|||||||
"cfg-if",
|
"cfg-if",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "enum_dispatch"
|
||||||
|
version = "0.3.13"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd"
|
||||||
|
dependencies = [
|
||||||
|
"once_cell",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.100",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "env_logger"
|
name = "env_logger"
|
||||||
version = "0.10.2"
|
version = "0.10.2"
|
||||||
@@ -1084,6 +1147,23 @@ version = "0.4.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
|
checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fjall"
|
||||||
|
version = "2.11.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder",
|
||||||
|
"byteview",
|
||||||
|
"dashmap 6.1.0",
|
||||||
|
"log",
|
||||||
|
"lsm-tree",
|
||||||
|
"path-absolutize",
|
||||||
|
"std-semaphore",
|
||||||
|
"tempfile",
|
||||||
|
"xxhash-rust",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fnv"
|
name = "fnv"
|
||||||
version = "1.0.7"
|
version = "1.0.7"
|
||||||
@@ -1200,7 +1280,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage"
|
name = "garage"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"assert-json-diff",
|
"assert-json-diff",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -1241,6 +1321,7 @@ dependencies = [
|
|||||||
"opentelemetry-prometheus",
|
"opentelemetry-prometheus",
|
||||||
"parse_duration",
|
"parse_duration",
|
||||||
"serde",
|
"serde",
|
||||||
|
"serde_bytes",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha1",
|
"sha1",
|
||||||
"sha2",
|
"sha2",
|
||||||
@@ -1250,12 +1331,13 @@ dependencies = [
|
|||||||
"timeago",
|
"timeago",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
"tracing-journald",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_api_admin"
|
name = "garage_api_admin"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"argon2",
|
"argon2",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -1281,7 +1363,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_api_common"
|
name = "garage_api_common"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64 0.21.7",
|
"base64 0.21.7",
|
||||||
"bytes",
|
"bytes",
|
||||||
@@ -1315,7 +1397,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_api_k2v"
|
name = "garage_api_k2v"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64 0.21.7",
|
"base64 0.21.7",
|
||||||
"err-derive",
|
"err-derive",
|
||||||
@@ -1338,7 +1420,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_api_s3"
|
name = "garage_api_s3"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
"async-compression",
|
"async-compression",
|
||||||
@@ -1383,7 +1465,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_block"
|
name = "garage_block"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-compression",
|
"async-compression",
|
||||||
@@ -1408,11 +1490,13 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_db"
|
name = "garage_db"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"err-derive",
|
"err-derive",
|
||||||
|
"fjall",
|
||||||
"heed",
|
"heed",
|
||||||
"mktemp",
|
"mktemp",
|
||||||
|
"parking_lot 0.12.3",
|
||||||
"r2d2",
|
"r2d2",
|
||||||
"r2d2_sqlite",
|
"r2d2_sqlite",
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
@@ -1421,7 +1505,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_model"
|
name = "garage_model"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"base64 0.21.7",
|
"base64 0.21.7",
|
||||||
@@ -1448,7 +1532,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_net"
|
name = "garage_net"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"bytes",
|
"bytes",
|
||||||
@@ -1473,7 +1557,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_rpc"
|
name = "garage_rpc"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -1505,7 +1589,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_table"
|
name = "garage_table"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -1526,7 +1610,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_util"
|
name = "garage_util"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -1558,7 +1642,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_web"
|
name = "garage_web"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"err-derive",
|
"err-derive",
|
||||||
"garage_api_common",
|
"garage_api_common",
|
||||||
@@ -1654,6 +1738,12 @@ dependencies = [
|
|||||||
"syn 2.0.100",
|
"syn 2.0.100",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "guardian"
|
||||||
|
version = "1.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "h2"
|
name = "h2"
|
||||||
version = "0.3.26"
|
version = "0.3.26"
|
||||||
@@ -2228,6 +2318,15 @@ dependencies = [
|
|||||||
"cfg-if",
|
"cfg-if",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "interval-heap"
|
||||||
|
version = "0.0.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6"
|
||||||
|
dependencies = [
|
||||||
|
"compare",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ipnet"
|
name = "ipnet"
|
||||||
version = "2.11.0"
|
version = "2.11.0"
|
||||||
@@ -2584,6 +2683,36 @@ dependencies = [
|
|||||||
"hashbrown 0.15.2",
|
"hashbrown 0.15.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lsm-tree"
|
||||||
|
version = "2.10.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ab73c02eadb3dc12c0024e5b61d6284e6d59064e67e74fbad77856caa56f62c7"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder",
|
||||||
|
"crossbeam-skiplist",
|
||||||
|
"double-ended-peekable",
|
||||||
|
"enum_dispatch",
|
||||||
|
"guardian",
|
||||||
|
"interval-heap",
|
||||||
|
"log",
|
||||||
|
"lz4_flex",
|
||||||
|
"path-absolutize",
|
||||||
|
"quick_cache",
|
||||||
|
"rustc-hash",
|
||||||
|
"self_cell",
|
||||||
|
"tempfile",
|
||||||
|
"value-log",
|
||||||
|
"varint-rs",
|
||||||
|
"xxhash-rust",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lz4_flex"
|
||||||
|
version = "0.11.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "matchers"
|
name = "matchers"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
@@ -2838,7 +2967,7 @@ checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"crossbeam-channel",
|
"crossbeam-channel",
|
||||||
"dashmap",
|
"dashmap 4.0.2",
|
||||||
"fnv",
|
"fnv",
|
||||||
"futures-channel",
|
"futures-channel",
|
||||||
"futures-executor",
|
"futures-executor",
|
||||||
@@ -2999,6 +3128,24 @@ version = "1.0.15"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
|
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "path-absolutize"
|
||||||
|
version = "3.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5"
|
||||||
|
dependencies = [
|
||||||
|
"path-dedot",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "path-dedot"
|
||||||
|
version = "3.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397"
|
||||||
|
dependencies = [
|
||||||
|
"once_cell",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pem"
|
name = "pem"
|
||||||
version = "3.0.5"
|
version = "3.0.5"
|
||||||
@@ -3294,6 +3441,16 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quick_cache"
|
||||||
|
version = "0.6.16"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786"
|
||||||
|
dependencies = [
|
||||||
|
"equivalent",
|
||||||
|
"hashbrown 0.15.2",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "quote"
|
name = "quote"
|
||||||
version = "1.0.40"
|
version = "1.0.40"
|
||||||
@@ -3531,6 +3688,12 @@ version = "0.1.24"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
|
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustc-hash"
|
||||||
|
version = "2.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustc_version"
|
name = "rustc_version"
|
||||||
version = "0.4.1"
|
version = "0.4.1"
|
||||||
@@ -3773,6 +3936,12 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "self_cell"
|
||||||
|
version = "1.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "semver"
|
name = "semver"
|
||||||
version = "1.0.26"
|
version = "1.0.26"
|
||||||
@@ -3986,6 +4155,12 @@ dependencies = [
|
|||||||
"syn 1.0.109",
|
"syn 1.0.109",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "std-semaphore"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "strsim"
|
name = "strsim"
|
||||||
version = "0.11.1"
|
version = "0.11.1"
|
||||||
@@ -4514,6 +4689,17 @@ dependencies = [
|
|||||||
"tracing",
|
"tracing",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tracing-journald"
|
||||||
|
version = "0.3.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fc0b4143302cf1022dac868d521e36e8b27691f72c84b3311750d5188ebba657"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"tracing-core",
|
||||||
|
"tracing-subscriber",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tracing-log"
|
name = "tracing-log"
|
||||||
version = "0.2.0"
|
version = "0.2.0"
|
||||||
@@ -4652,6 +4838,29 @@ version = "0.1.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
|
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "value-log"
|
||||||
|
version = "1.9.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder",
|
||||||
|
"byteview",
|
||||||
|
"interval-heap",
|
||||||
|
"log",
|
||||||
|
"path-absolutize",
|
||||||
|
"rustc-hash",
|
||||||
|
"tempfile",
|
||||||
|
"varint-rs",
|
||||||
|
"xxhash-rust",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "varint-rs"
|
||||||
|
version = "2.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "vcpkg"
|
name = "vcpkg"
|
||||||
version = "0.2.15"
|
version = "0.2.15"
|
||||||
|
|||||||
+15
-12
@@ -24,18 +24,18 @@ default-members = ["src/garage"]
|
|||||||
|
|
||||||
# Internal Garage crates
|
# Internal Garage crates
|
||||||
format_table = { version = "0.1.1", path = "src/format-table" }
|
format_table = { version = "0.1.1", path = "src/format-table" }
|
||||||
garage_api_common = { version = "1.1.0", path = "src/api/common" }
|
garage_api_common = { version = "1.2.0", path = "src/api/common" }
|
||||||
garage_api_admin = { version = "1.1.0", path = "src/api/admin" }
|
garage_api_admin = { version = "1.2.0", path = "src/api/admin" }
|
||||||
garage_api_s3 = { version = "1.1.0", path = "src/api/s3" }
|
garage_api_s3 = { version = "1.2.0", path = "src/api/s3" }
|
||||||
garage_api_k2v = { version = "1.1.0", path = "src/api/k2v" }
|
garage_api_k2v = { version = "1.2.0", path = "src/api/k2v" }
|
||||||
garage_block = { version = "1.1.0", path = "src/block" }
|
garage_block = { version = "1.2.0", path = "src/block" }
|
||||||
garage_db = { version = "1.1.0", path = "src/db", default-features = false }
|
garage_db = { version = "1.2.0", path = "src/db", default-features = false }
|
||||||
garage_model = { version = "1.1.0", path = "src/model", default-features = false }
|
garage_model = { version = "1.2.0", path = "src/model", default-features = false }
|
||||||
garage_net = { version = "1.1.0", path = "src/net" }
|
garage_net = { version = "1.2.0", path = "src/net" }
|
||||||
garage_rpc = { version = "1.1.0", path = "src/rpc" }
|
garage_rpc = { version = "1.2.0", path = "src/rpc" }
|
||||||
garage_table = { version = "1.1.0", path = "src/table" }
|
garage_table = { version = "1.2.0", path = "src/table" }
|
||||||
garage_util = { version = "1.1.0", path = "src/util" }
|
garage_util = { version = "1.2.0", path = "src/util" }
|
||||||
garage_web = { version = "1.1.0", path = "src/web" }
|
garage_web = { version = "1.2.0", path = "src/web" }
|
||||||
k2v-client = { version = "0.0.4", path = "src/k2v-client" }
|
k2v-client = { version = "0.0.4", path = "src/k2v-client" }
|
||||||
|
|
||||||
# External crates from crates.io
|
# External crates from crates.io
|
||||||
@@ -65,6 +65,7 @@ md-5 = "0.10"
|
|||||||
mktemp = "0.5"
|
mktemp = "0.5"
|
||||||
nix = { version = "0.29", default-features = false, features = ["fs"] }
|
nix = { version = "0.29", default-features = false, features = ["fs"] }
|
||||||
nom = "7.1"
|
nom = "7.1"
|
||||||
|
parking_lot = "0.12"
|
||||||
parse_duration = "2.1"
|
parse_duration = "2.1"
|
||||||
pin-project = "1.0.12"
|
pin-project = "1.0.12"
|
||||||
pnet_datalink = "0.34"
|
pnet_datalink = "0.34"
|
||||||
@@ -83,12 +84,14 @@ pretty_env_logger = "0.5"
|
|||||||
structopt = { version = "0.3", default-features = false }
|
structopt = { version = "0.3", default-features = false }
|
||||||
syslog-tracing = "0.3"
|
syslog-tracing = "0.3"
|
||||||
tracing = "0.1"
|
tracing = "0.1"
|
||||||
|
tracing-journald = "0.3.1"
|
||||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
|
|
||||||
heed = { version = "0.11", default-features = false, features = ["lmdb"] }
|
heed = { version = "0.11", default-features = false, features = ["lmdb"] }
|
||||||
rusqlite = "0.31.0"
|
rusqlite = "0.31.0"
|
||||||
r2d2 = "0.8"
|
r2d2 = "0.8"
|
||||||
r2d2_sqlite = "0.24"
|
r2d2_sqlite = "0.24"
|
||||||
|
fjall = "2.4"
|
||||||
|
|
||||||
async-compression = { version = "0.4", features = ["tokio", "zstd"] }
|
async-compression = { version = "0.4", features = ["tokio", "zstd"] }
|
||||||
zstd = { version = "0.13", default-features = false }
|
zstd = { version = "0.13", default-features = false }
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ In this section, we cover the following web applications:
|
|||||||
| [Mastodon](#mastodon) | ✅ | Natively supported |
|
| [Mastodon](#mastodon) | ✅ | Natively supported |
|
||||||
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
|
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
|
||||||
| [ejabberd](#ejabberd) | ✅ | `mod_s3_upload` |
|
| [ejabberd](#ejabberd) | ✅ | `mod_s3_upload` |
|
||||||
| [Pixelfed](#pixelfed) | ❓ | Not yet tested |
|
| [Pixelfed](#pixelfed) | ✅ | Natively supported |
|
||||||
| [Pleroma](#pleroma) | ❓ | Not yet tested |
|
| [Pleroma](#pleroma) | ❓ | Not yet tested |
|
||||||
| [Lemmy](#lemmy) | ✅ | Supported with pict-rs |
|
| [Lemmy](#lemmy) | ✅ | Supported with pict-rs |
|
||||||
| [Funkwhale](#funkwhale) | ❓ | Not yet tested |
|
| [Funkwhale](#funkwhale) | ❓ | Not yet tested |
|
||||||
@@ -191,10 +191,10 @@ garage key create peertube-key
|
|||||||
|
|
||||||
Keep the Key ID and the Secret key in a pad, they will be needed later.
|
Keep the Key ID and the Secret key in a pad, they will be needed later.
|
||||||
|
|
||||||
We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
|
We need two buckets, one for normal videos (named peertube-videos) and one for webtorrent videos (named peertube-playlists).
|
||||||
```bash
|
```bash
|
||||||
garage bucket create peertube-videos
|
garage bucket create peertube-videos
|
||||||
garage bucket create peertube-playlist
|
garage bucket create peertube-playlists
|
||||||
```
|
```
|
||||||
|
|
||||||
Now we allow our key to read and write on these buckets:
|
Now we allow our key to read and write on these buckets:
|
||||||
@@ -253,7 +253,7 @@ object_storage:
|
|||||||
proxify_private_files: false
|
proxify_private_files: false
|
||||||
|
|
||||||
streaming_playlists:
|
streaming_playlists:
|
||||||
bucket_name: 'peertube-playlist'
|
bucket_name: 'peertube-playlists'
|
||||||
|
|
||||||
# Keep it empty for our example
|
# Keep it empty for our example
|
||||||
prefix: ''
|
prefix: ''
|
||||||
|
|||||||
@@ -8,18 +8,18 @@ have published Ansible roles. We list them and compare them below.
|
|||||||
|
|
||||||
## Comparison of Ansible roles
|
## Comparison of Ansible roles
|
||||||
|
|
||||||
| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) |
|
| Feature | [ansible-role-garage](#zorun-ansible-role-garage) | [garage-docker-ansible-deploy](#moan0s-garage-docker-ansible-deploy) | [eddster ansible-role-garage](#eddster-ansible-role-garage) |
|
||||||
|------------------------------------|---------------------------------------------|---------------------------------------------------------------|
|
|------------------------------------|---------------------------------------------|---------------------------------------------------------------|---------------------------------|
|
||||||
| **Runtime** | Systemd | Docker |
|
| **Runtime** | Systemd | Docker | Systemd |
|
||||||
| **Target OS** | Any Linux | Any Linux |
|
| **Target OS** | Any Linux | Any Linux | Any Linux |
|
||||||
| **Architecture** | amd64, arm64, i686 | amd64, arm64 |
|
| **Architecture** | amd64, arm64, i686 | amd64, arm64 | arm64, arm, 386, amd64 |
|
||||||
| **Additional software** | None | Traefik |
|
| **Additional software** | None | Traefik | Ngnix and Keepalived (optional) |
|
||||||
| **Automatic node connection** | ❌ | ✅ |
|
| **Automatic node connection** | ❌ | ✅ | ✅ |
|
||||||
| **Layout management** | ❌ | ✅ |
|
| **Layout management** | ❌ | ✅ | ✅ |
|
||||||
| **Manage buckets & keys** | ❌ | ✅ (basic) |
|
| **Manage buckets & keys** | ❌ | ✅ (basic) | ✅ |
|
||||||
| **Allow custom Garage config** | ✅ | ❌ |
|
| **Allow custom Garage config** | ✅ | ❌ | ❌ |
|
||||||
| **Facilitate Garage upgrades** | ✅ | ❌ |
|
| **Facilitate Garage upgrades** | ✅ | ❌ | ✅ |
|
||||||
| **Multiple instances on one host** | ✅ | ✅ |
|
| **Multiple instances on one host** | ✅ | ✅ | ❌ |
|
||||||
|
|
||||||
|
|
||||||
## zorun/ansible-role-garage
|
## zorun/ansible-role-garage
|
||||||
@@ -49,3 +49,15 @@ structured DNS names, etc).
|
|||||||
|
|
||||||
As a result, this role makes it easier to start with Garage on Ansible,
|
As a result, this role makes it easier to start with Garage on Ansible,
|
||||||
but is less flexible.
|
but is less flexible.
|
||||||
|
|
||||||
|
## eddster2309/ansible-role-garage
|
||||||
|
|
||||||
|
[Source code](https://github.com/eddster2309/ansible-role-garage), [Ansible galaxy](https://galaxy.ansible.com/ui/standalone/roles/eddster2309/garage/)
|
||||||
|
|
||||||
|
This role is a opinionated but customisable role using the official Garage
|
||||||
|
static binaries and only requires Systemd. As such it should work on any
|
||||||
|
Linux based host. It includes all the nesscary configuration to
|
||||||
|
automatically setup a clustered Garage deployment. Most Garage
|
||||||
|
configuration options are exposed through Ansible variables so while you
|
||||||
|
can't provide a custom config you can get very close. It can optionally
|
||||||
|
installed a HA nginx deployment with Keepalived.
|
||||||
|
|||||||
@@ -96,14 +96,14 @@ to store 2 TB of data in total.
|
|||||||
## Get a Docker image
|
## Get a Docker image
|
||||||
|
|
||||||
Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
|
Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
|
||||||
We encourage you to use a fixed tag (eg. `v1.1.0`) and not the `latest` tag.
|
We encourage you to use a fixed tag (eg. `v1.2.0`) and not the `latest` tag.
|
||||||
For this example, we will use the latest published version at the time of the writing which is `v1.1.0` but it's up to you
|
For this example, we will use the latest published version at the time of the writing which is `v1.2.0` but it's up to you
|
||||||
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
|
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
|
||||||
|
|
||||||
For example:
|
For example:
|
||||||
|
|
||||||
```
|
```
|
||||||
sudo docker pull dxflrs/garage:v1.1.0
|
sudo docker pull dxflrs/garage:v1.2.0
|
||||||
```
|
```
|
||||||
|
|
||||||
## Deploying and configuring Garage
|
## Deploying and configuring Garage
|
||||||
@@ -171,7 +171,7 @@ docker run \
|
|||||||
-v /etc/garage.toml:/etc/garage.toml \
|
-v /etc/garage.toml:/etc/garage.toml \
|
||||||
-v /var/lib/garage/meta:/var/lib/garage/meta \
|
-v /var/lib/garage/meta:/var/lib/garage/meta \
|
||||||
-v /var/lib/garage/data:/var/lib/garage/data \
|
-v /var/lib/garage/data:/var/lib/garage/data \
|
||||||
dxflrs/garage:v1.1.0
|
dxflrs/garage:v1.2.0
|
||||||
```
|
```
|
||||||
|
|
||||||
With this command line, Garage should be started automatically at each boot.
|
With this command line, Garage should be started automatically at each boot.
|
||||||
@@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
|
|||||||
version: "3"
|
version: "3"
|
||||||
services:
|
services:
|
||||||
garage:
|
garage:
|
||||||
image: dxflrs/garage:v1.1.0
|
image: dxflrs/garage:v1.2.0
|
||||||
network_mode: "host"
|
network_mode: "host"
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
volumes:
|
volumes:
|
||||||
|
|||||||
@@ -132,7 +132,7 @@ docker run \
|
|||||||
-v /path/to/garage.toml:/etc/garage.toml \
|
-v /path/to/garage.toml:/etc/garage.toml \
|
||||||
-v /path/to/garage/meta:/var/lib/garage/meta \
|
-v /path/to/garage/meta:/var/lib/garage/meta \
|
||||||
-v /path/to/garage/data:/var/lib/garage/data \
|
-v /path/to/garage/data:/var/lib/garage/data \
|
||||||
dxflrs/garage:v1.1.0
|
dxflrs/garage:v1.2.0
|
||||||
```
|
```
|
||||||
|
|
||||||
Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903`
|
Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903`
|
||||||
@@ -182,11 +182,12 @@ ID Hostname Address Tag Zone Capacit
|
|||||||
## Creating a cluster layout
|
## Creating a cluster layout
|
||||||
|
|
||||||
Creating a cluster layout for a Garage deployment means informing Garage
|
Creating a cluster layout for a Garage deployment means informing Garage
|
||||||
of the disk space available on each node of the cluster
|
of the disk space available on each node of the cluster, `-c`,
|
||||||
as well as the zone (e.g. datacenter) each machine is located in.
|
as well as the name of the zone (e.g. datacenter), `-z`, each machine is located in.
|
||||||
|
|
||||||
For our test deployment, we are using only one node. The way in which we configure
|
For our test deployment, we are have only one node with zone named `dc1` and a
|
||||||
it does not matter, you can simply write:
|
capacity of `1G`, though the capacity is ignored for a single node deployment
|
||||||
|
and can be changed later when adding new nodes.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
garage layout assign -z dc1 -c 1G <node_id>
|
garage layout assign -z dc1 -c 1G <node_id>
|
||||||
|
|||||||
@@ -93,30 +93,30 @@ The following gives details about each available configuration option.
|
|||||||
|
|
||||||
[Environment variables](#env_variables).
|
[Environment variables](#env_variables).
|
||||||
|
|
||||||
Top-level configuration options:
|
Top-level configuration options, in alphabetical order:
|
||||||
|
[`allow_punycode`](#allow_punycode),
|
||||||
[`allow_world_readable_secrets`](#allow_world_readable_secrets),
|
[`allow_world_readable_secrets`](#allow_world_readable_secrets),
|
||||||
[`block_ram_buffer_max`](#block_ram_buffer_max),
|
[`block_ram_buffer_max`](#block_ram_buffer_max),
|
||||||
[`block_size`](#block_size),
|
[`block_size`](#block_size),
|
||||||
[`bootstrap_peers`](#bootstrap_peers),
|
[`bootstrap_peers`](#bootstrap_peers),
|
||||||
[`compression_level`](#compression_level),
|
[`compression_level`](#compression_level),
|
||||||
|
[`consistency_mode`](#consistency_mode),
|
||||||
[`data_dir`](#data_dir),
|
[`data_dir`](#data_dir),
|
||||||
[`data_fsync`](#data_fsync),
|
[`data_fsync`](#data_fsync),
|
||||||
[`db_engine`](#db_engine),
|
[`db_engine`](#db_engine),
|
||||||
[`disable_scrub`](#disable_scrub),
|
[`disable_scrub`](#disable_scrub),
|
||||||
[`use_local_tz`](#use_local_tz),
|
|
||||||
[`lmdb_map_size`](#lmdb_map_size),
|
[`lmdb_map_size`](#lmdb_map_size),
|
||||||
[`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval),
|
[`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval),
|
||||||
[`metadata_dir`](#metadata_dir),
|
[`metadata_dir`](#metadata_dir),
|
||||||
[`metadata_fsync`](#metadata_fsync),
|
[`metadata_fsync`](#metadata_fsync),
|
||||||
[`metadata_snapshots_dir`](#metadata_snapshots_dir),
|
[`metadata_snapshots_dir`](#metadata_snapshots_dir),
|
||||||
[`replication_factor`](#replication_factor),
|
[`replication_factor`](#replication_factor),
|
||||||
[`consistency_mode`](#consistency_mode),
|
|
||||||
[`rpc_bind_addr`](#rpc_bind_addr),
|
[`rpc_bind_addr`](#rpc_bind_addr),
|
||||||
[`rpc_bind_outgoing`](#rpc_bind_outgoing),
|
[`rpc_bind_outgoing`](#rpc_bind_outgoing),
|
||||||
[`rpc_public_addr`](#rpc_public_addr),
|
[`rpc_public_addr`](#rpc_public_addr),
|
||||||
[`rpc_public_addr_subnet`](#rpc_public_addr_subnet)
|
[`rpc_public_addr_subnet`](#rpc_public_addr_subnet)
|
||||||
[`rpc_secret`/`rpc_secret_file`](#rpc_secret).
|
[`rpc_secret`/`rpc_secret_file`](#rpc_secret),
|
||||||
[`allow_punycode`](#allow_punycode).
|
[`use_local_tz`](#use_local_tz).
|
||||||
|
|
||||||
The `[consul_discovery]` section:
|
The `[consul_discovery]` section:
|
||||||
[`api`](#consul_api),
|
[`api`](#consul_api),
|
||||||
@@ -153,13 +153,17 @@ The `[admin]` section:
|
|||||||
|
|
||||||
### Environment variables {#env_variables}
|
### Environment variables {#env_variables}
|
||||||
|
|
||||||
The following configuration parameter must be specified as an environment
|
The following configuration parameters must be specified as environment variables,
|
||||||
variable, it does not exist in the configuration file:
|
they do not exist in the configuration file:
|
||||||
|
|
||||||
- `GARAGE_LOG_TO_SYSLOG` (since `v0.9.4`): set this to `1` or `true` to make the
|
- `GARAGE_LOG_TO_SYSLOG` (since `v0.9.4`): set this to `1` or `true` to make the
|
||||||
Garage daemon send its logs to `syslog` (using the libc `syslog` function)
|
Garage daemon send its logs to `syslog` (using the libc `syslog` function)
|
||||||
instead of printing to stderr.
|
instead of printing to stderr.
|
||||||
|
|
||||||
|
- `GARAGE_LOG_TO_JOURNALD` (since `v1.2.0`): set this to `1` or `true` to make the
|
||||||
|
Garage daemon send its logs to `journald` (using the native protocol of `systemd-journald`)
|
||||||
|
instead of printing to stderr.
|
||||||
|
|
||||||
The following environment variables can be used to override the corresponding
|
The following environment variables can be used to override the corresponding
|
||||||
values in the configuration file:
|
values in the configuration file:
|
||||||
|
|
||||||
@@ -171,7 +175,7 @@ values in the configuration file:
|
|||||||
|
|
||||||
### Top-level configuration options
|
### Top-level configuration options
|
||||||
|
|
||||||
#### `replication_factor` {#replication_factor}
|
#### `replication_factor` (since `v1.0.0`) {#replication_factor}
|
||||||
|
|
||||||
The replication factor can be any positive integer smaller or equal the node count in your cluster.
|
The replication factor can be any positive integer smaller or equal the node count in your cluster.
|
||||||
The chosen replication factor has a big impact on the cluster's failure tolerancy and performance characteristics.
|
The chosen replication factor has a big impact on the cluster's failure tolerancy and performance characteristics.
|
||||||
@@ -219,7 +223,7 @@ is in progress. In theory, no data should be lost as rebalancing is a
|
|||||||
routine operation for Garage, although we cannot guarantee you that everything
|
routine operation for Garage, although we cannot guarantee you that everything
|
||||||
will go right in such an extreme scenario.
|
will go right in such an extreme scenario.
|
||||||
|
|
||||||
#### `consistency_mode` {#consistency_mode}
|
#### `consistency_mode` (since `v1.0.0`) {#consistency_mode}
|
||||||
|
|
||||||
The consistency mode setting determines the read and write behaviour of your cluster.
|
The consistency mode setting determines the read and write behaviour of your cluster.
|
||||||
|
|
||||||
@@ -329,6 +333,7 @@ Since `v0.8.0`, Garage can use alternative storage backends as follows:
|
|||||||
| --------- | ----------------- | ------------- |
|
| --------- | ----------------- | ------------- |
|
||||||
| [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
|
| [LMDB](https://www.symas.com/lmdb) (since `v0.8.0`, default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
|
||||||
| [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
|
| [Sqlite](https://sqlite.org) (since `v0.8.0`) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
|
||||||
|
| [Fjall](https://github.com/fjall-rs/fjall) (**experimental support** since `v1.3.0`) | `"fjall"` | `<metadata_dir>/db.fjall/` |
|
||||||
| [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` |
|
| [Sled](https://sled.rs) (old default, removed since `v1.0`) | `"sled"` | `<metadata_dir>/db/` |
|
||||||
|
|
||||||
Sled was supported until Garage v0.9.x, and was removed in Garage v1.0.
|
Sled was supported until Garage v0.9.x, and was removed in Garage v1.0.
|
||||||
@@ -365,6 +370,14 @@ LMDB works very well, but is known to have the following limitations:
|
|||||||
so it is not the best choice for high-performance storage clusters,
|
so it is not the best choice for high-performance storage clusters,
|
||||||
but it should work fine in many cases.
|
but it should work fine in many cases.
|
||||||
|
|
||||||
|
- Fjall: a storage engine based on LSM trees, which theoretically allow for
|
||||||
|
higher write throughput than other storage engines that are based on B-trees.
|
||||||
|
Using Fjall could potentially improve Garage's performance significantly in
|
||||||
|
write-heavy workloads. **Support for Fjall is experimental at this point**,
|
||||||
|
we have added it to Garage for evaluation purposes only. **Do not use it for
|
||||||
|
production-critical workloads.**
|
||||||
|
|
||||||
|
|
||||||
It is possible to convert Garage's metadata directory from one format to another
|
It is possible to convert Garage's metadata directory from one format to another
|
||||||
using the `garage convert-db` command, which should be used as follows:
|
using the `garage convert-db` command, which should be used as follows:
|
||||||
|
|
||||||
@@ -402,6 +415,7 @@ Here is how this option impacts the different database engines:
|
|||||||
|----------|------------------------------------|-------------------------------|
|
|----------|------------------------------------|-------------------------------|
|
||||||
| Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` |
|
| Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` |
|
||||||
| LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` |
|
| LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` |
|
||||||
|
| Fjall | default options | not supported |
|
||||||
|
|
||||||
Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).
|
Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the
|
|||||||
- 2022-05-25 - Many Ceph S3 endpoints are not documented but implemented. Following a notification from the Ceph community, we added them.
|
- 2022-05-25 - Many Ceph S3 endpoints are not documented but implemented. Following a notification from the Ceph community, we added them.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## High-level features
|
## High-level features
|
||||||
|
|
||||||
| Feature | Garage | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) |
|
| Feature | Garage | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) |
|
||||||
@@ -34,6 +33,7 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the
|
|||||||
| [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) | ✅ Implemented | ❌| ✅| ✅ | ✅ |
|
| [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) | ✅ Implemented | ❌| ✅| ✅ | ✅ |
|
||||||
| [Presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html) | ✅ Implemented | ❌| ✅ | ✅ | ✅(❓) |
|
| [Presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html) | ✅ Implemented | ❌| ✅ | ✅ | ✅(❓) |
|
||||||
| [SSE-C encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) | ✅ Implemented | ❓ | ✅ | ❌ | ✅ |
|
| [SSE-C encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) | ✅ Implemented | ❓ | ✅ | ❌ | ✅ |
|
||||||
|
| [Bucket versioning](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html) | ❌ Missing | ✅ | ✅ | ❌ | ✅ |
|
||||||
|
|
||||||
*Note:* OpenIO does not says if it supports presigned URLs. Because it is part
|
*Note:* OpenIO does not says if it supports presigned URLs. Because it is part
|
||||||
of signature v4 and they claim they support it without additional precisions,
|
of signature v4 and they claim they support it without additional precisions,
|
||||||
|
|||||||
@@ -70,7 +70,7 @@ Example response body:
|
|||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"node": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df",
|
"node": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df",
|
||||||
"garageVersion": "v1.1.0",
|
"garageVersion": "v1.2.0",
|
||||||
"garageFeatures": [
|
"garageFeatures": [
|
||||||
"k2v",
|
"k2v",
|
||||||
"lmdb",
|
"lmdb",
|
||||||
|
|||||||
@@ -53,6 +53,9 @@
|
|||||||
tests-sqlite = testWith {
|
tests-sqlite = testWith {
|
||||||
GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite";
|
GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite";
|
||||||
};
|
};
|
||||||
|
tests-fjall = testWith {
|
||||||
|
GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall";
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
# ---- developpment shell, for making native builds only ----
|
# ---- developpment shell, for making native builds only ----
|
||||||
|
|||||||
+2
-1
@@ -68,12 +68,13 @@ let
|
|||||||
rootFeatures = if features != null then
|
rootFeatures = if features != null then
|
||||||
features
|
features
|
||||||
else
|
else
|
||||||
([ "bundled-libs" "lmdb" "sqlite" "k2v" ] ++ (lib.optionals release [
|
([ "bundled-libs" "lmdb" "sqlite" "fjall" "k2v" ] ++ (lib.optionals release [
|
||||||
"consul-discovery"
|
"consul-discovery"
|
||||||
"kubernetes-discovery"
|
"kubernetes-discovery"
|
||||||
"metrics"
|
"metrics"
|
||||||
"telemetry-otlp"
|
"telemetry-otlp"
|
||||||
"syslog"
|
"syslog"
|
||||||
|
"journald"
|
||||||
]));
|
]));
|
||||||
|
|
||||||
featuresStr = lib.concatStringsSep "," rootFeatures;
|
featuresStr = lib.concatStringsSep "," rootFeatures;
|
||||||
|
|||||||
@@ -2,8 +2,8 @@ apiVersion: v2
|
|||||||
name: garage
|
name: garage
|
||||||
description: S3-compatible object store for small self-hosted geo-distributed deployments
|
description: S3-compatible object store for small self-hosted geo-distributed deployments
|
||||||
type: application
|
type: application
|
||||||
version: 0.7.0
|
version: 0.7.1
|
||||||
appVersion: "v1.1.0"
|
appVersion: "v1.2.0"
|
||||||
home: https://garagehq.deuxfleurs.fr/
|
home: https://garagehq.deuxfleurs.fr/
|
||||||
icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg
|
icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg
|
||||||
|
|
||||||
@@ -15,4 +15,4 @@ keywords:
|
|||||||
sources:
|
sources:
|
||||||
- https://git.deuxfleurs.fr/Deuxfleurs/garage.git
|
- https://git.deuxfleurs.fr/Deuxfleurs/garage.git
|
||||||
|
|
||||||
maintainers: []
|
maintainers: []
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# garage
|
# garage
|
||||||
|
|
||||||
  
|
  
|
||||||
|
|
||||||
S3-compatible object store for small self-hosted geo-distributed deployments
|
S3-compatible object store for small self-hosted geo-distributed deployments
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_api_admin"
|
name = "garage_api_admin"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_api_common"
|
name = "garage_api_common"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ use garage_util::metrics::{gen_trace_id, RecordDuration};
|
|||||||
use garage_util::socket_address::UnixOrTCPSocketAddress;
|
use garage_util::socket_address::UnixOrTCPSocketAddress;
|
||||||
|
|
||||||
use crate::helpers::{BoxBody, ErrorBody};
|
use crate::helpers::{BoxBody, ErrorBody};
|
||||||
|
use crate::signature::payload::Authorization;
|
||||||
|
|
||||||
pub trait ApiEndpoint: Send + Sync + 'static {
|
pub trait ApiEndpoint: Send + Sync + 'static {
|
||||||
fn name(&self) -> &'static str;
|
fn name(&self) -> &'static str;
|
||||||
@@ -58,6 +59,12 @@ pub trait ApiHandler: Send + Sync + 'static {
|
|||||||
req: Request<IncomingBody>,
|
req: Request<IncomingBody>,
|
||||||
endpoint: Self::Endpoint,
|
endpoint: Self::Endpoint,
|
||||||
) -> impl Future<Output = Result<Response<BoxBody<Self::Error>>, Self::Error>> + Send;
|
) -> impl Future<Output = Result<Response<BoxBody<Self::Error>>, Self::Error>> + Send;
|
||||||
|
|
||||||
|
/// Returns the key id used to authenticate this request. The ID returned must be safe to
|
||||||
|
/// log.
|
||||||
|
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
|
||||||
|
None
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct ApiServer<A: ApiHandler> {
|
pub struct ApiServer<A: ApiHandler> {
|
||||||
@@ -142,19 +149,20 @@ impl<A: ApiHandler> ApiServer<A> {
|
|||||||
) -> Result<Response<BoxBody<A::Error>>, http::Error> {
|
) -> Result<Response<BoxBody<A::Error>>, http::Error> {
|
||||||
let uri = req.uri().clone();
|
let uri = req.uri().clone();
|
||||||
|
|
||||||
if let Ok(forwarded_for_ip_addr) =
|
let source = if let Ok(forwarded_for_ip_addr) =
|
||||||
forwarded_headers::handle_forwarded_for_headers(req.headers())
|
forwarded_headers::handle_forwarded_for_headers(req.headers())
|
||||||
{
|
{
|
||||||
info!(
|
format!("{forwarded_for_ip_addr} (via {addr})")
|
||||||
"{} (via {}) {} {}",
|
|
||||||
forwarded_for_ip_addr,
|
|
||||||
addr,
|
|
||||||
req.method(),
|
|
||||||
uri
|
|
||||||
);
|
|
||||||
} else {
|
} else {
|
||||||
info!("{} {} {}", addr, req.method(), uri);
|
format!("{addr}")
|
||||||
}
|
};
|
||||||
|
// we only do this to log the access key, so we can discard any error
|
||||||
|
let key = self
|
||||||
|
.api_handler
|
||||||
|
.key_id_from_request(&req)
|
||||||
|
.map(|k| format!("(key {k}) "))
|
||||||
|
.unwrap_or_default();
|
||||||
|
info!("{source} {key}{} {uri}", req.method());
|
||||||
debug!("{:?}", req);
|
debug!("{:?}", req);
|
||||||
|
|
||||||
let tracer = opentelemetry::global::tracer("garage");
|
let tracer = opentelemetry::global::tracer("garage");
|
||||||
@@ -343,7 +351,11 @@ where
|
|||||||
|
|
||||||
while !*must_exit.borrow() {
|
while !*must_exit.borrow() {
|
||||||
let (stream, client_addr) = tokio::select! {
|
let (stream, client_addr) = tokio::select! {
|
||||||
acc = listener.accept() => acc?,
|
acc = listener.accept() => match acc {
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::ConnectionAborted => continue,
|
||||||
|
Err(e) => return Err(e.into()),
|
||||||
|
},
|
||||||
_ = must_exit.changed() => continue,
|
_ = must_exit.changed() => continue,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -417,7 +417,7 @@ pub async fn verify_v4(
|
|||||||
// ============ Authorization header, or X-Amz-* query params =========
|
// ============ Authorization header, or X-Amz-* query params =========
|
||||||
|
|
||||||
pub struct Authorization {
|
pub struct Authorization {
|
||||||
key_id: String,
|
pub key_id: String,
|
||||||
scope: String,
|
scope: String,
|
||||||
signed_headers: String,
|
signed_headers: String,
|
||||||
signature: String,
|
signature: String,
|
||||||
@@ -426,7 +426,7 @@ pub struct Authorization {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Authorization {
|
impl Authorization {
|
||||||
fn parse_header(headers: &HeaderMap) -> Result<Self, Error> {
|
pub fn parse_header(headers: &HeaderMap) -> Result<Self, Error> {
|
||||||
let authorization = headers
|
let authorization = headers
|
||||||
.get(AUTHORIZATION)
|
.get(AUTHORIZATION)
|
||||||
.ok_or_bad_request("Missing authorization header")?
|
.ok_or_bad_request("Missing authorization header")?
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_api_k2v"
|
name = "garage_api_k2v"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|||||||
@@ -176,6 +176,12 @@ impl ApiHandler for K2VApiServer {
|
|||||||
|
|
||||||
Ok(resp_ok)
|
Ok(resp_ok)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
|
||||||
|
garage_api_common::signature::payload::Authorization::parse_header(req.headers())
|
||||||
|
.map(|auth| auth.key_id)
|
||||||
|
.ok()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ApiEndpoint for K2VApiEndpoint {
|
impl ApiEndpoint for K2VApiEndpoint {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_api_s3"
|
name = "garage_api_s3"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|||||||
@@ -226,6 +226,7 @@ impl ApiHandler for S3ApiServer {
|
|||||||
Endpoint::DeleteBucket {} => handle_delete_bucket(ctx).await,
|
Endpoint::DeleteBucket {} => handle_delete_bucket(ctx).await,
|
||||||
Endpoint::GetBucketLocation {} => handle_get_bucket_location(ctx),
|
Endpoint::GetBucketLocation {} => handle_get_bucket_location(ctx),
|
||||||
Endpoint::GetBucketVersioning {} => handle_get_bucket_versioning(),
|
Endpoint::GetBucketVersioning {} => handle_get_bucket_versioning(),
|
||||||
|
Endpoint::GetBucketAcl {} => handle_get_bucket_acl(ctx),
|
||||||
Endpoint::ListObjects {
|
Endpoint::ListObjects {
|
||||||
delimiter,
|
delimiter,
|
||||||
encoding_type,
|
encoding_type,
|
||||||
@@ -342,6 +343,12 @@ impl ApiHandler for S3ApiServer {
|
|||||||
|
|
||||||
Ok(resp_ok)
|
Ok(resp_ok)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
|
||||||
|
garage_api_common::signature::payload::Authorization::parse_header(req.headers())
|
||||||
|
.map(|auth| auth.key_id)
|
||||||
|
.ok()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ApiEndpoint for S3ApiEndpoint {
|
impl ApiEndpoint for S3ApiEndpoint {
|
||||||
|
|||||||
+59
-1
@@ -5,7 +5,7 @@ use hyper::{Request, Response, StatusCode};
|
|||||||
use garage_model::bucket_alias_table::*;
|
use garage_model::bucket_alias_table::*;
|
||||||
use garage_model::bucket_table::Bucket;
|
use garage_model::bucket_table::Bucket;
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
use garage_model::key_table::Key;
|
use garage_model::key_table::{Key, KeyParams};
|
||||||
use garage_model::permission::BucketKeyPerm;
|
use garage_model::permission::BucketKeyPerm;
|
||||||
use garage_table::util::*;
|
use garage_table::util::*;
|
||||||
use garage_util::crdt::*;
|
use garage_util::crdt::*;
|
||||||
@@ -44,6 +44,55 @@ pub fn handle_get_bucket_versioning() -> Result<Response<ResBody>, Error> {
|
|||||||
.body(string_body(xml))?)
|
.body(string_body(xml))?)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn handle_get_bucket_acl(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
|
||||||
|
let ReqCtx {
|
||||||
|
bucket_id, api_key, ..
|
||||||
|
} = ctx;
|
||||||
|
let key_p = api_key.params().ok_or_internal_error(
|
||||||
|
"Key should not be in deleted state at this point (in handle_get_bucket_acl)",
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let mut grants: Vec<s3_xml::Grant> = vec![];
|
||||||
|
let kp = api_key.bucket_permissions(&bucket_id);
|
||||||
|
|
||||||
|
if kp.allow_owner {
|
||||||
|
grants.push(s3_xml::Grant {
|
||||||
|
grantee: create_grantee(&key_p, &api_key),
|
||||||
|
permission: s3_xml::Value("FULL_CONTROL".to_string()),
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
if kp.allow_read {
|
||||||
|
grants.push(s3_xml::Grant {
|
||||||
|
grantee: create_grantee(&key_p, &api_key),
|
||||||
|
permission: s3_xml::Value("READ".to_string()),
|
||||||
|
});
|
||||||
|
grants.push(s3_xml::Grant {
|
||||||
|
grantee: create_grantee(&key_p, &api_key),
|
||||||
|
permission: s3_xml::Value("READ_ACP".to_string()),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if kp.allow_write {
|
||||||
|
grants.push(s3_xml::Grant {
|
||||||
|
grantee: create_grantee(&key_p, &api_key),
|
||||||
|
permission: s3_xml::Value("WRITE".to_string()),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let access_control_policy = s3_xml::AccessControlPolicy {
|
||||||
|
xmlns: (),
|
||||||
|
owner: None,
|
||||||
|
acl: s3_xml::AccessControlList { entries: grants },
|
||||||
|
};
|
||||||
|
|
||||||
|
let xml = s3_xml::to_xml_with_header(&access_control_policy)?;
|
||||||
|
trace!("xml: {}", xml);
|
||||||
|
|
||||||
|
Ok(Response::builder()
|
||||||
|
.header("Content-Type", "application/xml")
|
||||||
|
.body(string_body(xml))?)
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn handle_list_buckets(
|
pub async fn handle_list_buckets(
|
||||||
garage: &Garage,
|
garage: &Garage,
|
||||||
api_key: &Key,
|
api_key: &Key,
|
||||||
@@ -311,6 +360,15 @@ fn parse_create_bucket_xml(xml_bytes: &[u8]) -> Option<Option<String>> {
|
|||||||
Some(ret)
|
Some(ret)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn create_grantee(key_params: &KeyParams, api_key: &Key) -> s3_xml::Grantee {
|
||||||
|
s3_xml::Grantee {
|
||||||
|
xmlns_xsi: (),
|
||||||
|
typ: "CanonicalUser".to_string(),
|
||||||
|
display_name: Some(s3_xml::Value(key_params.name.get().to_string())),
|
||||||
|
id: Some(s3_xml::Value(api_key.key_id.to_string())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
+49
-28
@@ -26,7 +26,7 @@ use garage_api_common::signature::checksum::*;
|
|||||||
use crate::api_server::{ReqBody, ResBody};
|
use crate::api_server::{ReqBody, ResBody};
|
||||||
use crate::encryption::EncryptionParams;
|
use crate::encryption::EncryptionParams;
|
||||||
use crate::error::*;
|
use crate::error::*;
|
||||||
use crate::get::{full_object_byte_stream, PreconditionHeaders};
|
use crate::get::{check_version_not_deleted, full_object_byte_stream, PreconditionHeaders};
|
||||||
use crate::multipart;
|
use crate::multipart;
|
||||||
use crate::put::{extract_metadata_headers, save_stream, ChecksumMode, SaveStreamResult};
|
use crate::put::{extract_metadata_headers, save_stream, ChecksumMode, SaveStreamResult};
|
||||||
use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION;
|
use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION;
|
||||||
@@ -237,6 +237,7 @@ async fn handle_copy_metaonly(
|
|||||||
.get(&source_version.uuid, &EmptyKey)
|
.get(&source_version.uuid, &EmptyKey)
|
||||||
.await?;
|
.await?;
|
||||||
let source_version = source_version.ok_or(Error::NoSuchKey)?;
|
let source_version = source_version.ok_or(Error::NoSuchKey)?;
|
||||||
|
check_version_not_deleted(&source_version)?;
|
||||||
|
|
||||||
// Write an "uploading" marker in Object table
|
// Write an "uploading" marker in Object table
|
||||||
// This holds a reference to the object in the Version table
|
// This holds a reference to the object in the Version table
|
||||||
@@ -428,6 +429,7 @@ pub async fn handle_upload_part_copy(
|
|||||||
.get(&source_object_version.uuid, &EmptyKey)
|
.get(&source_object_version.uuid, &EmptyKey)
|
||||||
.await?
|
.await?
|
||||||
.ok_or(Error::NoSuchKey)?;
|
.ok_or(Error::NoSuchKey)?;
|
||||||
|
check_version_not_deleted(&source_version)?;
|
||||||
|
|
||||||
// We want to reuse blocks from the source version as much as possible.
|
// We want to reuse blocks from the source version as much as possible.
|
||||||
// However, we still need to get the data from these blocks
|
// However, we still need to get the data from these blocks
|
||||||
@@ -559,6 +561,7 @@ pub async fn handle_upload_part_copy(
|
|||||||
|
|
||||||
let mut current_offset = 0;
|
let mut current_offset = 0;
|
||||||
let mut next_block = defragmenter.next().await?;
|
let mut next_block = defragmenter.next().await?;
|
||||||
|
let mut blocks_to_dup = dest_version.clone();
|
||||||
|
|
||||||
// TODO this could be optimized similarly to read_and_put_blocks
|
// TODO this could be optimized similarly to read_and_put_blocks
|
||||||
// low priority because uploadpartcopy is rarely used
|
// low priority because uploadpartcopy is rarely used
|
||||||
@@ -588,8 +591,7 @@ pub async fn handle_upload_part_copy(
|
|||||||
.unwrap()?;
|
.unwrap()?;
|
||||||
checksummer = checksummer_updated;
|
checksummer = checksummer_updated;
|
||||||
|
|
||||||
dest_version.blocks.clear();
|
let (version_block_key, version_block) = (
|
||||||
dest_version.blocks.put(
|
|
||||||
VersionBlockKey {
|
VersionBlockKey {
|
||||||
part_number,
|
part_number,
|
||||||
offset: current_offset,
|
offset: current_offset,
|
||||||
@@ -601,37 +603,56 @@ pub async fn handle_upload_part_copy(
|
|||||||
);
|
);
|
||||||
current_offset += data_len;
|
current_offset += data_len;
|
||||||
|
|
||||||
let block_ref = BlockRef {
|
let next = if let Some(final_data) = data_to_upload {
|
||||||
block: final_hash,
|
dest_version.blocks.clear();
|
||||||
version: dest_version_id,
|
dest_version.blocks.put(version_block_key, version_block);
|
||||||
deleted: false.into(),
|
let block_ref = BlockRef {
|
||||||
|
block: final_hash,
|
||||||
|
version: dest_version_id,
|
||||||
|
deleted: false.into(),
|
||||||
|
};
|
||||||
|
let (_, _, _, next) = futures::try_join!(
|
||||||
|
// Thing 1: if the block is not exactly a block that existed before,
|
||||||
|
// we need to insert that data as a new block.
|
||||||
|
garage.block_manager.rpc_put_block(
|
||||||
|
final_hash,
|
||||||
|
final_data,
|
||||||
|
dest_encryption.is_encrypted(),
|
||||||
|
None
|
||||||
|
),
|
||||||
|
// Thing 2: we need to insert the block in the version
|
||||||
|
garage.version_table.insert(&dest_version),
|
||||||
|
// Thing 3: we need to add a block reference
|
||||||
|
garage.block_ref_table.insert(&block_ref),
|
||||||
|
// Thing 4: we need to read the next block
|
||||||
|
defragmenter.next(),
|
||||||
|
)?;
|
||||||
|
next
|
||||||
|
} else {
|
||||||
|
blocks_to_dup.blocks.put(version_block_key, version_block);
|
||||||
|
defragmenter.next().await?
|
||||||
};
|
};
|
||||||
|
|
||||||
let (_, _, _, next) = futures::try_join!(
|
|
||||||
// Thing 1: if the block is not exactly a block that existed before,
|
|
||||||
// we need to insert that data as a new block.
|
|
||||||
async {
|
|
||||||
if let Some(final_data) = data_to_upload {
|
|
||||||
garage
|
|
||||||
.block_manager
|
|
||||||
.rpc_put_block(final_hash, final_data, dest_encryption.is_encrypted(), None)
|
|
||||||
.await
|
|
||||||
} else {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
},
|
|
||||||
// Thing 2: we need to insert the block in the version
|
|
||||||
garage.version_table.insert(&dest_version),
|
|
||||||
// Thing 3: we need to add a block reference
|
|
||||||
garage.block_ref_table.insert(&block_ref),
|
|
||||||
// Thing 4: we need to read the next block
|
|
||||||
defragmenter.next(),
|
|
||||||
)?;
|
|
||||||
next_block = next;
|
next_block = next;
|
||||||
}
|
}
|
||||||
|
|
||||||
assert_eq!(current_offset, source_range.length);
|
assert_eq!(current_offset, source_range.length);
|
||||||
|
|
||||||
|
// Put the duplicated blocks into the version & block_refs tables
|
||||||
|
let block_refs_to_put = blocks_to_dup
|
||||||
|
.blocks
|
||||||
|
.items()
|
||||||
|
.iter()
|
||||||
|
.map(|b| BlockRef {
|
||||||
|
block: b.1.hash,
|
||||||
|
version: dest_version_id,
|
||||||
|
deleted: false.into(),
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
futures::try_join!(
|
||||||
|
garage.version_table.insert(&blocks_to_dup),
|
||||||
|
garage.block_ref_table.insert_many(&block_refs_to_put[..]),
|
||||||
|
)?;
|
||||||
|
|
||||||
let checksums = checksummer.finalize();
|
let checksums = checksummer.finalize();
|
||||||
let etag = dest_encryption.etag_from_md5(&checksums.md5);
|
let etag = dest_encryption.etag_from_md5(&checksums.md5);
|
||||||
let checksum = checksums.extract(dest_object_checksum_algorithm);
|
let checksum = checksums.extract(dest_object_checksum_algorithm);
|
||||||
|
|||||||
+30
-2
@@ -19,12 +19,13 @@ use garage_net::stream::ByteStream;
|
|||||||
use garage_rpc::rpc_helper::OrderTag;
|
use garage_rpc::rpc_helper::OrderTag;
|
||||||
use garage_table::EmptyKey;
|
use garage_table::EmptyKey;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::error::OkOrMessage;
|
use garage_util::error::{Error as UtilError, OkOrMessage};
|
||||||
|
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
use garage_model::s3::object_table::*;
|
use garage_model::s3::object_table::*;
|
||||||
use garage_model::s3::version_table::*;
|
use garage_model::s3::version_table::*;
|
||||||
|
|
||||||
|
use garage_api_common::common_error::CommonError;
|
||||||
use garage_api_common::helpers::*;
|
use garage_api_common::helpers::*;
|
||||||
use garage_api_common::signature::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE};
|
use garage_api_common::signature::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE};
|
||||||
|
|
||||||
@@ -215,6 +216,7 @@ pub async fn handle_head_without_ctx(
|
|||||||
.get(&object_version.uuid, &EmptyKey)
|
.get(&object_version.uuid, &EmptyKey)
|
||||||
.await?
|
.await?
|
||||||
.ok_or(Error::NoSuchKey)?;
|
.ok_or(Error::NoSuchKey)?;
|
||||||
|
check_version_not_deleted(&version)?;
|
||||||
|
|
||||||
let (part_offset, part_end) =
|
let (part_offset, part_end) =
|
||||||
calculate_part_bounds(&version, pn).ok_or(Error::InvalidPart)?;
|
calculate_part_bounds(&version, pn).ok_or(Error::InvalidPart)?;
|
||||||
@@ -365,6 +367,21 @@ pub async fn handle_get_without_ctx(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn check_version_not_deleted(version: &Version) -> Result<(), Error> {
|
||||||
|
if version.deleted.get() {
|
||||||
|
// the version was deleted between when the object_table was consulted
|
||||||
|
// and now, this could mean the object was deleted, or overriden.
|
||||||
|
// Rather than say the key doesn't exist, return a transient error
|
||||||
|
// to signal the client to try again.
|
||||||
|
return Err(CommonError::InternalError(UtilError::Message(
|
||||||
|
"conflict/inconsistency between object and version state, version is deleted"
|
||||||
|
.to_string(),
|
||||||
|
))
|
||||||
|
.into());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn handle_get_full(
|
async fn handle_get_full(
|
||||||
garage: Arc<Garage>,
|
garage: Arc<Garage>,
|
||||||
version: &ObjectVersion,
|
version: &ObjectVersion,
|
||||||
@@ -431,6 +448,7 @@ pub fn full_object_byte_stream(
|
|||||||
.ok_or_message("channel closed")?;
|
.ok_or_message("channel closed")?;
|
||||||
|
|
||||||
let version = version_fut.await.unwrap()?.ok_or(Error::NoSuchKey)?;
|
let version = version_fut.await.unwrap()?.ok_or(Error::NoSuchKey)?;
|
||||||
|
check_version_not_deleted(&version)?;
|
||||||
for (i, (_, vb)) in version.blocks.items().iter().enumerate().skip(1) {
|
for (i, (_, vb)) in version.blocks.items().iter().enumerate().skip(1) {
|
||||||
let stream_block_i = encryption
|
let stream_block_i = encryption
|
||||||
.get_block(&garage, &vb.hash, Some(order_stream.order(i as u64)))
|
.get_block(&garage, &vb.hash, Some(order_stream.order(i as u64)))
|
||||||
@@ -446,6 +464,14 @@ pub fn full_object_byte_stream(
|
|||||||
{
|
{
|
||||||
Ok(()) => (),
|
Ok(()) => (),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
// TODO i think this is a bad idea, we should log
|
||||||
|
// an error and stop there. If the error happens to
|
||||||
|
// be exactly the size of what hasn't been streamed
|
||||||
|
// yet, the client will see the request as a
|
||||||
|
// success
|
||||||
|
// instead truncating the output notify the client
|
||||||
|
// something happened with their download, so that
|
||||||
|
// they can retry it
|
||||||
let _ = tx.send(error_stream_item(e)).await;
|
let _ = tx.send(error_stream_item(e)).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -497,7 +523,7 @@ async fn handle_get_range(
|
|||||||
.get(&version.uuid, &EmptyKey)
|
.get(&version.uuid, &EmptyKey)
|
||||||
.await?
|
.await?
|
||||||
.ok_or(Error::NoSuchKey)?;
|
.ok_or(Error::NoSuchKey)?;
|
||||||
|
check_version_not_deleted(&version)?;
|
||||||
let body =
|
let body =
|
||||||
body_from_blocks_range(garage, encryption, version.blocks.items(), begin, end);
|
body_from_blocks_range(garage, encryption, version.blocks.items(), begin, end);
|
||||||
Ok(resp_builder.body(body)?)
|
Ok(resp_builder.body(body)?)
|
||||||
@@ -548,6 +574,8 @@ async fn handle_get_part(
|
|||||||
.await?
|
.await?
|
||||||
.ok_or(Error::NoSuchKey)?;
|
.ok_or(Error::NoSuchKey)?;
|
||||||
|
|
||||||
|
check_version_not_deleted(&version)?;
|
||||||
|
|
||||||
let (begin, end) =
|
let (begin, end) =
|
||||||
calculate_part_bounds(&version, part_number).ok_or(Error::InvalidPart)?;
|
calculate_part_bounds(&version, part_number).ok_or(Error::InvalidPart)?;
|
||||||
|
|
||||||
|
|||||||
@@ -13,6 +13,10 @@ pub fn xmlns_tag<S: Serializer>(_v: &(), s: S) -> Result<S::Ok, S::Error> {
|
|||||||
s.serialize_str("http://s3.amazonaws.com/doc/2006-03-01/")
|
s.serialize_str("http://s3.amazonaws.com/doc/2006-03-01/")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn xmlns_xsi_tag<S: Serializer>(_v: &(), s: S) -> Result<S::Ok, S::Error> {
|
||||||
|
s.serialize_str("http://www.w3.org/2001/XMLSchema-instance")
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||||
pub struct Value(#[serde(rename = "$value")] pub String);
|
pub struct Value(#[serde(rename = "$value")] pub String);
|
||||||
|
|
||||||
@@ -319,6 +323,42 @@ pub struct PostObject {
|
|||||||
pub etag: Value,
|
pub etag: Value,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||||
|
pub struct Grantee {
|
||||||
|
#[serde(rename = "xmlns:xsi", serialize_with = "xmlns_xsi_tag")]
|
||||||
|
pub xmlns_xsi: (),
|
||||||
|
#[serde(rename = "xsi:type")]
|
||||||
|
pub typ: String,
|
||||||
|
#[serde(rename = "DisplayName")]
|
||||||
|
pub display_name: Option<Value>,
|
||||||
|
#[serde(rename = "ID")]
|
||||||
|
pub id: Option<Value>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||||
|
pub struct Grant {
|
||||||
|
#[serde(rename = "Grantee")]
|
||||||
|
pub grantee: Grantee,
|
||||||
|
#[serde(rename = "Permission")]
|
||||||
|
pub permission: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||||
|
pub struct AccessControlList {
|
||||||
|
#[serde(rename = "Grant")]
|
||||||
|
pub entries: Vec<Grant>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||||
|
pub struct AccessControlPolicy {
|
||||||
|
#[serde(serialize_with = "xmlns_tag")]
|
||||||
|
pub xmlns: (),
|
||||||
|
#[serde(rename = "Owner")]
|
||||||
|
pub owner: Option<Owner>,
|
||||||
|
#[serde(rename = "AccessControlList")]
|
||||||
|
pub acl: AccessControlList,
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -427,6 +467,43 @@ mod tests {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn get_bucket_acl_result() -> Result<(), ApiError> {
|
||||||
|
let grant = Grant {
|
||||||
|
grantee: Grantee {
|
||||||
|
xmlns_xsi: (),
|
||||||
|
typ: "CanonicalUser".to_string(),
|
||||||
|
display_name: Some(Value("owner_name".to_string())),
|
||||||
|
id: Some(Value("qsdfjklm".to_string())),
|
||||||
|
},
|
||||||
|
permission: Value("FULL_CONTROL".to_string()),
|
||||||
|
};
|
||||||
|
|
||||||
|
let get_bucket_acl = AccessControlPolicy {
|
||||||
|
xmlns: (),
|
||||||
|
owner: None,
|
||||||
|
acl: AccessControlList {
|
||||||
|
entries: vec![grant],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
to_xml_with_header(&get_bucket_acl)?,
|
||||||
|
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
|
||||||
|
<AccessControlPolicy xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\
|
||||||
|
<AccessControlList>\
|
||||||
|
<Grant>\
|
||||||
|
<Grantee xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"CanonicalUser\">\
|
||||||
|
<DisplayName>owner_name</DisplayName>\
|
||||||
|
<ID>qsdfjklm</ID>\
|
||||||
|
</Grantee>\
|
||||||
|
<Permission>FULL_CONTROL</Permission>\
|
||||||
|
</Grant>\
|
||||||
|
</AccessControlList>\
|
||||||
|
</AccessControlPolicy>"
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn delete_result() -> Result<(), ApiError> {
|
fn delete_result() -> Result<(), ApiError> {
|
||||||
let delete_result = DeleteResult {
|
let delete_result = DeleteResult {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_block"
|
name = "garage_block"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|||||||
@@ -408,8 +408,8 @@ impl BlockManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Get number of items in the refcount table
|
/// Get number of items in the refcount table
|
||||||
pub fn rc_len(&self) -> Result<usize, Error> {
|
pub fn rc_approximate_len(&self) -> Result<usize, Error> {
|
||||||
Ok(self.rc.rc_table.len()?)
|
Ok(self.rc.rc_table.approximate_len()?)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Send command to start/stop/manager scrub worker
|
/// Send command to start/stop/manager scrub worker
|
||||||
@@ -427,7 +427,7 @@ impl BlockManager {
|
|||||||
|
|
||||||
/// List all resync errors
|
/// List all resync errors
|
||||||
pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> {
|
pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> {
|
||||||
let mut blocks = Vec::with_capacity(self.resync.errors.len()?);
|
let mut blocks = Vec::with_capacity(self.resync.errors.approximate_len()?);
|
||||||
for ent in self.resync.errors.iter()? {
|
for ent in self.resync.errors.iter()? {
|
||||||
let (hash, cnt) = ent?;
|
let (hash, cnt) = ent?;
|
||||||
let cnt = ErrorCounter::decode(&cnt);
|
let cnt = ErrorCounter::decode(&cnt);
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ impl BlockManagerMetrics {
|
|||||||
.init(),
|
.init(),
|
||||||
_rc_size: meter
|
_rc_size: meter
|
||||||
.u64_value_observer("block.rc_size", move |observer| {
|
.u64_value_observer("block.rc_size", move |observer| {
|
||||||
if let Ok(value) = rc_tree.len() {
|
if let Ok(value) = rc_tree.approximate_len() {
|
||||||
observer.observe(value as u64, &[])
|
observer.observe(value as u64, &[])
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -58,7 +58,7 @@ impl BlockManagerMetrics {
|
|||||||
.init(),
|
.init(),
|
||||||
_resync_queue_len: meter
|
_resync_queue_len: meter
|
||||||
.u64_value_observer("block.resync_queue_length", move |observer| {
|
.u64_value_observer("block.resync_queue_length", move |observer| {
|
||||||
if let Ok(value) = resync_queue.len() {
|
if let Ok(value) = resync_queue.approximate_len() {
|
||||||
observer.observe(value as u64, &[]);
|
observer.observe(value as u64, &[]);
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -68,7 +68,7 @@ impl BlockManagerMetrics {
|
|||||||
.init(),
|
.init(),
|
||||||
_resync_errored_blocks: meter
|
_resync_errored_blocks: meter
|
||||||
.u64_value_observer("block.resync_errored_blocks", move |observer| {
|
.u64_value_observer("block.resync_errored_blocks", move |observer| {
|
||||||
if let Ok(value) = resync_errors.len() {
|
if let Ok(value) = resync_errors.approximate_len() {
|
||||||
observer.observe(value as u64, &[]);
|
observer.observe(value as u64, &[]);
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
+8
-6
@@ -106,13 +106,13 @@ impl BlockResyncManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Get length of resync queue
|
/// Get length of resync queue
|
||||||
pub fn queue_len(&self) -> Result<usize, Error> {
|
pub fn queue_approximate_len(&self) -> Result<usize, Error> {
|
||||||
Ok(self.queue.len()?)
|
Ok(self.queue.approximate_len()?)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get number of blocks that have an error
|
/// Get number of blocks that have an error
|
||||||
pub fn errors_len(&self) -> Result<usize, Error> {
|
pub fn errors_approximate_len(&self) -> Result<usize, Error> {
|
||||||
Ok(self.errors.len()?)
|
Ok(self.errors.approximate_len()?)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Clear the error counter for a block and put it in queue immediately
|
/// Clear the error counter for a block and put it in queue immediately
|
||||||
@@ -548,9 +548,11 @@ impl Worker for ResyncWorker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
WorkerStatus {
|
WorkerStatus {
|
||||||
queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64),
|
queue_length: Some(self.manager.resync.queue_approximate_len().unwrap_or(0) as u64),
|
||||||
tranquility: Some(tranquility),
|
tranquility: Some(tranquility),
|
||||||
persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64),
|
persistent_errors: Some(
|
||||||
|
self.manager.resync.errors_approximate_len().unwrap_or(0) as u64
|
||||||
|
),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+6
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_db"
|
name = "garage_db"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
@@ -16,10 +16,14 @@ err-derive.workspace = true
|
|||||||
tracing.workspace = true
|
tracing.workspace = true
|
||||||
|
|
||||||
heed = { workspace = true, optional = true }
|
heed = { workspace = true, optional = true }
|
||||||
|
|
||||||
rusqlite = { workspace = true, optional = true, features = ["backup"] }
|
rusqlite = { workspace = true, optional = true, features = ["backup"] }
|
||||||
r2d2 = { workspace = true, optional = true }
|
r2d2 = { workspace = true, optional = true }
|
||||||
r2d2_sqlite = { workspace = true, optional = true }
|
r2d2_sqlite = { workspace = true, optional = true }
|
||||||
|
|
||||||
|
fjall = { workspace = true, optional = true }
|
||||||
|
parking_lot = { workspace = true, optional = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
mktemp.workspace = true
|
mktemp.workspace = true
|
||||||
|
|
||||||
@@ -27,4 +31,5 @@ mktemp.workspace = true
|
|||||||
default = [ "lmdb", "sqlite" ]
|
default = [ "lmdb", "sqlite" ]
|
||||||
bundled-libs = [ "rusqlite?/bundled" ]
|
bundled-libs = [ "rusqlite?/bundled" ]
|
||||||
lmdb = [ "heed" ]
|
lmdb = [ "heed" ]
|
||||||
|
fjall = [ "dep:fjall", "dep:parking_lot" ]
|
||||||
sqlite = [ "rusqlite", "r2d2", "r2d2_sqlite" ]
|
sqlite = [ "rusqlite", "r2d2", "r2d2_sqlite" ]
|
||||||
|
|||||||
@@ -0,0 +1,453 @@
|
|||||||
|
use core::ops::Bound;
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard};
|
||||||
|
|
||||||
|
use fjall::{
|
||||||
|
PartitionCreateOptions, PersistMode, TransactionalKeyspace, TransactionalPartitionHandle,
|
||||||
|
WriteTransaction,
|
||||||
|
};
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
open::{Engine, OpenOpt},
|
||||||
|
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
|
||||||
|
TxResult, TxValueIter, Value, ValueIter,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub use fjall;
|
||||||
|
|
||||||
|
// --
|
||||||
|
|
||||||
|
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
|
||||||
|
info!("Opening Fjall database at: {}", path.display());
|
||||||
|
if opt.fsync {
|
||||||
|
return Err(Error(
|
||||||
|
"metadata_fsync is not supported with the Fjall database engine".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut config = fjall::Config::new(path);
|
||||||
|
if let Some(block_cache_size) = opt.fjall_block_cache_size {
|
||||||
|
config = config.cache_size(block_cache_size as u64);
|
||||||
|
}
|
||||||
|
let keyspace = config.open_transactional()?;
|
||||||
|
Ok(FjallDb::init(keyspace))
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- err
|
||||||
|
|
||||||
|
impl From<fjall::Error> for Error {
|
||||||
|
fn from(e: fjall::Error) -> Error {
|
||||||
|
Error(format!("fjall: {}", e).into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<fjall::LsmError> for Error {
|
||||||
|
fn from(e: fjall::LsmError) -> Error {
|
||||||
|
Error(format!("fjall lsm_tree: {}", e).into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<fjall::Error> for TxOpError {
|
||||||
|
fn from(e: fjall::Error) -> TxOpError {
|
||||||
|
TxOpError(e.into())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- db
|
||||||
|
|
||||||
|
pub struct FjallDb {
|
||||||
|
keyspace: TransactionalKeyspace,
|
||||||
|
trees: RwLock<Vec<(String, TransactionalPartitionHandle)>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
type ByteRefRangeBound<'r> = (Bound<&'r [u8]>, Bound<&'r [u8]>);
|
||||||
|
|
||||||
|
impl FjallDb {
|
||||||
|
pub fn init(keyspace: TransactionalKeyspace) -> Db {
|
||||||
|
let s = Self {
|
||||||
|
keyspace,
|
||||||
|
trees: RwLock::new(Vec::new()),
|
||||||
|
};
|
||||||
|
Db(Arc::new(s))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_tree(
|
||||||
|
&self,
|
||||||
|
i: usize,
|
||||||
|
) -> Result<MappedRwLockReadGuard<'_, TransactionalPartitionHandle>> {
|
||||||
|
RwLockReadGuard::try_map(self.trees.read(), |trees: &Vec<_>| {
|
||||||
|
trees.get(i).map(|tup| &tup.1)
|
||||||
|
})
|
||||||
|
.map_err(|_| Error("invalid tree id".into()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl IDb for FjallDb {
|
||||||
|
fn engine(&self) -> String {
|
||||||
|
"Fjall (EXPERIMENTAL!)".into()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn open_tree(&self, name: &str) -> Result<usize> {
|
||||||
|
let mut trees = self.trees.write();
|
||||||
|
let safe_name = encode_name(name)?;
|
||||||
|
if let Some(i) = trees.iter().position(|(name, _)| *name == safe_name) {
|
||||||
|
Ok(i)
|
||||||
|
} else {
|
||||||
|
let tree = self
|
||||||
|
.keyspace
|
||||||
|
.open_partition(&safe_name, PartitionCreateOptions::default())?;
|
||||||
|
let i = trees.len();
|
||||||
|
trees.push((safe_name, tree));
|
||||||
|
Ok(i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn list_trees(&self) -> Result<Vec<String>> {
|
||||||
|
Ok(self
|
||||||
|
.keyspace
|
||||||
|
.list_partitions()
|
||||||
|
.iter()
|
||||||
|
.map(|n| decode_name(&n))
|
||||||
|
.collect::<Result<Vec<_>>>()?)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
|
||||||
|
std::fs::create_dir_all(base_path)?;
|
||||||
|
let path = Engine::Fjall.db_path(base_path);
|
||||||
|
|
||||||
|
let source_state = self.keyspace.read_tx();
|
||||||
|
let copy_keyspace = fjall::Config::new(path).open()?;
|
||||||
|
|
||||||
|
for partition_name in self.keyspace.list_partitions() {
|
||||||
|
let source_partition = self
|
||||||
|
.keyspace
|
||||||
|
.open_partition(&partition_name, PartitionCreateOptions::default())?;
|
||||||
|
let copy_partition =
|
||||||
|
copy_keyspace.open_partition(&partition_name, PartitionCreateOptions::default())?;
|
||||||
|
|
||||||
|
for entry in source_state.iter(&source_partition) {
|
||||||
|
let (key, value) = entry?;
|
||||||
|
copy_partition.insert(key, value)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
copy_keyspace.persist(PersistMode::SyncAll)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
|
|
||||||
|
fn get(&self, tree_idx: usize, key: &[u8]) -> Result<Option<Value>> {
|
||||||
|
let tree = self.get_tree(tree_idx)?;
|
||||||
|
let tx = self.keyspace.read_tx();
|
||||||
|
let val = tx.get(&tree, key)?;
|
||||||
|
match val {
|
||||||
|
None => Ok(None),
|
||||||
|
Some(v) => Ok(Some(v.to_vec())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn approximate_len(&self, tree_idx: usize) -> Result<usize> {
|
||||||
|
let tree = self.get_tree(tree_idx)?;
|
||||||
|
Ok(tree.approximate_len())
|
||||||
|
}
|
||||||
|
fn is_empty(&self, tree_idx: usize) -> Result<bool> {
|
||||||
|
let tree = self.get_tree(tree_idx)?;
|
||||||
|
let tx = self.keyspace.read_tx();
|
||||||
|
Ok(tx.is_empty(&tree)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert(&self, tree_idx: usize, key: &[u8], value: &[u8]) -> Result<()> {
|
||||||
|
let tree = self.get_tree(tree_idx)?;
|
||||||
|
let mut tx = self.keyspace.write_tx();
|
||||||
|
tx.insert(&tree, key, value);
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove(&self, tree_idx: usize, key: &[u8]) -> Result<()> {
|
||||||
|
let tree = self.get_tree(tree_idx)?;
|
||||||
|
let mut tx = self.keyspace.write_tx();
|
||||||
|
tx.remove(&tree, key);
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn clear(&self, tree_idx: usize) -> Result<()> {
|
||||||
|
let mut trees = self.trees.write();
|
||||||
|
|
||||||
|
if tree_idx >= trees.len() {
|
||||||
|
return Err(Error("invalid tree id".into()));
|
||||||
|
}
|
||||||
|
let (name, tree) = trees.remove(tree_idx);
|
||||||
|
|
||||||
|
self.keyspace.delete_partition(tree)?;
|
||||||
|
let tree = self
|
||||||
|
.keyspace
|
||||||
|
.open_partition(&name, PartitionCreateOptions::default())?;
|
||||||
|
trees.insert(tree_idx, (name, tree));
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn iter(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
|
||||||
|
let tree = self.get_tree(tree_idx)?;
|
||||||
|
let tx = self.keyspace.read_tx();
|
||||||
|
Ok(Box::new(tx.iter(&tree).map(iterator_remap)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn iter_rev(&self, tree_idx: usize) -> Result<ValueIter<'_>> {
|
||||||
|
let tree = self.get_tree(tree_idx)?;
|
||||||
|
let tx = self.keyspace.read_tx();
|
||||||
|
Ok(Box::new(tx.iter(&tree).rev().map(iterator_remap)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn range<'r>(
|
||||||
|
&self,
|
||||||
|
tree_idx: usize,
|
||||||
|
low: Bound<&'r [u8]>,
|
||||||
|
high: Bound<&'r [u8]>,
|
||||||
|
) -> Result<ValueIter<'_>> {
|
||||||
|
let tree = self.get_tree(tree_idx)?;
|
||||||
|
let tx = self.keyspace.read_tx();
|
||||||
|
Ok(Box::new(
|
||||||
|
tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high))
|
||||||
|
.map(iterator_remap),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
fn range_rev<'r>(
|
||||||
|
&self,
|
||||||
|
tree_idx: usize,
|
||||||
|
low: Bound<&'r [u8]>,
|
||||||
|
high: Bound<&'r [u8]>,
|
||||||
|
) -> Result<ValueIter<'_>> {
|
||||||
|
let tree = self.get_tree(tree_idx)?;
|
||||||
|
let tx = self.keyspace.read_tx();
|
||||||
|
Ok(Box::new(
|
||||||
|
tx.range::<&'r [u8], ByteRefRangeBound>(&tree, (low, high))
|
||||||
|
.rev()
|
||||||
|
.map(iterator_remap),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
|
|
||||||
|
fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()> {
|
||||||
|
let trees = self.trees.read();
|
||||||
|
let mut tx = FjallTx {
|
||||||
|
trees: &trees[..],
|
||||||
|
tx: self.keyspace.write_tx(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let res = f.try_on(&mut tx);
|
||||||
|
match res {
|
||||||
|
TxFnResult::Ok(on_commit) => {
|
||||||
|
tx.tx.commit().map_err(Error::from).map_err(TxError::Db)?;
|
||||||
|
Ok(on_commit)
|
||||||
|
}
|
||||||
|
TxFnResult::Abort => {
|
||||||
|
tx.tx.rollback();
|
||||||
|
Err(TxError::Abort(()))
|
||||||
|
}
|
||||||
|
TxFnResult::DbErr => {
|
||||||
|
tx.tx.rollback();
|
||||||
|
Err(TxError::Db(Error(
|
||||||
|
"(this message will be discarded)".into(),
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
|
|
||||||
|
struct FjallTx<'a> {
|
||||||
|
trees: &'a [(String, TransactionalPartitionHandle)],
|
||||||
|
tx: WriteTransaction<'a>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> FjallTx<'a> {
|
||||||
|
fn get_tree(&self, i: usize) -> TxOpResult<&TransactionalPartitionHandle> {
|
||||||
|
self.trees.get(i).map(|tup| &tup.1).ok_or_else(|| {
|
||||||
|
TxOpError(Error(
|
||||||
|
"invalid tree id (it might have been openned after the transaction started)".into(),
|
||||||
|
))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> ITx for FjallTx<'a> {
|
||||||
|
fn get(&self, tree_idx: usize, key: &[u8]) -> TxOpResult<Option<Value>> {
|
||||||
|
let tree = self.get_tree(tree_idx)?;
|
||||||
|
match self.tx.get(tree, key)? {
|
||||||
|
Some(v) => Ok(Some(v.to_vec())),
|
||||||
|
None => Ok(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fn len(&self, tree_idx: usize) -> TxOpResult<usize> {
|
||||||
|
let tree = self.get_tree(tree_idx)?;
|
||||||
|
Ok(self.tx.len(tree)? as usize)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn insert(&mut self, tree_idx: usize, key: &[u8], value: &[u8]) -> TxOpResult<()> {
|
||||||
|
let tree = self.get_tree(tree_idx)?.clone();
|
||||||
|
self.tx.insert(&tree, key, value);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
fn remove(&mut self, tree_idx: usize, key: &[u8]) -> TxOpResult<()> {
|
||||||
|
let tree = self.get_tree(tree_idx)?.clone();
|
||||||
|
self.tx.remove(&tree, key);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
fn clear(&mut self, _tree_idx: usize) -> TxOpResult<()> {
|
||||||
|
unimplemented!("LSM tree clearing in cross-partition transaction is not supported")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn iter(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
|
||||||
|
let tree = self.get_tree(tree_idx)?.clone();
|
||||||
|
Ok(Box::new(self.tx.iter(&tree).map(iterator_remap_tx)))
|
||||||
|
}
|
||||||
|
fn iter_rev(&self, tree_idx: usize) -> TxOpResult<TxValueIter<'_>> {
|
||||||
|
let tree = self.get_tree(tree_idx)?.clone();
|
||||||
|
Ok(Box::new(self.tx.iter(&tree).rev().map(iterator_remap_tx)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn range<'r>(
|
||||||
|
&self,
|
||||||
|
tree_idx: usize,
|
||||||
|
low: Bound<&'r [u8]>,
|
||||||
|
high: Bound<&'r [u8]>,
|
||||||
|
) -> TxOpResult<TxValueIter<'_>> {
|
||||||
|
let tree = self.get_tree(tree_idx)?;
|
||||||
|
let low = clone_bound(low);
|
||||||
|
let high = clone_bound(high);
|
||||||
|
Ok(Box::new(
|
||||||
|
self.tx
|
||||||
|
.range::<Vec<u8>, ByteVecRangeBounds>(&tree, (low, high))
|
||||||
|
.map(iterator_remap_tx),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
fn range_rev<'r>(
|
||||||
|
&self,
|
||||||
|
tree_idx: usize,
|
||||||
|
low: Bound<&'r [u8]>,
|
||||||
|
high: Bound<&'r [u8]>,
|
||||||
|
) -> TxOpResult<TxValueIter<'_>> {
|
||||||
|
let tree = self.get_tree(tree_idx)?;
|
||||||
|
let low = clone_bound(low);
|
||||||
|
let high = clone_bound(high);
|
||||||
|
Ok(Box::new(
|
||||||
|
self.tx
|
||||||
|
.range::<Vec<u8>, ByteVecRangeBounds>(&tree, (low, high))
|
||||||
|
.rev()
|
||||||
|
.map(iterator_remap_tx),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- maps fjall's (k, v) to ours
|
||||||
|
|
||||||
|
fn iterator_remap(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> Result<(Value, Value)> {
|
||||||
|
r.map(|(k, v)| (k.to_vec(), v.to_vec()))
|
||||||
|
.map_err(|e| e.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn iterator_remap_tx(r: fjall::Result<(fjall::Slice, fjall::Slice)>) -> TxOpResult<(Value, Value)> {
|
||||||
|
r.map(|(k, v)| (k.to_vec(), v.to_vec()))
|
||||||
|
.map_err(|e| e.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- utils to deal with Garage's tightness on Bound lifetimes
|
||||||
|
|
||||||
|
type ByteVecBound = Bound<Vec<u8>>;
|
||||||
|
type ByteVecRangeBounds = (ByteVecBound, ByteVecBound);
|
||||||
|
|
||||||
|
fn clone_bound(bound: Bound<&[u8]>) -> ByteVecBound {
|
||||||
|
let value = match bound {
|
||||||
|
Bound::Excluded(v) | Bound::Included(v) => v.to_vec(),
|
||||||
|
Bound::Unbounded => vec![],
|
||||||
|
};
|
||||||
|
|
||||||
|
match bound {
|
||||||
|
Bound::Included(_) => Bound::Included(value),
|
||||||
|
Bound::Excluded(_) => Bound::Excluded(value),
|
||||||
|
Bound::Unbounded => Bound::Unbounded,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// -- utils to encode table names --
|
||||||
|
|
||||||
|
fn encode_name(s: &str) -> Result<String> {
|
||||||
|
let base = 'A' as u32;
|
||||||
|
|
||||||
|
let mut ret = String::with_capacity(s.len() + 10);
|
||||||
|
for c in s.chars() {
|
||||||
|
if c.is_alphanumeric() || c == '_' || c == '-' || c == '#' {
|
||||||
|
ret.push(c);
|
||||||
|
} else if c <= u8::MAX as char {
|
||||||
|
ret.push('$');
|
||||||
|
let c_hi = c as u32 / 16;
|
||||||
|
let c_lo = c as u32 % 16;
|
||||||
|
ret.push(char::from_u32(base + c_hi).unwrap());
|
||||||
|
ret.push(char::from_u32(base + c_lo).unwrap());
|
||||||
|
} else {
|
||||||
|
return Err(Error(
|
||||||
|
format!("table name {} could not be safely encoded", s).into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(ret)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_name(s: &str) -> Result<String> {
|
||||||
|
use std::convert::TryFrom;
|
||||||
|
|
||||||
|
let errfn = || Error(format!("encoded table name {} is invalid", s).into());
|
||||||
|
let c_map = |c: char| {
|
||||||
|
let c = c as u32;
|
||||||
|
let base = 'A' as u32;
|
||||||
|
if (base..base + 16).contains(&c) {
|
||||||
|
Some(c - base)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut ret = String::with_capacity(s.len());
|
||||||
|
let mut it = s.chars();
|
||||||
|
while let Some(c) = it.next() {
|
||||||
|
if c == '$' {
|
||||||
|
let c_hi = it.next().and_then(c_map).ok_or_else(errfn)?;
|
||||||
|
let c_lo = it.next().and_then(c_map).ok_or_else(errfn)?;
|
||||||
|
let c_dec = char::try_from(c_hi * 16 + c_lo).map_err(|_| errfn())?;
|
||||||
|
ret.push(c_dec);
|
||||||
|
} else {
|
||||||
|
ret.push(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(ret)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_encdec_name() {
|
||||||
|
for name in [
|
||||||
|
"testname",
|
||||||
|
"test_name",
|
||||||
|
"test name",
|
||||||
|
"test$name",
|
||||||
|
"test:name@help.me$get/this**right",
|
||||||
|
] {
|
||||||
|
let encname = encode_name(name).unwrap();
|
||||||
|
assert!(!encname.contains(' '));
|
||||||
|
assert!(!encname.contains('.'));
|
||||||
|
assert!(!encname.contains('*'));
|
||||||
|
assert_eq!(*name, decode_name(&encname).unwrap());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+11
-4
@@ -1,6 +1,8 @@
|
|||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate tracing;
|
extern crate tracing;
|
||||||
|
|
||||||
|
#[cfg(feature = "fjall")]
|
||||||
|
pub mod fjall_adapter;
|
||||||
#[cfg(feature = "lmdb")]
|
#[cfg(feature = "lmdb")]
|
||||||
pub mod lmdb_adapter;
|
pub mod lmdb_adapter;
|
||||||
#[cfg(feature = "sqlite")]
|
#[cfg(feature = "sqlite")]
|
||||||
@@ -152,7 +154,7 @@ impl Db {
|
|||||||
let tree_names = other.list_trees()?;
|
let tree_names = other.list_trees()?;
|
||||||
for name in tree_names {
|
for name in tree_names {
|
||||||
let tree = self.open_tree(&name)?;
|
let tree = self.open_tree(&name)?;
|
||||||
if tree.len()? > 0 {
|
if !tree.is_empty()? {
|
||||||
return Err(Error(format!("tree {} already contains data", name).into()));
|
return Err(Error(format!("tree {} already contains data", name).into()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -194,8 +196,12 @@ impl Tree {
|
|||||||
self.0.get(self.1, key.as_ref())
|
self.0.get(self.1, key.as_ref())
|
||||||
}
|
}
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn len(&self) -> Result<usize> {
|
pub fn approximate_len(&self) -> Result<usize> {
|
||||||
self.0.len(self.1)
|
self.0.approximate_len(self.1)
|
||||||
|
}
|
||||||
|
#[inline]
|
||||||
|
pub fn is_empty(&self) -> Result<bool> {
|
||||||
|
self.0.is_empty(self.1)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
@@ -333,7 +339,8 @@ pub(crate) trait IDb: Send + Sync {
|
|||||||
fn snapshot(&self, path: &PathBuf) -> Result<()>;
|
fn snapshot(&self, path: &PathBuf) -> Result<()>;
|
||||||
|
|
||||||
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
|
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
|
||||||
fn len(&self, tree: usize) -> Result<usize>;
|
fn approximate_len(&self, tree: usize) -> Result<usize>;
|
||||||
|
fn is_empty(&self, tree: usize) -> Result<bool>;
|
||||||
|
|
||||||
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>;
|
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()>;
|
||||||
fn remove(&self, tree: usize, key: &[u8]) -> Result<()>;
|
fn remove(&self, tree: usize, key: &[u8]) -> Result<()>;
|
||||||
|
|||||||
+94
-29
@@ -1,8 +1,8 @@
|
|||||||
use core::ops::Bound;
|
use core::ops::Bound;
|
||||||
use core::ptr::NonNull;
|
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::convert::TryInto;
|
use std::convert::TryInto;
|
||||||
|
use std::marker::PhantomPinned;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
use std::sync::{Arc, RwLock};
|
use std::sync::{Arc, RwLock};
|
||||||
@@ -11,12 +11,55 @@ use heed::types::ByteSlice;
|
|||||||
use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database};
|
use heed::{BytesDecode, Env, RoTxn, RwTxn, UntypedDatabase as Database};
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
|
open::{Engine, OpenOpt},
|
||||||
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
|
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
|
||||||
TxResult, TxValueIter, Value, ValueIter,
|
TxResult, TxValueIter, Value, ValueIter,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub use heed;
|
pub use heed;
|
||||||
|
|
||||||
|
// ---- top-level open function
|
||||||
|
|
||||||
|
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
|
||||||
|
info!("Opening LMDB database at: {}", path.display());
|
||||||
|
if let Err(e) = std::fs::create_dir_all(&path) {
|
||||||
|
return Err(Error(
|
||||||
|
format!("Unable to create LMDB data directory: {}", e).into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let map_size = match opt.lmdb_map_size {
|
||||||
|
None => recommended_map_size(),
|
||||||
|
Some(v) => v - (v % 4096),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut env_builder = heed::EnvOpenOptions::new();
|
||||||
|
env_builder.max_dbs(100);
|
||||||
|
env_builder.map_size(map_size);
|
||||||
|
env_builder.max_readers(2048);
|
||||||
|
unsafe {
|
||||||
|
env_builder.flag(heed::flags::Flags::MdbNoRdAhead);
|
||||||
|
env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
|
||||||
|
if !opt.fsync {
|
||||||
|
env_builder.flag(heed::flags::Flags::MdbNoSync);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match env_builder.open(&path) {
|
||||||
|
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
|
||||||
|
return Err(Error(
|
||||||
|
"OutOfMemory error while trying to open LMDB database. This can happen \
|
||||||
|
if your operating system is not allowing you to use sufficient virtual \
|
||||||
|
memory address space. Please check that no limit is set (ulimit -v). \
|
||||||
|
You may also try to set a smaller `lmdb_map_size` configuration parameter. \
|
||||||
|
On 32-bit machines, you should probably switch to another database engine."
|
||||||
|
.into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
|
||||||
|
Ok(db) => Ok(LmdbDb::init(db)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// -- err
|
// -- err
|
||||||
|
|
||||||
impl From<heed::Error> for Error {
|
impl From<heed::Error> for Error {
|
||||||
@@ -104,10 +147,9 @@ impl IDb for LmdbDb {
|
|||||||
Ok(ret2)
|
Ok(ret2)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn snapshot(&self, to: &PathBuf) -> Result<()> {
|
fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
|
||||||
std::fs::create_dir_all(to)?;
|
std::fs::create_dir_all(base_path)?;
|
||||||
let mut path = to.clone();
|
let path = Engine::Lmdb.db_path(base_path);
|
||||||
path.push("data.mdb");
|
|
||||||
self.db
|
self.db
|
||||||
.copy_to_path(path, heed::CompactionOption::Enabled)?;
|
.copy_to_path(path, heed::CompactionOption::Enabled)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -126,11 +168,16 @@ impl IDb for LmdbDb {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn len(&self, tree: usize) -> Result<usize> {
|
fn approximate_len(&self, tree: usize) -> Result<usize> {
|
||||||
let tree = self.get_tree(tree)?;
|
let tree = self.get_tree(tree)?;
|
||||||
let tx = self.db.read_txn()?;
|
let tx = self.db.read_txn()?;
|
||||||
Ok(tree.len(&tx)?.try_into().unwrap())
|
Ok(tree.len(&tx)?.try_into().unwrap())
|
||||||
}
|
}
|
||||||
|
fn is_empty(&self, tree: usize) -> Result<bool> {
|
||||||
|
let tree = self.get_tree(tree)?;
|
||||||
|
let tx = self.db.read_txn()?;
|
||||||
|
Ok(tree.is_empty(&tx)?)
|
||||||
|
}
|
||||||
|
|
||||||
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
|
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
|
||||||
let tree = self.get_tree(tree)?;
|
let tree = self.get_tree(tree)?;
|
||||||
@@ -159,13 +206,15 @@ impl IDb for LmdbDb {
|
|||||||
fn iter(&self, tree: usize) -> Result<ValueIter<'_>> {
|
fn iter(&self, tree: usize) -> Result<ValueIter<'_>> {
|
||||||
let tree = self.get_tree(tree)?;
|
let tree = self.get_tree(tree)?;
|
||||||
let tx = self.db.read_txn()?;
|
let tx = self.db.read_txn()?;
|
||||||
TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?))
|
// Safety: the cloture does not store its argument anywhere,
|
||||||
|
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.iter(tx)?)) }
|
||||||
}
|
}
|
||||||
|
|
||||||
fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> {
|
fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> {
|
||||||
let tree = self.get_tree(tree)?;
|
let tree = self.get_tree(tree)?;
|
||||||
let tx = self.db.read_txn()?;
|
let tx = self.db.read_txn()?;
|
||||||
TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?))
|
// Safety: the cloture does not store its argument anywhere,
|
||||||
|
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_iter(tx)?)) }
|
||||||
}
|
}
|
||||||
|
|
||||||
fn range<'r>(
|
fn range<'r>(
|
||||||
@@ -176,7 +225,8 @@ impl IDb for LmdbDb {
|
|||||||
) -> Result<ValueIter<'_>> {
|
) -> Result<ValueIter<'_>> {
|
||||||
let tree = self.get_tree(tree)?;
|
let tree = self.get_tree(tree)?;
|
||||||
let tx = self.db.read_txn()?;
|
let tx = self.db.read_txn()?;
|
||||||
TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?))
|
// Safety: the cloture does not store its argument anywhere,
|
||||||
|
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.range(tx, &(low, high))?)) }
|
||||||
}
|
}
|
||||||
fn range_rev<'r>(
|
fn range_rev<'r>(
|
||||||
&self,
|
&self,
|
||||||
@@ -186,7 +236,8 @@ impl IDb for LmdbDb {
|
|||||||
) -> Result<ValueIter<'_>> {
|
) -> Result<ValueIter<'_>> {
|
||||||
let tree = self.get_tree(tree)?;
|
let tree = self.get_tree(tree)?;
|
||||||
let tx = self.db.read_txn()?;
|
let tx = self.db.read_txn()?;
|
||||||
TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?))
|
// Safety: the cloture does not store its argument anywhere,
|
||||||
|
unsafe { TxAndIterator::make(tx, |tx| Ok(tree.rev_range(tx, &(low, high))?)) }
|
||||||
}
|
}
|
||||||
|
|
||||||
// ----
|
// ----
|
||||||
@@ -316,28 +367,41 @@ where
|
|||||||
{
|
{
|
||||||
tx: RoTxn<'a>,
|
tx: RoTxn<'a>,
|
||||||
iter: Option<I>,
|
iter: Option<I>,
|
||||||
|
_pin: PhantomPinned,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a, I> TxAndIterator<'a, I>
|
impl<'a, I> TxAndIterator<'a, I>
|
||||||
where
|
where
|
||||||
I: Iterator<Item = IteratorItem<'a>> + 'a,
|
I: Iterator<Item = IteratorItem<'a>> + 'a,
|
||||||
{
|
{
|
||||||
fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>>
|
fn iter(self: Pin<&mut Self>) -> &mut Option<I> {
|
||||||
|
// Safety: iter is not structural
|
||||||
|
unsafe { &mut self.get_unchecked_mut().iter }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Safety: iterfun must not store its argument anywhere but in its result.
|
||||||
|
unsafe fn make<F>(tx: RoTxn<'a>, iterfun: F) -> Result<ValueIter<'a>>
|
||||||
where
|
where
|
||||||
F: FnOnce(&'a RoTxn<'a>) -> Result<I>,
|
F: FnOnce(&'a RoTxn<'a>) -> Result<I>,
|
||||||
{
|
{
|
||||||
let res = TxAndIterator { tx, iter: None };
|
let res = TxAndIterator {
|
||||||
|
tx,
|
||||||
|
iter: None,
|
||||||
|
_pin: PhantomPinned,
|
||||||
|
};
|
||||||
let mut boxed = Box::pin(res);
|
let mut boxed = Box::pin(res);
|
||||||
|
|
||||||
// This unsafe allows us to bypass lifetime checks
|
let tx_lifetime_overextended: &'a RoTxn<'a> = {
|
||||||
let tx = unsafe { NonNull::from(&boxed.tx).as_ref() };
|
let tx = &boxed.tx;
|
||||||
let iter = iterfun(tx)?;
|
// Safety: Artificially extending the lifetime because
|
||||||
|
// this reference will only be stored and accessed from the
|
||||||
|
// returned ValueIter which guarantees that it is destroyed
|
||||||
|
// before the tx it is pointing to.
|
||||||
|
unsafe { &*&raw const *tx }
|
||||||
|
};
|
||||||
|
let iter = iterfun(&tx_lifetime_overextended)?;
|
||||||
|
|
||||||
let mut_ref = Pin::as_mut(&mut boxed);
|
*boxed.as_mut().iter() = Some(iter);
|
||||||
// This unsafe allows us to write in a field of the pinned struct
|
|
||||||
unsafe {
|
|
||||||
Pin::get_unchecked_mut(mut_ref).iter = Some(iter);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(Box::new(TxAndIteratorPin(boxed)))
|
Ok(Box::new(TxAndIteratorPin(boxed)))
|
||||||
}
|
}
|
||||||
@@ -348,8 +412,10 @@ where
|
|||||||
I: Iterator<Item = IteratorItem<'a>> + 'a,
|
I: Iterator<Item = IteratorItem<'a>> + 'a,
|
||||||
{
|
{
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
// ensure the iterator is dropped before the RoTxn it references
|
// Safety: `new_unchecked` is okay because we know this value is never
|
||||||
drop(self.iter.take());
|
// used again after being dropped.
|
||||||
|
let this = unsafe { Pin::new_unchecked(self) };
|
||||||
|
drop(this.iter().take());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -365,13 +431,12 @@ where
|
|||||||
|
|
||||||
fn next(&mut self) -> Option<Self::Item> {
|
fn next(&mut self) -> Option<Self::Item> {
|
||||||
let mut_ref = Pin::as_mut(&mut self.0);
|
let mut_ref = Pin::as_mut(&mut self.0);
|
||||||
// This unsafe allows us to mutably access the iterator field
|
let next = mut_ref.iter().as_mut()?.next()?;
|
||||||
let next = unsafe { Pin::get_unchecked_mut(mut_ref).iter.as_mut()?.next() };
|
let res = match next {
|
||||||
match next {
|
Err(e) => Err(e.into()),
|
||||||
None => None,
|
Ok((k, v)) => Ok((k.to_vec(), v.to_vec())),
|
||||||
Some(Err(e)) => Some(Err(e.into())),
|
};
|
||||||
Some(Ok((k, v))) => Some(Ok((k.to_vec(), v.to_vec()))),
|
Some(res)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+28
-44
@@ -11,6 +11,7 @@ use crate::{Db, Error, Result};
|
|||||||
pub enum Engine {
|
pub enum Engine {
|
||||||
Lmdb,
|
Lmdb,
|
||||||
Sqlite,
|
Sqlite,
|
||||||
|
Fjall,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Engine {
|
impl Engine {
|
||||||
@@ -19,8 +20,26 @@ impl Engine {
|
|||||||
match self {
|
match self {
|
||||||
Self::Lmdb => "lmdb",
|
Self::Lmdb => "lmdb",
|
||||||
Self::Sqlite => "sqlite",
|
Self::Sqlite => "sqlite",
|
||||||
|
Self::Fjall => "fjall",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Return engine-specific DB path from base path
|
||||||
|
pub fn db_path(&self, base_path: &PathBuf) -> PathBuf {
|
||||||
|
let mut ret = base_path.clone();
|
||||||
|
match self {
|
||||||
|
Self::Lmdb => {
|
||||||
|
ret.push("db.lmdb");
|
||||||
|
}
|
||||||
|
Self::Sqlite => {
|
||||||
|
ret.push("db.sqlite");
|
||||||
|
}
|
||||||
|
Self::Fjall => {
|
||||||
|
ret.push("db.fjall");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ret
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Display for Engine {
|
impl std::fmt::Display for Engine {
|
||||||
@@ -36,10 +55,11 @@ impl std::str::FromStr for Engine {
|
|||||||
match text {
|
match text {
|
||||||
"lmdb" | "heed" => Ok(Self::Lmdb),
|
"lmdb" | "heed" => Ok(Self::Lmdb),
|
||||||
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
|
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
|
||||||
|
"fjall" => Ok(Self::Fjall),
|
||||||
"sled" => Err(Error("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into())),
|
"sled" => Err(Error("Sled is no longer supported as a database engine. Converting your old metadata db can be done using an older Garage binary (e.g. v0.9.4).".into())),
|
||||||
kind => Err(Error(
|
kind => Err(Error(
|
||||||
format!(
|
format!(
|
||||||
"Invalid DB engine: {} (options are: lmdb, sqlite)",
|
"Invalid DB engine: {} (options are: lmdb, sqlite, fjall)",
|
||||||
kind
|
kind
|
||||||
)
|
)
|
||||||
.into(),
|
.into(),
|
||||||
@@ -51,6 +71,7 @@ impl std::str::FromStr for Engine {
|
|||||||
pub struct OpenOpt {
|
pub struct OpenOpt {
|
||||||
pub fsync: bool,
|
pub fsync: bool,
|
||||||
pub lmdb_map_size: Option<usize>,
|
pub lmdb_map_size: Option<usize>,
|
||||||
|
pub fjall_block_cache_size: Option<usize>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for OpenOpt {
|
impl Default for OpenOpt {
|
||||||
@@ -58,6 +79,7 @@ impl Default for OpenOpt {
|
|||||||
Self {
|
Self {
|
||||||
fsync: false,
|
fsync: false,
|
||||||
lmdb_map_size: None,
|
lmdb_map_size: None,
|
||||||
|
fjall_block_cache_size: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -66,53 +88,15 @@ pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
|
|||||||
match engine {
|
match engine {
|
||||||
// ---- Sqlite DB ----
|
// ---- Sqlite DB ----
|
||||||
#[cfg(feature = "sqlite")]
|
#[cfg(feature = "sqlite")]
|
||||||
Engine::Sqlite => {
|
Engine::Sqlite => crate::sqlite_adapter::open_db(path, opt),
|
||||||
info!("Opening Sqlite database at: {}", path.display());
|
|
||||||
let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
|
|
||||||
Ok(crate::sqlite_adapter::SqliteDb::new(manager, opt.fsync)?)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- LMDB DB ----
|
// ---- LMDB DB ----
|
||||||
#[cfg(feature = "lmdb")]
|
#[cfg(feature = "lmdb")]
|
||||||
Engine::Lmdb => {
|
Engine::Lmdb => crate::lmdb_adapter::open_db(path, opt),
|
||||||
info!("Opening LMDB database at: {}", path.display());
|
|
||||||
if let Err(e) = std::fs::create_dir_all(&path) {
|
|
||||||
return Err(Error(
|
|
||||||
format!("Unable to create LMDB data directory: {}", e).into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
let map_size = match opt.lmdb_map_size {
|
// ---- Fjall DB ----
|
||||||
None => crate::lmdb_adapter::recommended_map_size(),
|
#[cfg(feature = "fjall")]
|
||||||
Some(v) => v - (v % 4096),
|
Engine::Fjall => crate::fjall_adapter::open_db(path, opt),
|
||||||
};
|
|
||||||
|
|
||||||
let mut env_builder = heed::EnvOpenOptions::new();
|
|
||||||
env_builder.max_dbs(100);
|
|
||||||
env_builder.map_size(map_size);
|
|
||||||
env_builder.max_readers(2048);
|
|
||||||
unsafe {
|
|
||||||
env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoRdAhead);
|
|
||||||
env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
|
|
||||||
if !opt.fsync {
|
|
||||||
env_builder.flag(heed::flags::Flags::MdbNoSync);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
match env_builder.open(&path) {
|
|
||||||
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
|
|
||||||
return Err(Error(
|
|
||||||
"OutOfMemory error while trying to open LMDB database. This can happen \
|
|
||||||
if your operating system is not allowing you to use sufficient virtual \
|
|
||||||
memory address space. Please check that no limit is set (ulimit -v). \
|
|
||||||
You may also try to set a smaller `lmdb_map_size` configuration parameter. \
|
|
||||||
On 32-bit machines, you should probably switch to another database engine."
|
|
||||||
.into(),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
|
|
||||||
Ok(db) => Ok(crate::lmdb_adapter::LmdbDb::init(db)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
|
// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
|
||||||
// attribute is added so that we won't have to change this match in case stop building
|
// attribute is added so that we won't have to change this match in case stop building
|
||||||
|
|||||||
@@ -11,12 +11,23 @@ use r2d2_sqlite::SqliteConnectionManager;
|
|||||||
use rusqlite::{params, Rows, Statement, Transaction};
|
use rusqlite::{params, Rows, Statement, Transaction};
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
|
open::{Engine, OpenOpt},
|
||||||
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
|
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
|
||||||
TxResult, TxValueIter, Value, ValueIter,
|
TxResult, TxValueIter, Value, ValueIter,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub use rusqlite;
|
pub use rusqlite;
|
||||||
|
|
||||||
|
// ---- top-level open function
|
||||||
|
|
||||||
|
pub(crate) fn open_db(path: &PathBuf, opt: &OpenOpt) -> Result<Db> {
|
||||||
|
info!("Opening Sqlite database at: {}", path.display());
|
||||||
|
let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
|
||||||
|
Ok(SqliteDb::new(manager, opt.fsync)?)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
|
|
||||||
type Connection = r2d2::PooledConnection<SqliteConnectionManager>;
|
type Connection = r2d2::PooledConnection<SqliteConnectionManager>;
|
||||||
|
|
||||||
// --- err
|
// --- err
|
||||||
@@ -139,17 +150,32 @@ impl IDb for SqliteDb {
|
|||||||
Ok(trees)
|
Ok(trees)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn snapshot(&self, to: &PathBuf) -> Result<()> {
|
fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
|
||||||
fn progress(p: rusqlite::backup::Progress) {
|
fn progress(p: rusqlite::backup::Progress) {
|
||||||
let percent = (p.pagecount - p.remaining) * 100 / p.pagecount;
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
info!("Sqlite snapshot progress: {}%", percent);
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
static LAST_LOG_TIME: AtomicU64 = AtomicU64::new(0);
|
||||||
|
|
||||||
|
let now = SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.expect("Fix your clock :o")
|
||||||
|
.as_millis() as u64;
|
||||||
|
if now >= LAST_LOG_TIME.load(Ordering::Relaxed) + 10 * 1000 {
|
||||||
|
let percent = (p.pagecount - p.remaining) * 100 / p.pagecount;
|
||||||
|
info!("Sqlite snapshot progress: {}%", percent);
|
||||||
|
|
||||||
|
LAST_LOG_TIME.fetch_max(now, Ordering::Relaxed);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
std::fs::create_dir_all(to)?;
|
|
||||||
let mut path = to.clone();
|
std::fs::create_dir_all(base_path)?;
|
||||||
path.push("db.sqlite");
|
let path = Engine::Sqlite.db_path(&base_path);
|
||||||
|
|
||||||
self.db
|
self.db
|
||||||
.get()?
|
.get()?
|
||||||
.backup(rusqlite::DatabaseName::Main, path, Some(progress))?;
|
.backup(rusqlite::DatabaseName::Main, path, Some(progress))?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -160,7 +186,7 @@ impl IDb for SqliteDb {
|
|||||||
self.internal_get(&self.db.get()?, &tree, key)
|
self.internal_get(&self.db.get()?, &tree, key)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn len(&self, tree: usize) -> Result<usize> {
|
fn approximate_len(&self, tree: usize) -> Result<usize> {
|
||||||
let tree = self.get_tree(tree)?;
|
let tree = self.get_tree(tree)?;
|
||||||
let db = self.db.get()?;
|
let db = self.db.get()?;
|
||||||
|
|
||||||
@@ -172,6 +198,10 @@ impl IDb for SqliteDb {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_empty(&self, tree: usize) -> Result<bool> {
|
||||||
|
Ok(self.approximate_len(tree)? == 0)
|
||||||
|
}
|
||||||
|
|
||||||
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
|
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<()> {
|
||||||
let tree = self.get_tree(tree)?;
|
let tree = self.get_tree(tree)?;
|
||||||
let db = self.db.get()?;
|
let db = self.db.get()?;
|
||||||
|
|||||||
+14
-2
@@ -1,7 +1,7 @@
|
|||||||
use crate::*;
|
use crate::*;
|
||||||
|
|
||||||
fn test_suite(db: Db) {
|
fn test_suite(db: Db) {
|
||||||
let tree = db.open_tree("tree").unwrap();
|
let tree = db.open_tree("tree:this_is_a_tree").unwrap();
|
||||||
|
|
||||||
let ka: &[u8] = &b"test"[..];
|
let ka: &[u8] = &b"test"[..];
|
||||||
let kb: &[u8] = &b"zwello"[..];
|
let kb: &[u8] = &b"zwello"[..];
|
||||||
@@ -14,7 +14,7 @@ fn test_suite(db: Db) {
|
|||||||
|
|
||||||
assert!(tree.insert(ka, va).is_ok());
|
assert!(tree.insert(ka, va).is_ok());
|
||||||
assert_eq!(tree.get(ka).unwrap().unwrap(), va);
|
assert_eq!(tree.get(ka).unwrap().unwrap(), va);
|
||||||
assert_eq!(tree.len().unwrap(), 1);
|
assert_eq!(tree.iter().unwrap().count(), 1);
|
||||||
|
|
||||||
// ---- test transaction logic ----
|
// ---- test transaction logic ----
|
||||||
|
|
||||||
@@ -148,3 +148,15 @@ fn test_sqlite_db() {
|
|||||||
let db = SqliteDb::new(manager, false).unwrap();
|
let db = SqliteDb::new(manager, false).unwrap();
|
||||||
test_suite(db);
|
test_suite(db);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[cfg(feature = "fjall")]
|
||||||
|
fn test_fjall_db() {
|
||||||
|
use crate::fjall_adapter::{fjall, FjallDb};
|
||||||
|
|
||||||
|
let path = mktemp::Temp::new_dir().unwrap();
|
||||||
|
let config = fjall::Config::new(path).temporary(true);
|
||||||
|
let keyspace = config.open_transactional().unwrap();
|
||||||
|
let db = FjallDb::init(keyspace);
|
||||||
|
test_suite(db);
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage"
|
name = "garage"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
@@ -49,6 +49,8 @@ structopt.workspace = true
|
|||||||
git-version.workspace = true
|
git-version.workspace = true
|
||||||
|
|
||||||
serde.workspace = true
|
serde.workspace = true
|
||||||
|
serde_json.workspace = true
|
||||||
|
serde_bytes.workspace = true
|
||||||
|
|
||||||
futures.workspace = true
|
futures.workspace = true
|
||||||
tokio.workspace = true
|
tokio.workspace = true
|
||||||
@@ -57,6 +59,7 @@ opentelemetry.workspace = true
|
|||||||
opentelemetry-prometheus = { workspace = true, optional = true }
|
opentelemetry-prometheus = { workspace = true, optional = true }
|
||||||
opentelemetry-otlp = { workspace = true, optional = true }
|
opentelemetry-otlp = { workspace = true, optional = true }
|
||||||
syslog-tracing = { workspace = true, optional = true }
|
syslog-tracing = { workspace = true, optional = true }
|
||||||
|
tracing-journald = { workspace = true, optional = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
garage_api_common.workspace = true
|
garage_api_common.workspace = true
|
||||||
@@ -90,6 +93,7 @@ k2v = [ "garage_util/k2v", "garage_api_k2v" ]
|
|||||||
# Database engines
|
# Database engines
|
||||||
lmdb = [ "garage_model/lmdb" ]
|
lmdb = [ "garage_model/lmdb" ]
|
||||||
sqlite = [ "garage_model/sqlite" ]
|
sqlite = [ "garage_model/sqlite" ]
|
||||||
|
fjall = [ "garage_model/fjall" ]
|
||||||
|
|
||||||
# Automatic registration and discovery via Consul API
|
# Automatic registration and discovery via Consul API
|
||||||
consul-discovery = [ "garage_rpc/consul-discovery" ]
|
consul-discovery = [ "garage_rpc/consul-discovery" ]
|
||||||
@@ -101,6 +105,8 @@ metrics = [ "garage_api_admin/metrics", "opentelemetry-prometheus" ]
|
|||||||
telemetry-otlp = [ "opentelemetry-otlp" ]
|
telemetry-otlp = [ "opentelemetry-otlp" ]
|
||||||
# Logging to syslog
|
# Logging to syslog
|
||||||
syslog = [ "syslog-tracing" ]
|
syslog = [ "syslog-tracing" ]
|
||||||
|
# Logging to journald
|
||||||
|
journald = [ "tracing-journald" ]
|
||||||
|
|
||||||
# NOTE: bundled-libs and system-libs should be treat as mutually exclusive;
|
# NOTE: bundled-libs and system-libs should be treat as mutually exclusive;
|
||||||
# exactly one of them should be enabled.
|
# exactly one of them should be enabled.
|
||||||
|
|||||||
@@ -101,6 +101,7 @@ impl AdminRpcHandler {
|
|||||||
let mut obj_dels = 0;
|
let mut obj_dels = 0;
|
||||||
let mut mpu_dels = 0;
|
let mut mpu_dels = 0;
|
||||||
let mut ver_dels = 0;
|
let mut ver_dels = 0;
|
||||||
|
let mut br_dels = 0;
|
||||||
|
|
||||||
for hash in blocks {
|
for hash in blocks {
|
||||||
let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
|
let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
|
||||||
@@ -131,12 +132,19 @@ impl AdminRpcHandler {
|
|||||||
ver_dels += 1;
|
ver_dels += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !br.deleted.get() {
|
||||||
|
let mut br = br;
|
||||||
|
br.deleted.set();
|
||||||
|
self.garage.block_ref_table.insert(&br).await?;
|
||||||
|
br_dels += 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(AdminRpc::Ok(format!(
|
Ok(AdminRpc::Ok(format!(
|
||||||
"Purged {} blocks, {} versions, {} objects, {} multipart uploads",
|
"Purged {} blocks: marked {} block refs, {} versions, {} objects and {} multipart uploads as deleted",
|
||||||
blocks.len(),
|
blocks.len(),
|
||||||
|
br_dels,
|
||||||
ver_dels,
|
ver_dels,
|
||||||
obj_dels,
|
obj_dels,
|
||||||
mpu_dels,
|
mpu_dels,
|
||||||
|
|||||||
+12
-7
@@ -219,7 +219,7 @@ impl AdminRpcHandler {
|
|||||||
|
|
||||||
// Gather block manager statistics
|
// Gather block manager statistics
|
||||||
writeln!(&mut ret, "\nBlock manager stats:").unwrap();
|
writeln!(&mut ret, "\nBlock manager stats:").unwrap();
|
||||||
let rc_len = self.garage.block_manager.rc_len()?.to_string();
|
let rc_len = self.garage.block_manager.rc_approximate_len()?.to_string();
|
||||||
|
|
||||||
writeln!(
|
writeln!(
|
||||||
&mut ret,
|
&mut ret,
|
||||||
@@ -230,13 +230,13 @@ impl AdminRpcHandler {
|
|||||||
writeln!(
|
writeln!(
|
||||||
&mut ret,
|
&mut ret,
|
||||||
" resync queue length: {}",
|
" resync queue length: {}",
|
||||||
self.garage.block_manager.resync.queue_len()?
|
self.garage.block_manager.resync.queue_approximate_len()?
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
writeln!(
|
writeln!(
|
||||||
&mut ret,
|
&mut ret,
|
||||||
" blocks with resync errors: {}",
|
" blocks with resync errors: {}",
|
||||||
self.garage.block_manager.resync.errors_len()?
|
self.garage.block_manager.resync.errors_approximate_len()?
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
@@ -346,16 +346,21 @@ impl AdminRpcHandler {
|
|||||||
F: TableSchema + 'static,
|
F: TableSchema + 'static,
|
||||||
R: TableReplication + 'static,
|
R: TableReplication + 'static,
|
||||||
{
|
{
|
||||||
let data_len = t.data.store.len().map_err(GarageError::from)?.to_string();
|
let data_len = t
|
||||||
let mkl_len = t.merkle_updater.merkle_tree_len()?.to_string();
|
.data
|
||||||
|
.store
|
||||||
|
.approximate_len()
|
||||||
|
.map_err(GarageError::from)?
|
||||||
|
.to_string();
|
||||||
|
let mkl_len = t.merkle_updater.merkle_tree_approximate_len()?.to_string();
|
||||||
|
|
||||||
Ok(format!(
|
Ok(format!(
|
||||||
" {}\t{}\t{}\t{}\t{}",
|
" {}\t{}\t{}\t{}\t{}",
|
||||||
F::TABLE_NAME,
|
F::TABLE_NAME,
|
||||||
data_len,
|
data_len,
|
||||||
mkl_len,
|
mkl_len,
|
||||||
t.merkle_updater.todo_len()?,
|
t.merkle_updater.todo_approximate_len()?,
|
||||||
t.data.gc_todo_len()?
|
t.data.gc_todo_approximate_len()?
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -71,6 +71,10 @@ pub enum NodeOperation {
|
|||||||
/// Connect to Garage node that is currently isolated from the system
|
/// Connect to Garage node that is currently isolated from the system
|
||||||
#[structopt(name = "connect", version = garage_version())]
|
#[structopt(name = "connect", version = garage_version())]
|
||||||
Connect(ConnectNodeOpt),
|
Connect(ConnectNodeOpt),
|
||||||
|
|
||||||
|
/// Dump the content of a metadata table as JSON lines
|
||||||
|
#[structopt(name = "dump", version = garage_version())]
|
||||||
|
Dump(DumpNodeOpt),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(StructOpt, Debug)]
|
#[derive(StructOpt, Debug)]
|
||||||
@@ -88,6 +92,12 @@ pub struct ConnectNodeOpt {
|
|||||||
pub(crate) node: String,
|
pub(crate) node: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(StructOpt, Debug)]
|
||||||
|
pub struct DumpNodeOpt {
|
||||||
|
/// Name of the data table to dump
|
||||||
|
pub(crate) what: String,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(StructOpt, Debug)]
|
#[derive(StructOpt, Debug)]
|
||||||
pub enum LayoutOperation {
|
pub enum LayoutOperation {
|
||||||
/// Assign role to Garage node
|
/// Assign role to Garage node
|
||||||
|
|||||||
+41
-1
@@ -145,11 +145,14 @@ async fn main() {
|
|||||||
let res = match opt.cmd {
|
let res = match opt.cmd {
|
||||||
Command::Server => server::run_server(opt.config_file, opt.secrets).await,
|
Command::Server => server::run_server(opt.config_file, opt.secrets).await,
|
||||||
Command::OfflineRepair(repair_opt) => {
|
Command::OfflineRepair(repair_opt) => {
|
||||||
repair::offline::offline_repair(opt.config_file, opt.secrets, repair_opt).await
|
repair::offline::offline_repair(opt.config_file, opt.secrets, repair_opt)
|
||||||
}
|
}
|
||||||
Command::ConvertDb(conv_opt) => {
|
Command::ConvertDb(conv_opt) => {
|
||||||
cli::convert_db::do_conversion(conv_opt).map_err(From::from)
|
cli::convert_db::do_conversion(conv_opt).map_err(From::from)
|
||||||
}
|
}
|
||||||
|
Command::Node(NodeOperation::Dump(dump_opt)) => {
|
||||||
|
repair::offline::dump(opt.config_file, opt.secrets, dump_opt)
|
||||||
|
}
|
||||||
Command::Node(NodeOperation::NodeId(node_id_opt)) => {
|
Command::Node(NodeOperation::NodeId(node_id_opt)) => {
|
||||||
node_id_command(opt.config_file, node_id_opt.quiet)
|
node_id_command(opt.config_file, node_id_opt.quiet)
|
||||||
}
|
}
|
||||||
@@ -208,6 +211,43 @@ fn init_logging(opt: &Opt) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if std::env::var("GARAGE_LOG_TO_JOURNALD")
|
||||||
|
.map(|x| x == "1" || x == "true")
|
||||||
|
.unwrap_or(false)
|
||||||
|
{
|
||||||
|
#[cfg(feature = "journald")]
|
||||||
|
{
|
||||||
|
use tracing_journald::{Priority, PriorityMappings};
|
||||||
|
use tracing_subscriber::layer::SubscriberExt;
|
||||||
|
use tracing_subscriber::util::SubscriberInitExt;
|
||||||
|
|
||||||
|
let registry = tracing_subscriber::registry()
|
||||||
|
.with(tracing_subscriber::fmt::layer().with_writer(std::io::sink))
|
||||||
|
.with(env_filter);
|
||||||
|
match tracing_journald::layer() {
|
||||||
|
Ok(layer) => {
|
||||||
|
registry
|
||||||
|
.with(layer.with_priority_mappings(PriorityMappings {
|
||||||
|
info: Priority::Informational,
|
||||||
|
debug: Priority::Debug,
|
||||||
|
..PriorityMappings::new()
|
||||||
|
}))
|
||||||
|
.init();
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("Couldn't connect to journald: {}.", e);
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "journald"))]
|
||||||
|
{
|
||||||
|
eprintln!("Journald support is not enabled in this build.");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
tracing_subscriber::fmt()
|
tracing_subscriber::fmt()
|
||||||
.with_writer(std::io::stderr)
|
.with_writer(std::io::stderr)
|
||||||
.with_env_filter(env_filter)
|
.with_env_filter(env_filter)
|
||||||
|
|||||||
@@ -1,14 +1,18 @@
|
|||||||
|
use std::io::Write;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
use garage_util::config::*;
|
use garage_util::config::*;
|
||||||
use garage_util::error::*;
|
use garage_util::error::*;
|
||||||
|
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
|
use garage_table::{replication::TableReplication, *};
|
||||||
|
|
||||||
use crate::cli::structs::*;
|
use crate::cli::structs::*;
|
||||||
use crate::secrets::{fill_secrets, Secrets};
|
use crate::secrets::{fill_secrets, Secrets};
|
||||||
|
|
||||||
pub async fn offline_repair(
|
pub fn offline_repair(
|
||||||
config_file: PathBuf,
|
config_file: PathBuf,
|
||||||
secrets: Secrets,
|
secrets: Secrets,
|
||||||
opt: OfflineRepairOpt,
|
opt: OfflineRepairOpt,
|
||||||
@@ -45,3 +49,100 @@ pub async fn offline_repair(
|
|||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn dump(config_file: PathBuf, secrets: Secrets, opt: DumpNodeOpt) -> Result<(), Error> {
|
||||||
|
let what = opt.what.as_str();
|
||||||
|
|
||||||
|
info!("Loading configuration...");
|
||||||
|
let config = fill_secrets(read_config(config_file)?, secrets)?;
|
||||||
|
|
||||||
|
info!("Initializing Garage main data store...");
|
||||||
|
let garage = Garage::new(config)?;
|
||||||
|
|
||||||
|
match what {
|
||||||
|
"bucket" | "buckets" => dump_table_inner(&garage.bucket_table),
|
||||||
|
"bucket_alias" | "bucket_aliases" => dump_table_inner(&garage.bucket_alias_table),
|
||||||
|
"key" | "keys" => dump_table_inner(&garage.key_table),
|
||||||
|
"object" | "objects" => dump_table_inner(&garage.object_table),
|
||||||
|
"object_counter" | "object_counters" => Err(Error::Message(
|
||||||
|
"object_counters cannot be JSON-serialized".into(),
|
||||||
|
)),
|
||||||
|
"mpu" => dump_table_inner(&garage.mpu_table),
|
||||||
|
"mpu_counter" | "mpu_counters" => Err(Error::Message(
|
||||||
|
"mpu_counters cannot be JSON-serialized".into(),
|
||||||
|
)),
|
||||||
|
"version" | "versions" => dump_table_inner(&garage.version_table),
|
||||||
|
"block_ref" | "block_refs" => dump_table_inner(&garage.block_ref_table),
|
||||||
|
#[cfg(feature = "k2v")]
|
||||||
|
"k2v_item" | "k2v_items" => dump_table_inner(&garage.k2v.item_table),
|
||||||
|
//#[cfg(feature = "k2v")]
|
||||||
|
"k2v_counter" | "k2v_counters" => Err(Error::Message(
|
||||||
|
"k2v_counters cannot be JSON-serialized".into(),
|
||||||
|
)),
|
||||||
|
other => {
|
||||||
|
let mut stdout = std::io::stdout().lock();
|
||||||
|
match other {
|
||||||
|
"cluster_layout" => Err(Error::Message(
|
||||||
|
"cluster_layout cannot be JSON-serialized".into(),
|
||||||
|
)),
|
||||||
|
_ => Err(Error::Message(format!("invalid thing to dump: {}", what))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct DumpEntry<'a, T: Serialize> {
|
||||||
|
#[serde(with = "serde_bytes")]
|
||||||
|
partition_key: &'a [u8],
|
||||||
|
#[serde(with = "serde_bytes")]
|
||||||
|
sort_key: &'a [u8],
|
||||||
|
entry: &'a T,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dump_table_inner<F, R>(table: &Table<F, R>) -> Result<(), Error>
|
||||||
|
where
|
||||||
|
F: TableSchema,
|
||||||
|
R: TableReplication,
|
||||||
|
{
|
||||||
|
eprintln!("Dumping table {}...", F::TABLE_NAME);
|
||||||
|
|
||||||
|
let mut stdout = std::io::stdout().lock();
|
||||||
|
|
||||||
|
for line in table.data.store.iter()? {
|
||||||
|
let (_k, v) = line?;
|
||||||
|
let v_dec = table.data.decode_entry(&v)?;
|
||||||
|
let pkh = v_dec.partition_key().hash();
|
||||||
|
let dump_entry = DumpEntry {
|
||||||
|
partition_key: pkh.as_slice(),
|
||||||
|
sort_key: v_dec.sort_key().sort_key(),
|
||||||
|
entry: &v_dec,
|
||||||
|
};
|
||||||
|
dump_line(&mut stdout, dump_entry)?;
|
||||||
|
}
|
||||||
|
stdout.flush()?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dump_line<T: Serialize>(
|
||||||
|
mut stdout: &mut std::io::StdoutLock<'static>,
|
||||||
|
dump_entry: T,
|
||||||
|
) -> Result<(), Error> {
|
||||||
|
let mut ser = serde_json::ser::Serializer::with_formatter(&mut stdout, DumpFormatter);
|
||||||
|
dump_entry.serialize(&mut ser)?;
|
||||||
|
stdout.write_all(b"\n")?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
struct DumpFormatter;
|
||||||
|
impl serde_json::ser::Formatter for DumpFormatter {
|
||||||
|
fn write_byte_array<W>(&mut self, writer: &mut W, value: &[u8]) -> std::io::Result<()>
|
||||||
|
where
|
||||||
|
W: ?Sized + std::io::Write,
|
||||||
|
{
|
||||||
|
writer.write_all(b"\"")?;
|
||||||
|
writer.write_all(hex::encode(&value).as_bytes())?;
|
||||||
|
writer.write_all(b"\"")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+15
-4
@@ -183,10 +183,21 @@ fn watch_shutdown_signal() -> watch::Receiver<bool> {
|
|||||||
let mut sigterm =
|
let mut sigterm =
|
||||||
signal(SignalKind::terminate()).expect("Failed to install SIGTERM handler");
|
signal(SignalKind::terminate()).expect("Failed to install SIGTERM handler");
|
||||||
let mut sighup = signal(SignalKind::hangup()).expect("Failed to install SIGHUP handler");
|
let mut sighup = signal(SignalKind::hangup()).expect("Failed to install SIGHUP handler");
|
||||||
tokio::select! {
|
loop {
|
||||||
_ = sigint.recv() => info!("Received SIGINT, shutting down."),
|
tokio::select! {
|
||||||
_ = sigterm.recv() => info!("Received SIGTERM, shutting down."),
|
_ = sigint.recv() => {
|
||||||
_ = sighup.recv() => info!("Received SIGHUP, shutting down."),
|
info!("Received SIGINT, shutting down.");
|
||||||
|
break
|
||||||
|
}
|
||||||
|
_ = sigterm.recv() => {
|
||||||
|
info!("Received SIGTERM, shutting down.");
|
||||||
|
break
|
||||||
|
}
|
||||||
|
_ = sighup.recv() => {
|
||||||
|
info!("Received SIGHUP, reload not supported.");
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
send_cancel.send(true).unwrap();
|
send_cancel.send(true).unwrap();
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -606,3 +606,45 @@ async fn test_website_puny() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_website_object_not_found() {
|
||||||
|
const BCKT_NAME: &str = "not-found";
|
||||||
|
let ctx = common::context();
|
||||||
|
let _bucket = ctx.create_bucket(BCKT_NAME);
|
||||||
|
|
||||||
|
let client = Client::builder(TokioExecutor::new()).build_http();
|
||||||
|
|
||||||
|
let req = |suffix| {
|
||||||
|
Request::builder()
|
||||||
|
.method("GET")
|
||||||
|
.uri(format!("http://127.0.0.1:{}/", ctx.garage.web_port))
|
||||||
|
.header("Host", format!("{}{}", BCKT_NAME, suffix))
|
||||||
|
.body(Body::new(Bytes::new()))
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
|
|
||||||
|
ctx.garage
|
||||||
|
.command()
|
||||||
|
.args(["bucket", "website", "--allow", BCKT_NAME])
|
||||||
|
.quiet()
|
||||||
|
.expect_success_status("Could not allow website on bucket");
|
||||||
|
|
||||||
|
let resp = client.request(req("")).await.unwrap();
|
||||||
|
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
|
||||||
|
// the error we return by default are *not* xml
|
||||||
|
assert_eq!(
|
||||||
|
resp.headers().get(http::header::CONTENT_TYPE).unwrap(),
|
||||||
|
"text/html; charset=utf-8"
|
||||||
|
);
|
||||||
|
let result = String::from_utf8(
|
||||||
|
resp.into_body()
|
||||||
|
.collect()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.to_bytes()
|
||||||
|
.to_vec(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(result.contains("not found"));
|
||||||
|
}
|
||||||
|
|||||||
@@ -72,6 +72,16 @@ impl K2vClient {
|
|||||||
.enable_http2()
|
.enable_http2()
|
||||||
.build();
|
.build();
|
||||||
let client = HttpClient::builder(TokioExecutor::new()).build(connector);
|
let client = HttpClient::builder(TokioExecutor::new()).build(connector);
|
||||||
|
Self::new_with_client(config, client)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create a new K2V client with an external client.
|
||||||
|
/// Useful for example if you plan on creating many clients but you want to mutualize the
|
||||||
|
/// underlying thread pools & co.
|
||||||
|
pub fn new_with_client(
|
||||||
|
config: K2vClientConfig,
|
||||||
|
client: HttpClient<HttpsConnector<HttpConnector>, Body>,
|
||||||
|
) -> Result<Self, Error> {
|
||||||
let user_agent: std::borrow::Cow<str> = match &config.user_agent {
|
let user_agent: std::borrow::Cow<str> = match &config.user_agent {
|
||||||
Some(ua) => ua.into(),
|
Some(ua) => ua.into(),
|
||||||
None => format!("k2v/{}", env!("CARGO_PKG_VERSION")).into(),
|
None => format!("k2v/{}", env!("CARGO_PKG_VERSION")).into(),
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_model"
|
name = "garage_model"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
@@ -44,3 +44,4 @@ default = [ "lmdb", "sqlite" ]
|
|||||||
k2v = [ "garage_util/k2v" ]
|
k2v = [ "garage_util/k2v" ]
|
||||||
lmdb = [ "garage_db/lmdb" ]
|
lmdb = [ "garage_db/lmdb" ]
|
||||||
sqlite = [ "garage_db/sqlite" ]
|
sqlite = [ "garage_db/sqlite" ]
|
||||||
|
fjall = [ "garage_db/fjall" ]
|
||||||
+5
-9
@@ -116,21 +116,17 @@ impl Garage {
|
|||||||
info!("Opening database...");
|
info!("Opening database...");
|
||||||
let db_engine = db::Engine::from_str(&config.db_engine)
|
let db_engine = db::Engine::from_str(&config.db_engine)
|
||||||
.ok_or_message("Invalid `db_engine` value in configuration file")?;
|
.ok_or_message("Invalid `db_engine` value in configuration file")?;
|
||||||
let mut db_path = config.metadata_dir.clone();
|
let db_path = db_engine.db_path(&config.metadata_dir);
|
||||||
match db_engine {
|
|
||||||
db::Engine::Sqlite => {
|
|
||||||
db_path.push("db.sqlite");
|
|
||||||
}
|
|
||||||
db::Engine::Lmdb => {
|
|
||||||
db_path.push("db.lmdb");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let db_opt = db::OpenOpt {
|
let db_opt = db::OpenOpt {
|
||||||
fsync: config.metadata_fsync,
|
fsync: config.metadata_fsync,
|
||||||
lmdb_map_size: match config.lmdb_map_size {
|
lmdb_map_size: match config.lmdb_map_size {
|
||||||
v if v == usize::default() => None,
|
v if v == usize::default() => None,
|
||||||
v => Some(v),
|
v => Some(v),
|
||||||
},
|
},
|
||||||
|
fjall_block_cache_size: match config.fjall_block_cache_size {
|
||||||
|
v if v == usize::default() => None,
|
||||||
|
v => Some(v),
|
||||||
|
},
|
||||||
};
|
};
|
||||||
let db = db::open_db(&db_path, db_engine, &db_opt)
|
let db = db::open_db(&db_path, db_engine, &db_opt)
|
||||||
.ok_or_message("Unable to open metadata db")?;
|
.ok_or_message("Unable to open metadata db")?;
|
||||||
|
|||||||
@@ -121,13 +121,13 @@ impl Worker for LifecycleWorker {
|
|||||||
mpu_aborted,
|
mpu_aborted,
|
||||||
..
|
..
|
||||||
} => {
|
} => {
|
||||||
let n_objects = self.garage.object_table.data.store.len().ok();
|
let n_objects = self.garage.object_table.data.store.approximate_len().ok();
|
||||||
let progress = match n_objects {
|
let progress = match n_objects {
|
||||||
None => "...".to_string(),
|
Some(total) if total > 0 => format!(
|
||||||
Some(total) => format!(
|
|
||||||
"~{:.2}%",
|
"~{:.2}%",
|
||||||
100. * std::cmp::min(*counter, total) as f32 / total as f32
|
100. * std::cmp::min(*counter, total) as f32 / total as f32
|
||||||
),
|
),
|
||||||
|
_ => "...".to_string(),
|
||||||
};
|
};
|
||||||
WorkerStatus {
|
WorkerStatus {
|
||||||
progress: Some(progress),
|
progress: Some(progress),
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_net"
|
name = "garage_net"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_rpc"
|
name = "garage_rpc"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_table"
|
name = "garage_table"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|||||||
+2
-2
@@ -367,7 +367,7 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn gc_todo_len(&self) -> Result<usize, Error> {
|
pub fn gc_todo_approximate_len(&self) -> Result<usize, Error> {
|
||||||
Ok(self.gc_todo.len()?)
|
Ok(self.gc_todo.approximate_len()?)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -313,7 +313,7 @@ impl<F: TableSchema, R: TableReplication> Worker for GcWorker<F, R> {
|
|||||||
|
|
||||||
fn status(&self) -> WorkerStatus {
|
fn status(&self) -> WorkerStatus {
|
||||||
WorkerStatus {
|
WorkerStatus {
|
||||||
queue_length: Some(self.gc.data.gc_todo_len().unwrap_or(0) as u64),
|
queue_length: Some(self.gc.data.gc_todo_approximate_len().unwrap_or(0) as u64),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+5
-5
@@ -287,12 +287,12 @@ impl<F: TableSchema, R: TableReplication> MerkleUpdater<F, R> {
|
|||||||
MerkleNode::decode_opt(&ent)
|
MerkleNode::decode_opt(&ent)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn merkle_tree_len(&self) -> Result<usize, Error> {
|
pub fn merkle_tree_approximate_len(&self) -> Result<usize, Error> {
|
||||||
Ok(self.data.merkle_tree.len()?)
|
Ok(self.data.merkle_tree.approximate_len()?)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn todo_len(&self) -> Result<usize, Error> {
|
pub fn todo_approximate_len(&self) -> Result<usize, Error> {
|
||||||
Ok(self.data.merkle_todo.len()?)
|
Ok(self.data.merkle_todo.approximate_len()?)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -306,7 +306,7 @@ impl<F: TableSchema, R: TableReplication> Worker for MerkleWorker<F, R> {
|
|||||||
|
|
||||||
fn status(&self) -> WorkerStatus {
|
fn status(&self) -> WorkerStatus {
|
||||||
WorkerStatus {
|
WorkerStatus {
|
||||||
queue_length: Some(self.0.todo_len().unwrap_or(0) as u64),
|
queue_length: Some(self.0.todo_approximate_len().unwrap_or(0) as u64),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ impl TableMetrics {
|
|||||||
.u64_value_observer(
|
.u64_value_observer(
|
||||||
"table.size",
|
"table.size",
|
||||||
move |observer| {
|
move |observer| {
|
||||||
if let Ok(value) = store.len() {
|
if let Ok(value) = store.approximate_len() {
|
||||||
observer.observe(
|
observer.observe(
|
||||||
value as u64,
|
value as u64,
|
||||||
&[KeyValue::new("table_name", table_name)],
|
&[KeyValue::new("table_name", table_name)],
|
||||||
@@ -48,7 +48,7 @@ impl TableMetrics {
|
|||||||
.u64_value_observer(
|
.u64_value_observer(
|
||||||
"table.merkle_tree_size",
|
"table.merkle_tree_size",
|
||||||
move |observer| {
|
move |observer| {
|
||||||
if let Ok(value) = merkle_tree.len() {
|
if let Ok(value) = merkle_tree.approximate_len() {
|
||||||
observer.observe(
|
observer.observe(
|
||||||
value as u64,
|
value as u64,
|
||||||
&[KeyValue::new("table_name", table_name)],
|
&[KeyValue::new("table_name", table_name)],
|
||||||
@@ -62,7 +62,7 @@ impl TableMetrics {
|
|||||||
.u64_value_observer(
|
.u64_value_observer(
|
||||||
"table.merkle_updater_todo_queue_length",
|
"table.merkle_updater_todo_queue_length",
|
||||||
move |observer| {
|
move |observer| {
|
||||||
if let Ok(v) = merkle_todo.len() {
|
if let Ok(v) = merkle_todo.approximate_len() {
|
||||||
observer.observe(
|
observer.observe(
|
||||||
v as u64,
|
v as u64,
|
||||||
&[KeyValue::new("table_name", table_name)],
|
&[KeyValue::new("table_name", table_name)],
|
||||||
@@ -76,7 +76,7 @@ impl TableMetrics {
|
|||||||
.u64_value_observer(
|
.u64_value_observer(
|
||||||
"table.gc_todo_queue_length",
|
"table.gc_todo_queue_length",
|
||||||
move |observer| {
|
move |observer| {
|
||||||
if let Ok(value) = gc_todo.len() {
|
if let Ok(value) = gc_todo.approximate_len() {
|
||||||
observer.observe(
|
observer.observe(
|
||||||
value as u64,
|
value as u64,
|
||||||
&[KeyValue::new("table_name", table_name)],
|
&[KeyValue::new("table_name", table_name)],
|
||||||
|
|||||||
+1
-1
@@ -27,7 +27,7 @@ impl<F: TableSchema, R: TableReplication> Worker for InsertQueueWorker<F, R> {
|
|||||||
|
|
||||||
fn status(&self) -> WorkerStatus {
|
fn status(&self) -> WorkerStatus {
|
||||||
WorkerStatus {
|
WorkerStatus {
|
||||||
queue_length: Some(self.0.data.insert_queue.len().unwrap_or(0) as u64),
|
queue_length: Some(self.0.data.insert_queue.approximate_len().unwrap_or(0) as u64),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_util"
|
name = "garage_util"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|||||||
@@ -122,6 +122,10 @@ pub struct Config {
|
|||||||
#[serde(deserialize_with = "deserialize_capacity", default)]
|
#[serde(deserialize_with = "deserialize_capacity", default)]
|
||||||
pub lmdb_map_size: usize,
|
pub lmdb_map_size: usize,
|
||||||
|
|
||||||
|
/// Fjall block cache size
|
||||||
|
#[serde(deserialize_with = "deserialize_capacity", default)]
|
||||||
|
pub fjall_block_cache_size: usize,
|
||||||
|
|
||||||
// -- APIs
|
// -- APIs
|
||||||
/// Configuration for S3 api
|
/// Configuration for S3 api
|
||||||
pub s3_api: S3ApiConfig,
|
pub s3_api: S3ApiConfig,
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "garage_web"
|
name = "garage_web"
|
||||||
version = "1.1.0"
|
version = "1.2.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
|
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|||||||
+22
-2
@@ -397,10 +397,30 @@ fn error_to_res(e: Error) -> Response<BoxBody<Error>> {
|
|||||||
// was a HEAD request or we couldn't get the error document)
|
// was a HEAD request or we couldn't get the error document)
|
||||||
// We do NOT enter this code path when returning the bucket's
|
// We do NOT enter this code path when returning the bucket's
|
||||||
// error document (this is handled in serve_file)
|
// error document (this is handled in serve_file)
|
||||||
let body = string_body(format!("{}\n", e));
|
let mut body_str = format!(
|
||||||
let mut http_error = Response::new(body);
|
r"<title>{http_code} {code_text}</title>
|
||||||
|
<h1>{http_code} {code_text}</h1>",
|
||||||
|
http_code = e.http_status_code().as_u16(),
|
||||||
|
code_text = e.http_status_code().canonical_reason().unwrap_or("Unknown"),
|
||||||
|
);
|
||||||
|
if let Error::ApiError(ref err) = e {
|
||||||
|
body_str.push_str(&format!(
|
||||||
|
r"
|
||||||
|
<ul>
|
||||||
|
<li>Code: {s3_code}</li>
|
||||||
|
<li>Message: {s3_message}.</li>
|
||||||
|
</ul>",
|
||||||
|
s3_code = err.aws_code(),
|
||||||
|
s3_message = err,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut http_error = Response::new(string_body(body_str));
|
||||||
*http_error.status_mut() = e.http_status_code();
|
*http_error.status_mut() = e.http_status_code();
|
||||||
e.add_headers(http_error.headers_mut());
|
e.add_headers(http_error.headers_mut());
|
||||||
|
http_error.headers_mut().insert(
|
||||||
|
http::header::CONTENT_TYPE,
|
||||||
|
"text/html; charset=utf-8".parse().unwrap(),
|
||||||
|
);
|
||||||
http_error
|
http_error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user